|
|
@@ -0,0 +1,314 @@
|
|
|
+package admin
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "github.com/go-nunu/nunu-layout-advanced/internal/service"
|
|
|
+ "github.com/tidwall/gjson"
|
|
|
+ "go.uber.org/zap"
|
|
|
+)
|
|
|
+
|
|
|
+type WafLogDataCleanService interface {
|
|
|
+ ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData
|
|
|
+}
|
|
|
+func NewWafLogDataCleanService(
|
|
|
+ service *service.Service,
|
|
|
+) WafLogDataCleanService {
|
|
|
+ return &wafLogDataCleanService{
|
|
|
+ Service: service,
|
|
|
+
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+type wafLogDataCleanService struct {
|
|
|
+ *service.Service
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+// BackendInfo 后端服务器信息
|
|
|
+type BackendInfo struct {
|
|
|
+ Addr string `json:"addr,omitempty"` // 后端地址
|
|
|
+ CustomHost string `json:"customHost,omitempty"` // 自定义Host头
|
|
|
+}
|
|
|
+
|
|
|
+// CleanedExtraData 使用动态结构存储解析后的数据
|
|
|
+type CleanedExtraData struct {
|
|
|
+ // 核心字段 - 新的数组结构
|
|
|
+ BackendList []BackendInfo `json:"backendList,omitempty"` // 完整的后端信息数组
|
|
|
+
|
|
|
+ // 向后兼容字段
|
|
|
+ AddrBackendList []string `json:"addrBackendList,omitempty"` // 只包含地址的数组
|
|
|
+ CustomHostList []string `json:"customHostList,omitempty"` // 只包含customHost的数组
|
|
|
+ CustomHost []string `json:"customHost,omitempty"` // customHost数组，与CustomHostList相同
|
|
|
+
|
|
|
+ // 基础字段
|
|
|
+ Port string `json:"port,omitempty"`
|
|
|
+ Domain string `json:"domain,omitempty"`
|
|
|
+ Comment string `json:"comment,omitempty"`
|
|
|
+
|
|
|
+ // 扩展字段
|
|
|
+ UID int64 `json:"uid,omitempty"`
|
|
|
+ HostID int64 `json:"hostId,omitempty"`
|
|
|
+ Proxy bool `json:"proxy,omitempty"`
|
|
|
+ IsHttps int `json:"isHttps,omitempty"`
|
|
|
+
|
|
|
+ // 动态字段存储，用于存储任意其他字段
|
|
|
+ DynamicFields map[string]interface{} `json:"dynamicFields,omitempty"`
|
|
|
+
|
|
|
+ // 原始数据备份，用于调试和回溯
|
|
|
+ RawData map[string]interface{} `json:"rawData,omitempty"`
|
|
|
+}
|
|
|
+
|
|
|
+// backendInfo 用于解析 "web" 类型中 backendList JSON 字符串的内部结构
|
|
|
+type backendInfo struct {
|
|
|
+ Addr string `json:"addr"`
|
|
|
+ CustomHost string `json:"customHost"`
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+// parseWafLogExtraData 使用gjson解析动态JSON结构，简洁高效
|
|
|
+func (s *wafLogDataCleanService) ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData {
|
|
|
+ var result CleanedExtraData
|
|
|
+ result.DynamicFields = make(map[string]interface{})
|
|
|
+
|
|
|
+ if len(extraDataBytes) == 0 {
|
|
|
+ return result
|
|
|
+ }
|
|
|
+
|
|
|
+ jsonStr := string(extraDataBytes)
|
|
|
+ if !gjson.Valid(jsonStr) {
|
|
|
+ s.Logger.Warn("ExtraData 不是有效的JSON", zap.String("raw_data", jsonStr))
|
|
|
+ return result
|
|
|
+ }
|
|
|
+
|
|
|
+ // 解析并保存原始数据
|
|
|
+ var rawData map[string]interface{}
|
|
|
+ json.Unmarshal(extraDataBytes, &rawData)
|
|
|
+ result.RawData = rawData
|
|
|
+
|
|
|
+ // 使用gjson进行智能字段提取
|
|
|
+ s.extractWithGjson(jsonStr, apiName, &result)
|
|
|
+
|
|
|
+ return result
|
|
|
+}
|
|
|
+
|
|
|
+// extractWithGjson 使用gjson进行智能字段提取
|
|
|
+func (s *wafLogDataCleanService) extractWithGjson(jsonStr, apiName string, result *CleanedExtraData) {
|
|
|
+ // 提取顶层字段
|
|
|
+ if uid := gjson.Get(jsonStr, "uid"); uid.Exists() {
|
|
|
+ result.UID = uid.Int()
|
|
|
+ }
|
|
|
+ if hostId := gjson.Get(jsonStr, "hostId"); hostId.Exists() {
|
|
|
+ result.HostID = hostId.Int()
|
|
|
+ }
|
|
|
+
|
|
|
+ // 定义常见字段路径的优先级列表
|
|
|
+ fieldPaths := map[string][]string{
|
|
|
+ "comment": {"comment", "data.comment", "desc", "description", "remark", "note"},
|
|
|
+ "port": {"port", "data.port", "config.port", "server.port"},
|
|
|
+ "domain": {"domain", "data.domain", "host", "data.host", "hostname"},
|
|
|
+ "proxy": {"proxy", "data.proxy"},
|
|
|
+ "isHttps": {"isHttps", "data.isHttps"},
|
|
|
+ }
|
|
|
+
|
|
|
+ // 提取基础字段
|
|
|
+ for fieldName, paths := range fieldPaths {
|
|
|
+ value := s.getFirstValidPath(jsonStr, paths)
|
|
|
+ if value != "" {
|
|
|
+ switch fieldName {
|
|
|
+ case "comment":
|
|
|
+ result.Comment = value
|
|
|
+ case "port":
|
|
|
+ result.Port = value
|
|
|
+ case "domain":
|
|
|
+ result.Domain = value
|
|
|
+ case "proxy":
|
|
|
+ result.Proxy = gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Bool()
|
|
|
+ case "isHttps":
|
|
|
+ result.IsHttps = int(gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Int())
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 智能提取 backendList
|
|
|
+ s.extractBackendListWithGjson(jsonStr, apiName, result)
|
|
|
+
|
|
|
+ // 提取所有其他动态字段
|
|
|
+ s.extractDynamicFields(jsonStr, result)
|
|
|
+}
|
|
|
+
|
|
|
+// getFirstValidPath 从多个路径中获取第一个有效值
|
|
|
+func (s *wafLogDataCleanService) getFirstValidPath(jsonStr string, paths []string) string {
|
|
|
+ for _, path := range paths {
|
|
|
+ if value := gjson.Get(jsonStr, path); value.Exists() && value.String() != "" {
|
|
|
+ return value.String()
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return ""
|
|
|
+}
|
|
|
+
|
|
|
+// getFirstValidPathName 获取第一个有效路径的名称
|
|
|
+func (s *wafLogDataCleanService) getFirstValidPathName(jsonStr string, paths []string) string {
|
|
|
+ for _, path := range paths {
|
|
|
+ if gjson.Get(jsonStr, path).Exists() {
|
|
|
+ return path
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return ""
|
|
|
+}
|
|
|
+
|
|
|
+// extractBackendListWithGjson 使用gjson智能提取后端列表
|
|
|
+func (s *wafLogDataCleanService) extractBackendListWithGjson(jsonStr, apiName string, result *CleanedExtraData) {
|
|
|
+ // 定义可能的后端列表字段路径
|
|
|
+ backendPaths := []string{
|
|
|
+ "backendList", "data.backendList", "backends", "data.backends",
|
|
|
+ "backend_list", "data.backend_list", "servers", "data.servers",
|
|
|
+ "upstreams", "data.upstreams", "targets", "data.targets",
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, path := range backendPaths {
|
|
|
+ backendResult := gjson.Get(jsonStr, path)
|
|
|
+ if !backendResult.Exists() {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ // 根据数据类型进行处理
|
|
|
+ switch {
|
|
|
+ case backendResult.IsArray():
|
|
|
+ s.processArrayBackends(backendResult, result)
|
|
|
+ case backendResult.IsObject():
|
|
|
+ s.processObjectBackends(backendResult, result)
|
|
|
+ default:
|
|
|
+ // 字符串类型，可能是JSON字符串或单个地址
|
|
|
+ s.processStringBackends(backendResult.String(), result)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 找到有效数据就退出
|
|
|
+ if len(result.AddrBackendList) > 0 || len(result.CustomHostList) > 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// processArrayBackends 处理数组格式的后端列表
|
|
|
+func (s *wafLogDataCleanService) processArrayBackends(backendResult gjson.Result, result *CleanedExtraData) {
|
|
|
+ backendResult.ForEach(func(key, value gjson.Result) bool {
|
|
|
+ if value.IsObject() {
|
|
|
+ // 创建BackendInfo结构
|
|
|
+ backend := BackendInfo{}
|
|
|
+
|
|
|
+ // 尝试提取地址字段
|
|
|
+ addr := s.getFirstValidPath(value.Raw, []string{"addr", "address", "host", "server", "endpoint", "url"})
|
|
|
+ if addr != "" {
|
|
|
+ backend.Addr = addr
|
|
|
+ result.AddrBackendList = append(result.AddrBackendList, addr)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 提取customHost
|
|
|
+ if customHost := gjson.Get(value.Raw, "customHost").String(); customHost != "" {
|
|
|
+ backend.CustomHost = customHost
|
|
|
+ result.CustomHostList = append(result.CustomHostList, customHost)
|
|
|
+ result.CustomHost = append(result.CustomHost, customHost)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 只有当有有效数据时才添加到BackendList
|
|
|
+ if backend.Addr != "" || backend.CustomHost != "" {
|
|
|
+ result.BackendList = append(result.BackendList, backend)
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 直接作为地址处理
|
|
|
+ if addr := value.String(); addr != "" {
|
|
|
+ result.AddrBackendList = append(result.AddrBackendList, addr)
|
|
|
+ result.BackendList = append(result.BackendList, BackendInfo{Addr: addr})
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+// processObjectBackends 处理对象格式的后端列表
|
|
|
+func (s *wafLogDataCleanService) processObjectBackends(backendResult gjson.Result, result *CleanedExtraData) {
|
|
|
+ // 创建BackendInfo结构
|
|
|
+ backend := BackendInfo{}
|
|
|
+
|
|
|
+ addr := s.getFirstValidPath(backendResult.Raw, []string{"addr", "address", "host", "server", "endpoint", "url"})
|
|
|
+ if addr != "" {
|
|
|
+ backend.Addr = addr
|
|
|
+ result.AddrBackendList = append(result.AddrBackendList, addr)
|
|
|
+ }
|
|
|
+
|
|
|
+ if customHost := gjson.Get(backendResult.Raw, "customHost").String(); customHost != "" {
|
|
|
+ backend.CustomHost = customHost
|
|
|
+ result.CustomHostList = append(result.CustomHostList, customHost)
|
|
|
+ result.CustomHost = append(result.CustomHost, customHost)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 只有当有有效数据时才添加到BackendList
|
|
|
+ if backend.Addr != "" || backend.CustomHost != "" {
|
|
|
+ result.BackendList = append(result.BackendList, backend)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// processStringBackends 处理字符串格式的后端列表
|
|
|
+func (s *wafLogDataCleanService) processStringBackends(backendStr string, result *CleanedExtraData) {
|
|
|
+ if backendStr == "" {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 尝试作为JSON解析
|
|
|
+ if gjson.Valid(backendStr) {
|
|
|
+ parsed := gjson.Parse(backendStr)
|
|
|
+ if parsed.IsArray() {
|
|
|
+ s.processArrayBackends(parsed, result)
|
|
|
+ return
|
|
|
+ } else if parsed.IsObject() {
|
|
|
+ s.processObjectBackends(parsed, result)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 作为单个地址处理
|
|
|
+ result.AddrBackendList = append(result.AddrBackendList, backendStr)
|
|
|
+ result.BackendList = append(result.BackendList, BackendInfo{Addr: backendStr})
|
|
|
+}
|
|
|
+
|
|
|
+// extractDynamicFields 提取所有动态字段到DynamicFields中
|
|
|
+func (s *wafLogDataCleanService) extractDynamicFields(jsonStr string, result *CleanedExtraData) {
|
|
|
+ // 已知的核心字段，不放入动态字段中
|
|
|
+ knownFields := map[string]bool{
|
|
|
+ "comment": true, "port": true, "domain": true,
|
|
|
+ "backendList": true, "backends": true, "backend_list": true,
|
|
|
+ "data": true, // data字段的内容会被单独处理
|
|
|
+ }
|
|
|
+
|
|
|
+ // 遍历顶层字段
|
|
|
+ gjson.Parse(jsonStr).ForEach(func(key, value gjson.Result) bool {
|
|
|
+ fieldName := key.String()
|
|
|
+ if !knownFields[fieldName] {
|
|
|
+ // 将未知字段存储到动态字段中
|
|
|
+ result.DynamicFields[fieldName] = value.Value()
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
+
|
|
|
+ // 特殊处理data字段中的未知字段
|
|
|
+ dataResult := gjson.Get(jsonStr, "data")
|
|
|
+ if dataResult.Exists() && dataResult.IsObject() {
|
|
|
+ dataKnownFields := map[string]bool{
|
|
|
+ "port": true, "domain": true, "backendList": true,
|
|
|
+ "backends": true, "backend_list": true,
|
|
|
+ }
|
|
|
+
|
|
|
+ dataResult.ForEach(func(key, value gjson.Result) bool {
|
|
|
+ fieldName := key.String()
|
|
|
+ if !dataKnownFields[fieldName] {
|
|
|
+ // 使用data.前缀避免冲突
|
|
|
+ result.DynamicFields["data."+fieldName] = value.Value()
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
