package admin import ( "encoding/json" "github.com/go-nunu/nunu-layout-advanced/internal/service" "github.com/tidwall/gjson" "go.uber.org/zap" ) type WafLogDataCleanService interface { ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData } func NewWafLogDataCleanService( service *service.Service, ) WafLogDataCleanService { return &wafLogDataCleanService{ Service: service, } } type wafLogDataCleanService struct { *service.Service } // BackendInfo 后端服务器信息 type BackendInfo struct { Addr string `json:"addr,omitempty"` // 后端地址 CustomHost string `json:"customHost,omitempty"` // 自定义Host头 } // CleanedExtraData 使用动态结构存储解析后的数据 type CleanedExtraData struct { // 核心字段 - 新的数组结构 BackendList []BackendInfo `json:"backendList,omitempty"` // 完整的后端信息数组 // 向后兼容字段 AddrBackendList []string `json:"addrBackendList,omitempty"` // 只包含地址的数组 CustomHostList []string `json:"customHostList,omitempty"` // 只包含customHost的数组 CustomHost []string `json:"customHost,omitempty"` // customHost数组,与CustomHostList相同 // 基础字段 Port string `json:"port,omitempty"` Domain string `json:"domain,omitempty"` Comment string `json:"comment,omitempty"` // 扩展字段 UID int64 `json:"uid,omitempty"` HostID int64 `json:"hostId,omitempty"` Proxy bool `json:"proxy,omitempty"` IsHttps int `json:"isHttps,omitempty"` // 动态字段存储,用于存储任意其他字段 DynamicFields map[string]interface{} `json:"dynamicFields,omitempty"` // 原始数据备份,用于调试和回溯 RawData map[string]interface{} `json:"rawData,omitempty"` } // backendInfo 用于解析 "web" 类型中 backendList JSON 字符串的内部结构 type backendInfo struct { Addr string `json:"addr"` CustomHost string `json:"customHost"` } // parseWafLogExtraData 使用gjson解析动态JSON结构,简洁高效 func (s *wafLogDataCleanService) ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData { var result CleanedExtraData result.DynamicFields = make(map[string]interface{}) if len(extraDataBytes) == 0 { return result } jsonStr := string(extraDataBytes) if !gjson.Valid(jsonStr) { s.Logger.Warn("ExtraData 不是有效的JSON", zap.String("raw_data", jsonStr)) return result } // 解析并保存原始数据 var rawData map[string]interface{} json.Unmarshal(extraDataBytes, &rawData) result.RawData = rawData // 使用gjson进行智能字段提取 s.extractWithGjson(jsonStr, apiName, &result) return result } // extractWithGjson 使用gjson进行智能字段提取 func (s *wafLogDataCleanService) extractWithGjson(jsonStr, apiName string, result *CleanedExtraData) { // 提取顶层字段 if uid := gjson.Get(jsonStr, "uid"); uid.Exists() { result.UID = uid.Int() } if hostId := gjson.Get(jsonStr, "hostId"); hostId.Exists() { result.HostID = hostId.Int() } // 定义常见字段路径的优先级列表 fieldPaths := map[string][]string{ "comment": {"comment", "data.comment", "desc", "description", "remark", "note"}, "port": {"port", "data.port", "config.port", "server.port"}, "domain": {"domain", "data.domain", "host", "data.host", "hostname"}, "proxy": {"proxy", "data.proxy"}, "isHttps": {"isHttps", "data.isHttps"}, } // 提取基础字段 for fieldName, paths := range fieldPaths { value := s.getFirstValidPath(jsonStr, paths) if value != "" { switch fieldName { case "comment": result.Comment = value case "port": result.Port = value case "domain": result.Domain = value case "proxy": result.Proxy = gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Bool() case "isHttps": result.IsHttps = int(gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Int()) } } } // 智能提取 backendList s.extractBackendListWithGjson(jsonStr, apiName, result) // 提取所有其他动态字段 s.extractDynamicFields(jsonStr, result) } // getFirstValidPath 从多个路径中获取第一个有效值 func (s *wafLogDataCleanService) getFirstValidPath(jsonStr string, paths []string) string { for _, path := range paths { if value := gjson.Get(jsonStr, path); value.Exists() && value.String() != "" { return value.String() } } return "" } // getFirstValidPathName 获取第一个有效路径的名称 func (s *wafLogDataCleanService) getFirstValidPathName(jsonStr string, paths []string) string { for _, path := range paths { if gjson.Get(jsonStr, path).Exists() { return path } } return "" } // extractBackendListWithGjson 使用gjson智能提取后端列表 func (s *wafLogDataCleanService) extractBackendListWithGjson(jsonStr, apiName string, result *CleanedExtraData) { // 定义可能的后端列表字段路径 backendPaths := []string{ "backendList", "data.backendList", "backends", "data.backends", "backend_list", "data.backend_list", "servers", "data.servers", "upstreams", "data.upstreams", "targets", "data.targets", } for _, path := range backendPaths { backendResult := gjson.Get(jsonStr, path) if !backendResult.Exists() { continue } // 根据数据类型进行处理 switch { case backendResult.IsArray(): s.processArrayBackends(backendResult, result) case backendResult.IsObject(): s.processObjectBackends(backendResult, result) default: // 字符串类型,可能是JSON字符串或单个地址 s.processStringBackends(backendResult.String(), result) } // 找到有效数据就退出 if len(result.AddrBackendList) > 0 || len(result.CustomHostList) > 0 { break } } } // processArrayBackends 处理数组格式的后端列表 func (s *wafLogDataCleanService) processArrayBackends(backendResult gjson.Result, result *CleanedExtraData) { backendResult.ForEach(func(key, value gjson.Result) bool { if value.IsObject() { // 创建BackendInfo结构 backend := BackendInfo{} // 尝试提取地址字段 addr := s.getFirstValidPath(value.Raw, []string{"addr", "address", "host", "server", "endpoint", "url"}) if addr != "" { backend.Addr = addr result.AddrBackendList = append(result.AddrBackendList, addr) } // 提取customHost if customHost := gjson.Get(value.Raw, "customHost").String(); customHost != "" { backend.CustomHost = customHost result.CustomHostList = append(result.CustomHostList, customHost) result.CustomHost = append(result.CustomHost, customHost) } // 只有当有有效数据时才添加到BackendList if backend.Addr != "" || backend.CustomHost != "" { result.BackendList = append(result.BackendList, backend) } } else { // 直接作为地址处理 if addr := value.String(); addr != "" { result.AddrBackendList = append(result.AddrBackendList, addr) result.BackendList = append(result.BackendList, BackendInfo{Addr: addr}) } } return true }) } // processObjectBackends 处理对象格式的后端列表 func (s *wafLogDataCleanService) processObjectBackends(backendResult gjson.Result, result *CleanedExtraData) { // 创建BackendInfo结构 backend := BackendInfo{} addr := s.getFirstValidPath(backendResult.Raw, []string{"addr", "address", "host", "server", "endpoint", "url"}) if addr != "" { backend.Addr = addr result.AddrBackendList = append(result.AddrBackendList, addr) } if customHost := gjson.Get(backendResult.Raw, "customHost").String(); customHost != "" { backend.CustomHost = customHost result.CustomHostList = append(result.CustomHostList, customHost) result.CustomHost = append(result.CustomHost, customHost) } // 只有当有有效数据时才添加到BackendList if backend.Addr != "" || backend.CustomHost != "" { result.BackendList = append(result.BackendList, backend) } } // processStringBackends 处理字符串格式的后端列表 func (s *wafLogDataCleanService) processStringBackends(backendStr string, result *CleanedExtraData) { if backendStr == "" { return } // 尝试作为JSON解析 if gjson.Valid(backendStr) { parsed := gjson.Parse(backendStr) if parsed.IsArray() { s.processArrayBackends(parsed, result) return } else if parsed.IsObject() { s.processObjectBackends(parsed, result) return } } // 作为单个地址处理 result.AddrBackendList = append(result.AddrBackendList, backendStr) result.BackendList = append(result.BackendList, BackendInfo{Addr: backendStr}) } // extractDynamicFields 提取所有动态字段到DynamicFields中 func (s *wafLogDataCleanService) extractDynamicFields(jsonStr string, result *CleanedExtraData) { // 已知的核心字段,不放入动态字段中 knownFields := map[string]bool{ "comment": true, "port": true, "domain": true, "backendList": true, "backends": true, "backend_list": true, "data": true, // data字段的内容会被单独处理 } // 遍历顶层字段 gjson.Parse(jsonStr).ForEach(func(key, value gjson.Result) bool { fieldName := key.String() if !knownFields[fieldName] { // 将未知字段存储到动态字段中 result.DynamicFields[fieldName] = value.Value() } return true }) // 特殊处理data字段中的未知字段 dataResult := gjson.Get(jsonStr, "data") if dataResult.Exists() && dataResult.IsObject() { dataKnownFields := map[string]bool{ "port": true, "domain": true, "backendList": true, "backends": true, "backend_list": true, } dataResult.ForEach(func(key, value gjson.Result) bool { fieldName := key.String() if !dataKnownFields[fieldName] { // 使用data.前缀避免冲突 result.DynamicFields["data."+fieldName] = value.Value() } return true }) } }