go_gameshield/internal/service/admin/waflogdataclean.go

package admin

import (
	"encoding/json"
	"fmt"
	"github.com/go-nunu/nunu-layout-advanced/internal/service"
	"github.com/tidwall/gjson"
	"go.uber.org/zap"
	"strings"
)

type WafLogDataCleanService interface {
	ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData
	FormatBackendList(backendList interface{}) string
}
func NewWafLogDataCleanService(
    service *service.Service,
) WafLogDataCleanService {
	return &wafLogDataCleanService{
		Service:        service,

	}
}

type wafLogDataCleanService struct {
	*service.Service

}

// BackendInfo 后端服务器信息
type BackendInfo struct {
	Addr       string `json:"addr,omitempty"`       // 后端地址
	CustomHost string `json:"customHost,omitempty"` // 自定义Host头
	IsHttps    int    `json:"isHttps,omitempty"`    // 是否为HTTPS
}

// CleanedExtraData 使用动态结构存储解析后的数据
type CleanedExtraData struct {
	// 核心字段 - 新的数组结构
	BackendList     []BackendInfo `json:"backendList,omitempty"`     // 完整的后端信息数组
	
	// 向后兼容字段
	AddrBackendList []string `json:"addrBackendList,omitempty"` // 只包含地址的数组
	CustomHostList  []string `json:"customHostList,omitempty"`  // 只包含customHost的数组
	CustomHost      []string `json:"customHost,omitempty"`      // customHost数组，与CustomHostList相同
	
	// 基础字段
	Port            string   `json:"port,omitempty"`
	Domain          string   `json:"domain,omitempty"`
	Comment         string   `json:"comment,omitempty"`
	
	// 扩展字段
	UID             int64    `json:"uid,omitempty"`
	HostID          int64    `json:"hostId,omitempty"`
	Proxy           bool     `json:"proxy,omitempty"`
	IsHttps         int      `json:"isHttps,omitempty"`
	RuleID          []int64    `json:"ruleId,omitempty"`

	// 其他字段
	AllowAndDenyIps string   `json:"allowAndDenyIps,omitempty"`
	
	// 动态字段存储，用于存储任意其他字段
	DynamicFields map[string]interface{} `json:"dynamicFields,omitempty"`
	
	// 原始数据备份，用于调试和回溯
	RawData map[string]interface{} `json:"rawData,omitempty"`
}




// parseWafLogExtraData 使用gjson解析动态JSON结构，简洁高效
func (s *wafLogDataCleanService) ParseWafLogExtraData(extraDataBytes json.RawMessage, apiName string) CleanedExtraData {
	var result CleanedExtraData
	result.DynamicFields = make(map[string]interface{})
	
	if len(extraDataBytes) == 0 {
		return result
	}

	jsonStr := string(extraDataBytes)
	if !gjson.Valid(jsonStr) {
		s.Logger.Warn("ExtraData 不是有效的JSON", zap.String("raw_data", jsonStr))
		return result
	}

	// 解析并保存原始数据
	var rawData map[string]interface{}
	json.Unmarshal(extraDataBytes, &rawData)
	result.RawData = rawData

	// 使用gjson进行智能字段提取
	s.extractWithGjson(jsonStr, apiName, &result)

	return result
}

// extractWithGjson 使用gjson进行智能字段提取
func (s *wafLogDataCleanService) extractWithGjson(jsonStr, apiName string, result *CleanedExtraData) {
	// 提取顶层字段
	if uid := gjson.Get(jsonStr, "uid"); uid.Exists() {
		result.UID = uid.Int()
	}
	if hostId := gjson.Get(jsonStr, "hostId"); hostId.Exists() {
		result.HostID = hostId.Int()
	}
	
	// 定义常见字段路径的优先级列表
	fieldPaths := map[string][]string{
		"comment": {"comment", "data.comment", "desc", "description", "remark", "note"},
		"port":    {"port", "data.port", "config.port", "server.port"},
		"domain":  {"domain", "data.domain", "host", "data.host", "hostname"},
		"proxy":   {"proxy", "data.proxy"},
		"isHttps": {"isHttps", "data.isHttps"},
		"ids":     {"ids", "data.ids", "ruleIds", "data.ruleIds", "ruleId", "data.ruleId"},
		"ip":      {"ip","newIp","ips"},
	}
	
	// 提取基础字段
	for fieldName, paths := range fieldPaths {
		value := s.getFirstValidPath(jsonStr, paths)
		if value != "" {
			switch fieldName {
			case "comment":
				result.Comment = value
			case "port":
				result.Port = value
			case "domain":
				result.Domain = value
			case "proxy":
				result.Proxy = gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Bool()
			case "isHttps":
				result.IsHttps = int(gjson.Get(jsonStr, s.getFirstValidPathName(jsonStr, paths)).Int())
			case "ids":
				result.RuleID = s.extractRuleIDs(jsonStr, paths)
			case "ip":
				result.AllowAndDenyIps = value

			}

		}
	}
	
	// 智能提取 backendList
	s.extractBackendListWithGjson(jsonStr, apiName, result)
	
	// 提取所有其他动态字段
	s.extractDynamicFields(jsonStr, result)
}

// getFirstValidPath 从多个路径中获取第一个有效值
func (s *wafLogDataCleanService) getFirstValidPath(jsonStr string, paths []string) string {
	for _, path := range paths {
		if value := gjson.Get(jsonStr, path); value.Exists() && value.String() != "" {
			return value.String()
		}
	}
	return ""
}

// getFirstValidPathName 获取第一个有效路径的名称
func (s *wafLogDataCleanService) getFirstValidPathName(jsonStr string, paths []string) string {
	for _, path := range paths {
		if gjson.Get(jsonStr, path).Exists() {
			return path
		}
	}
	return ""
}

// extractRuleIDs 提取规则ID数组
func (s *wafLogDataCleanService) extractRuleIDs(jsonStr string, paths []string) []int64 {
	var ruleIDs []int64
	
	for _, path := range paths {
		ruleResult := gjson.Get(jsonStr, path)
		if !ruleResult.Exists() {
			continue
		}
		
		switch {
		case ruleResult.IsArray():
			// 如果是数组，遍历提取每个ID
			ruleResult.ForEach(func(key, value gjson.Result) bool {
				if id := value.Int(); id > 0 {
					ruleIDs = append(ruleIDs, id)
				}
				return true
			})
		default:
			// 如果是单个值，添加到数组中
			if id := ruleResult.Int(); id > 0 {
				ruleIDs = append(ruleIDs, id)
			}
		}
		
		// 找到有效数据就退出
		if len(ruleIDs) > 0 {
			break
		}
	}
	
	return ruleIDs
}

// extractBackendListWithGjson 使用gjson智能提取后端列表
func (s *wafLogDataCleanService) extractBackendListWithGjson(jsonStr, apiName string, result *CleanedExtraData) {
	// 定义可能的后端列表字段路径
	backendPaths := []string{
		"data.backendList", "backendList", "backends", "data.backends",
		"backend_list", "data.backend_list", "servers", "data.servers",
		"upstreams", "data.upstreams", "targets", "data.targets",
	}
	
	for _, path := range backendPaths {
		backendResult := gjson.Get(jsonStr, path)
		if !backendResult.Exists() {
			continue
		}
		
		// 根据数据类型进行处理
		switch {
		case backendResult.IsArray():
			s.processArrayBackends(backendResult, result)

		}
		
		// 找到有效数据就退出
		if len(result.AddrBackendList) > 0 || len(result.CustomHostList) > 0 {
			break
		}
	}
}

// processArrayBackends 处理数组格式的后端列表
func (s *wafLogDataCleanService) processArrayBackends(backendResult gjson.Result, result *CleanedExtraData) {
	backendResult.ForEach(func(key, value gjson.Result) bool {
		if value.IsObject() {
			// 创建BackendInfo结构
			backend := BackendInfo{}

			isHttps := gjson.Get(value.Raw, "isHttps").Int()

			// 尝试提取地址字段
			addr := s.getFirstValidPath(value.Raw, []string{"addr", "address", "host", "server", "endpoint", "url"})

			if isHttps == 1 {
				addr = "https://" + addr
			}else {
				addr = "http://" + addr
			}

			if addr != "" {
				backend.Addr = addr
				result.AddrBackendList = append(result.AddrBackendList, addr)
			}

			// 提取customHost
			if customHost := gjson.Get(value.Raw, "customHost").String(); customHost != "" {
				backend.CustomHost = customHost
				result.CustomHostList = append(result.CustomHostList, customHost)
				result.CustomHost = append(result.CustomHost, customHost)
			}

			// 只有当有有效数据时才添加到BackendList
			if backend.Addr != "" || backend.CustomHost != "" {
				result.BackendList = append(result.BackendList, backend)
			}
		} else {
			// 直接作为地址处理
			if addr := value.String(); addr != "" {
				result.AddrBackendList = append(result.AddrBackendList, addr)
				result.BackendList = append(result.BackendList, BackendInfo{Addr: addr})
			}
		}
		return true
	})
}


// extractDynamicFields 提取所有动态字段到DynamicFields中
func (s *wafLogDataCleanService) extractDynamicFields(jsonStr string, result *CleanedExtraData) {
	// 已知的核心字段，不放入动态字段中
	knownFields := map[string]bool{
		"comment": true, "port": true, "domain": true, 
		"backendList": true, "backends": true, "backend_list": true,
		"data": true, // data字段的内容会被单独处理
	}
	
	// 遍历顶层字段
	gjson.Parse(jsonStr).ForEach(func(key, value gjson.Result) bool {
		fieldName := key.String()
		if !knownFields[fieldName] {
			// 将未知字段存储到动态字段中
			result.DynamicFields[fieldName] = value.Value()
		}
		return true
	})
	
	// 特殊处理data字段中的未知字段
	dataResult := gjson.Get(jsonStr, "data")
	if dataResult.Exists() && dataResult.IsObject() {
		dataKnownFields := map[string]bool{
			"port": true, "domain": true, "backendList": true, 
			"backends": true, "backend_list": true,
		}
		
		dataResult.ForEach(func(key, value gjson.Result) bool {
			fieldName := key.String()
			if !dataKnownFields[fieldName] {
				// 使用data.前缀避免冲突
				result.DynamicFields["data."+fieldName] = value.Value()
			}
			return true
		})
	}
}



// formatBackendList 格式化后端地址列表
func (s *wafLogDataCleanService) FormatBackendList(backendList interface{}) string {
	if backendList == nil {
		return ""
	}

	switch v := backendList.(type) {
	case string:
		return v
	case []string:
		if len(v) == 0 {
			return ""
		}
		return strings.Join(v, ", ")
	case []int64:
		// 处理 []int64 类型的数组（如 RuleId）
		if len(v) == 0 {
			return ""
		}
		var strList []string
		for _, id := range v {
			strList = append(strList, fmt.Sprintf("%d", id))
		}
		return strings.Join(strList, ", ")
	case []interface{}:
		// 处理 []interface{} 类型的数组
		if len(v) == 0 {
			return ""
		}
		var strList []string
		for _, item := range v {
			if str := fmt.Sprintf("%v", item); str != "" && str != "<nil>" {
				strList = append(strList, str)
			}
		}
		return strings.Join(strList, ", ")
	default:
		// 对于其他类型，先转换为字符串再处理
		str := fmt.Sprintf("%v", v)
		// 处理 Go 数组格式 [item1 item2] -> item1, item2
		if strings.HasPrefix(str, "[") && strings.HasSuffix(str, "]") {
			// 移除方括号
			content := strings.Trim(str, "[]")
			if content != "" {
				// 按空格分割并用逗号连接
				parts := strings.Fields(content)
				if len(parts) > 1 {
					return strings.Join(parts, ", ")
				}
				return content
			}
		}
		// 处理其他包含空格的字符串
		if strings.Contains(str, " ") && !strings.Contains(str, "\n") {
			parts := strings.Fields(str)
			if len(parts) > 1 {
				return strings.Join(parts, ", ")
			}
		}
		return str
	}
}