|
@@ -89,25 +89,14 @@ type wafLogService struct {
|
|
wafLogDataCleanService WafLogDataCleanService
|
|
wafLogDataCleanService WafLogDataCleanService
|
|
}
|
|
}
|
|
func (s *wafLogService) getFirstPathSegment(path string) (segment []string, ok bool) {
|
|
func (s *wafLogService) getFirstPathSegment(path string) (segment []string, ok bool) {
|
|
- // 1. 为了统一处理,先去掉路径最前面的 "/"
|
|
|
|
- // 这样 "/v1/admin" 会变成 "v1/admin",而 "v1/admin" 保持不变
|
|
|
|
trimmedPath := strings.TrimPrefix(path, "/")
|
|
trimmedPath := strings.TrimPrefix(path, "/")
|
|
-
|
|
|
|
- // 如果去掉 "/" 后字符串为空(比如原路径是 "/" 或 ""),则无法提取
|
|
|
|
if trimmedPath == "" {
|
|
if trimmedPath == "" {
|
|
return nil, false
|
|
return nil, false
|
|
}
|
|
}
|
|
-
|
|
|
|
- // 2. 使用 "/" 作为分隔符来切割字符串
|
|
|
|
- // "v1/admin/menus" 会被切割成一个字符串切片 (slice): ["v1", "admin", "menus"]
|
|
|
|
parts := strings.Split(trimmedPath, "/")
|
|
parts := strings.Split(trimmedPath, "/")
|
|
-
|
|
|
|
- // 3. 只要切片不为空,第一个元素就是我们需要的值
|
|
|
|
- // len(parts) > 0 这个检查可以保证程序不会因为空切片而出错
|
|
|
|
if len(parts) > 0 {
|
|
if len(parts) > 0 {
|
|
return parts, true
|
|
return parts, true
|
|
}
|
|
}
|
|
-
|
|
|
|
return nil, false
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
|
|
@@ -208,7 +197,6 @@ func (s *wafLogService) BatchAddWafLog(ctx context.Context, reqs []*adminApi.Waf
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
|
|
- // 调用repository层的批量插入方法
|
|
|
|
return s.wafLogRepository.BatchAddWafLog(ctx, wafLogs)
|
|
return s.wafLogRepository.BatchAddWafLog(ctx, wafLogs)
|
|
}
|
|
}
|
|
|
|
|
|
@@ -216,31 +204,26 @@ func (s *wafLogService) PublishIpWafLogTask(ctx context.Context, req adminApi.Wa
|
|
|
|
|
|
payload := &req
|
|
payload := &req
|
|
|
|
|
|
- // Serialize the message
|
|
|
|
msgBody, err := json.Marshal(payload)
|
|
msgBody, err := json.Marshal(payload)
|
|
if err != nil {
|
|
if err != nil {
|
|
s.Logger.Error("序列化 WafLog 任务消息失败", zap.Error(err), zap.Int("hostId", payload.HostId), zap.Int("uid", payload.Uid), zap.Any("req", req))
|
|
s.Logger.Error("序列化 WafLog 任务消息失败", zap.Error(err), zap.Int("hostId", payload.HostId), zap.Int("uid", payload.Uid), zap.Any("req", req))
|
|
return
|
|
return
|
|
}
|
|
}
|
|
|
|
|
|
- // Get task configuration
|
|
|
|
taskCfg, ok := s.mq.GetTaskConfig("waf_log")
|
|
taskCfg, ok := s.mq.GetTaskConfig("waf_log")
|
|
if !ok {
|
|
if !ok {
|
|
s.Logger.Error("无法获取“waf_Log”任务配置")
|
|
s.Logger.Error("无法获取“waf_Log”任务配置")
|
|
return
|
|
return
|
|
}
|
|
}
|
|
|
|
|
|
- // Construct the routing key dynamically based on the action
|
|
|
|
routingKey := fmt.Sprintf("wafLog.%s", "add")
|
|
routingKey := fmt.Sprintf("wafLog.%s", "add")
|
|
|
|
|
|
- // Construct the amqp.Publishing message
|
|
|
|
publishingMsg := amqp.Publishing{
|
|
publishingMsg := amqp.Publishing{
|
|
ContentType: "application/json",
|
|
ContentType: "application/json",
|
|
Body: msgBody,
|
|
Body: msgBody,
|
|
- DeliveryMode: amqp.Persistent, // Persistent message
|
|
|
|
|
|
+ DeliveryMode: amqp.Persistent,
|
|
}
|
|
}
|
|
|
|
|
|
- // Publish the message
|
|
|
|
err = s.mq.PublishWithCh(taskCfg.Exchange, routingKey, publishingMsg)
|
|
err = s.mq.PublishWithCh(taskCfg.Exchange, routingKey, publishingMsg)
|
|
if err != nil {
|
|
if err != nil {
|
|
s.Logger.Error("发布 WafLog 任务消息失败", zap.Error(err), zap.Int("hostId", payload.HostId), zap.Int("uid", payload.Uid), zap.Any("req", req))
|
|
s.Logger.Error("发布 WafLog 任务消息失败", zap.Error(err), zap.Int("hostId", payload.HostId), zap.Int("uid", payload.Uid), zap.Any("req", req))
|
|
@@ -250,29 +233,21 @@ func (s *wafLogService) PublishIpWafLogTask(ctx context.Context, req adminApi.Wa
|
|
}
|
|
}
|
|
|
|
|
|
func (s *wafLogService) ExPortWafLog(ctx context.Context, req adminApi.ExportWafLog) ([]adminApi.ExportWafLogRes, error) {
|
|
func (s *wafLogService) ExPortWafLog(ctx context.Context, req adminApi.ExportWafLog) ([]adminApi.ExportWafLogRes, error) {
|
|
- // 获取原始数据
|
|
|
|
data, err := s.wafLogRepository.ExportWafLog(ctx, req)
|
|
data, err := s.wafLogRepository.ExportWafLog(ctx, req)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, err
|
|
return nil, err
|
|
}
|
|
}
|
|
-
|
|
|
|
- // 使用优化后的转换方法,避免N+1查询
|
|
|
|
return s.convertRawDataToExportResults(ctx, data)
|
|
return s.convertRawDataToExportResults(ctx, data)
|
|
}
|
|
}
|
|
|
|
|
|
-// SmartExportWafLog 智能导出WAF日志为Excel
|
|
|
|
func (s *wafLogService) SmartExportWafLog(ctx context.Context, req adminApi.ExportWafLog, w http.ResponseWriter) error {
|
|
func (s *wafLogService) SmartExportWafLog(ctx context.Context, req adminApi.ExportWafLog, w http.ResponseWriter) error {
|
|
- // 1. 先获取总数量用于智能选择传输方式
|
|
|
|
count, err := s.wafLogRepository.GetWafLogExportCount(ctx, req)
|
|
count, err := s.wafLogRepository.GetWafLogExportCount(ctx, req)
|
|
if err != nil {
|
|
if err != nil {
|
|
return fmt.Errorf("获取导出数据总数失败: %w", err)
|
|
return fmt.Errorf("获取导出数据总数失败: %w", err)
|
|
}
|
|
}
|
|
|
|
|
|
- // 2. 智能选择导出方式
|
|
|
|
- // 估算每行数据大小约200字节(包含用户名、IP、API名称、域名等字段)
|
|
|
|
exportType := excel.SmartExport(count, 200)
|
|
exportType := excel.SmartExport(count, 200)
|
|
|
|
|
|
- // 3. 设置Excel表头映射
|
|
|
|
headers := []string{"name", "request_ip", "host_id", "rule_id", "api_name", "addr_backend_list", "allow_and_deny_ips", "domain", "custom_host", "expose_addr", "comment", "created_at"}
|
|
headers := []string{"name", "request_ip", "host_id", "rule_id", "api_name", "addr_backend_list", "allow_and_deny_ips", "domain", "custom_host", "expose_addr", "comment", "created_at"}
|
|
headerMap := map[string]string{
|
|
headerMap := map[string]string{
|
|
"name": "用户名",
|
|
"name": "用户名",
|
|
@@ -289,13 +264,11 @@ func (s *wafLogService) SmartExportWafLog(ctx context.Context, req adminApi.Expo
|
|
"created_at": "操作时间",
|
|
"created_at": "操作时间",
|
|
}
|
|
}
|
|
|
|
|
|
- // 4. 创建Excel生成器
|
|
|
|
generator := excel.NewExcelGenerator("WAF日志", headers, headerMap)
|
|
generator := excel.NewExcelGenerator("WAF日志", headers, headerMap)
|
|
if err := generator.WriteHeaders(); err != nil {
|
|
if err := generator.WriteHeaders(); err != nil {
|
|
return fmt.Errorf("写入Excel表头失败: %w", err)
|
|
return fmt.Errorf("写入Excel表头失败: %w", err)
|
|
}
|
|
}
|
|
|
|
|
|
- // 5. 根据导出类型选择不同的处理方式
|
|
|
|
switch exportType {
|
|
switch exportType {
|
|
case excel.ExportTypeNormal:
|
|
case excel.ExportTypeNormal:
|
|
return s.normalExportWafLog(ctx, req, generator, w)
|
|
return s.normalExportWafLog(ctx, req, generator, w)
|
|
@@ -308,15 +281,12 @@ func (s *wafLogService) SmartExportWafLog(ctx context.Context, req adminApi.Expo
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
-// normalExportWafLog 普通导出(小文件)
|
|
|
|
func (s *wafLogService) normalExportWafLog(ctx context.Context, req adminApi.ExportWafLog, generator *excel.ExcelGenerator, w http.ResponseWriter) error {
|
|
func (s *wafLogService) normalExportWafLog(ctx context.Context, req adminApi.ExportWafLog, generator *excel.ExcelGenerator, w http.ResponseWriter) error {
|
|
- // 获取所有数据(已经优化了批量查询)
|
|
|
|
exportData, err := s.ExPortWafLog(ctx, req)
|
|
exportData, err := s.ExPortWafLog(ctx, req)
|
|
if err != nil {
|
|
if err != nil {
|
|
return fmt.Errorf("获取导出数据失败: %w", err)
|
|
return fmt.Errorf("获取导出数据失败: %w", err)
|
|
}
|
|
}
|
|
|
|
|
|
- // 转换数据格式
|
|
|
|
data := make([]map[string]interface{}, 0, len(exportData))
|
|
data := make([]map[string]interface{}, 0, len(exportData))
|
|
for _, item := range exportData {
|
|
for _, item := range exportData {
|
|
row := map[string]interface{}{
|
|
row := map[string]interface{}{
|
|
@@ -336,12 +306,10 @@ func (s *wafLogService) normalExportWafLog(ctx context.Context, req adminApi.Exp
|
|
data = append(data, row)
|
|
data = append(data, row)
|
|
}
|
|
}
|
|
|
|
|
|
- // 写入数据
|
|
|
|
if err := generator.WriteRows(data); err != nil {
|
|
if err := generator.WriteRows(data); err != nil {
|
|
return fmt.Errorf("写入Excel数据失败: %w", err)
|
|
return fmt.Errorf("写入Excel数据失败: %w", err)
|
|
}
|
|
}
|
|
|
|
|
|
- // 普通导出
|
|
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
return excel.NormalExport(generator, w, excel.TransferOption{
|
|
return excel.NormalExport(generator, w, excel.TransferOption{
|
|
FileName: fileName,
|
|
FileName: fileName,
|
|
@@ -349,37 +317,31 @@ func (s *wafLogService) normalExportWafLog(ctx context.Context, req adminApi.Exp
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
|
|
-// streamExportWafLog 流式导出(大文件)
|
|
|
|
func (s *wafLogService) streamExportWafLog(ctx context.Context, req adminApi.ExportWafLog, generator *excel.ExcelGenerator, w http.ResponseWriter) error {
|
|
func (s *wafLogService) streamExportWafLog(ctx context.Context, req adminApi.ExportWafLog, generator *excel.ExcelGenerator, w http.ResponseWriter) error {
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
|
|
|
|
- // 设置响应头
|
|
|
|
w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
|
w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName))
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName))
|
|
w.Header().Set("Transfer-Encoding", "chunked")
|
|
w.Header().Set("Transfer-Encoding", "chunked")
|
|
|
|
|
|
- // 分批处理数据,每批1000条
|
|
|
|
pageSize := 1000
|
|
pageSize := 1000
|
|
page := 1
|
|
page := 1
|
|
|
|
|
|
for {
|
|
for {
|
|
- // 使用分页导出方法
|
|
|
|
exportData, err := s.wafLogRepository.ExportWafLogWithPagination(ctx, req, page, pageSize)
|
|
exportData, err := s.wafLogRepository.ExportWafLogWithPagination(ctx, req, page, pageSize)
|
|
if err != nil {
|
|
if err != nil {
|
|
return fmt.Errorf("获取第%d页数据失败: %w", page, err)
|
|
return fmt.Errorf("获取第%d页数据失败: %w", page, err)
|
|
}
|
|
}
|
|
|
|
|
|
- // 转换为导出格式(复用原有的ExPortWafLog逻辑)
|
|
|
|
exportResults, err := s.convertRawDataToExportResults(ctx, exportData)
|
|
exportResults, err := s.convertRawDataToExportResults(ctx, exportData)
|
|
if err != nil {
|
|
if err != nil {
|
|
return fmt.Errorf("转换导出数据失败: %w", err)
|
|
return fmt.Errorf("转换导出数据失败: %w", err)
|
|
}
|
|
}
|
|
|
|
|
|
if len(exportResults) == 0 {
|
|
if len(exportResults) == 0 {
|
|
- break // 没有更多数据
|
|
|
|
|
|
+ break
|
|
}
|
|
}
|
|
|
|
|
|
- // 转换并写入当前批次数据
|
|
|
|
for _, item := range exportResults {
|
|
for _, item := range exportResults {
|
|
row := map[string]interface{}{
|
|
row := map[string]interface{}{
|
|
"name": item.Name,
|
|
"name": item.Name,
|
|
@@ -401,7 +363,6 @@ func (s *wafLogService) streamExportWafLog(ctx context.Context, req adminApi.Exp
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- // 如果当前批次数据少于页大小,说明已经是最后一页
|
|
|
|
if len(exportResults) < pageSize {
|
|
if len(exportResults) < pageSize {
|
|
break
|
|
break
|
|
}
|
|
}
|
|
@@ -409,90 +370,52 @@ func (s *wafLogService) streamExportWafLog(ctx context.Context, req adminApi.Exp
|
|
page++
|
|
page++
|
|
}
|
|
}
|
|
|
|
|
|
- // 流式导出
|
|
|
|
return excel.StreamExport(generator, w, excel.TransferOption{
|
|
return excel.StreamExport(generator, w, excel.TransferOption{
|
|
FileName: fileName,
|
|
FileName: fileName,
|
|
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
|
|
-// chunkExportWafLog 分块导出(超大文件)
|
|
|
|
func (s *wafLogService) chunkExportWafLog(ctx context.Context, req adminApi.ExportWafLog, w http.ResponseWriter, totalRecords int) error {
|
|
func (s *wafLogService) chunkExportWafLog(ctx context.Context, req adminApi.ExportWafLog, w http.ResponseWriter, totalRecords int) error {
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
fileName := fmt.Sprintf("waf_logs_%s.xlsx", time.Now().Format("20060102_150405"))
|
|
- pageSize := 5000 // 每个分块5000条记录
|
|
|
|
|
|
+ pageSize := 5000
|
|
|
|
|
|
- // 分块导出需要前端配合实现
|
|
|
|
excel.ChunkExport(w, excel.TransferOption{
|
|
excel.ChunkExport(w, excel.TransferOption{
|
|
FileName: fileName,
|
|
FileName: fileName,
|
|
- ContentType: "application/json", // 返回分块信息
|
|
|
|
|
|
+ ContentType: "application/json",
|
|
}, totalRecords, pageSize)
|
|
}, totalRecords, pageSize)
|
|
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-
|
|
|
|
-// convertRawDataToExportResults 将原始数据转换为导出结果(复用原有的ExPortWafLog逻辑)
|
|
|
|
-func (s *wafLogService) convertRawDataToExportResults(ctx context.Context, rawData []model.WafLog) ([]adminApi.ExportWafLogRes, error) {
|
|
|
|
|
|
+// convertRawDataToExportResults 将从数据库获取的带有关联IP的数据转换为最终导出格式
|
|
|
|
+func (s *wafLogService) convertRawDataToExportResults(ctx context.Context, rawData []model.WafLogWithGatewayIP) ([]adminApi.ExportWafLogRes, error) {
|
|
if len(rawData) == 0 {
|
|
if len(rawData) == 0 {
|
|
return []adminApi.ExportWafLogRes{}, nil
|
|
return []adminApi.ExportWafLogRes{}, nil
|
|
}
|
|
}
|
|
|
|
|
|
- // 批量准备逻辑保持不变...
|
|
|
|
- hostIds := make([]int64, 0, len(rawData))
|
|
|
|
- uids := make([]int64, 0, len(rawData))
|
|
|
|
- maxCreatedAt := time.Time{}
|
|
|
|
- for _, v := range rawData {
|
|
|
|
- hostIds = append(hostIds, int64(v.HostId))
|
|
|
|
- uids = append(uids, int64(v.Uid))
|
|
|
|
- if v.CreatedAt.After(maxCreatedAt) {
|
|
|
|
- maxCreatedAt = v.CreatedAt
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- gatewayMap, err := s.wafLogRepository.BatchGetWafLogGateWayIps(ctx, hostIds, uids, maxCreatedAt)
|
|
|
|
- if err != nil {
|
|
|
|
- s.Logger.Warn("批量获取网关组失败,降级为单个查询", zap.Error(err))
|
|
|
|
- gatewayMap = make(map[string]model.WafLog)
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
var res []adminApi.ExportWafLogRes
|
|
var res []adminApi.ExportWafLogRes
|
|
for _, v := range rawData {
|
|
for _, v := range rawData {
|
|
- // --- 核心改动:一行代码完成所有数据清洗 ---
|
|
|
|
|
|
+ // --- 数据清洗 ---
|
|
cleanedData := s.wafLogDataCleanService.ParseWafLogExtraData(v.ExtraData, v.ApiName)
|
|
cleanedData := s.wafLogDataCleanService.ParseWafLogExtraData(v.ExtraData, v.ApiName)
|
|
|
|
|
|
- // 网关 IP 获取逻辑保持不变,但使用清洗后的 port
|
|
|
|
|
|
+ // --- 网关 IP 处理 ---
|
|
var exposeAddr []string
|
|
var exposeAddr []string
|
|
- key := fmt.Sprintf("%d_%d", v.HostId, v.Uid)
|
|
|
|
- if gatewayModel, exists := gatewayMap[key]; exists {
|
|
|
|
|
|
+ // 直接使用查询结果中附带的 gateway_ip_data
|
|
|
|
+ if len(v.GatewayIpData) > 0 && string(v.GatewayIpData) != "null" {
|
|
var gateWayIps []string
|
|
var gateWayIps []string
|
|
- if err := json.Unmarshal(gatewayModel.ExtraData, &gateWayIps); err == nil && len(gateWayIps) > 0 && cleanedData.Port != "" {
|
|
|
|
|
|
+ // 解析从数据库子查询得到的JSON数据
|
|
|
|
+ if err := json.Unmarshal(v.GatewayIpData, &gateWayIps); err == nil && len(gateWayIps) > 0 && cleanedData.Port != "" {
|
|
for _, ip := range gateWayIps {
|
|
for _, ip := range gateWayIps {
|
|
exposeAddr = append(exposeAddr, ip+":"+cleanedData.Port)
|
|
exposeAddr = append(exposeAddr, ip+":"+cleanedData.Port)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
- } else {
|
|
|
|
- // 降级查询逻辑...
|
|
|
|
- gateWayIpModel, err := s.wafLogRepository.GetWafLogGateWayIp(ctx, int64(v.HostId), int64(v.Uid), v.CreatedAt)
|
|
|
|
- if err == nil {
|
|
|
|
- var gateWayIps []string
|
|
|
|
- if err := json.Unmarshal(gateWayIpModel.ExtraData, &gateWayIps); err == nil && len(gateWayIps) > 0 && cleanedData.Port != "" {
|
|
|
|
- for _, ip := range gateWayIps {
|
|
|
|
- exposeAddr = append(exposeAddr, ip+":"+cleanedData.Port)
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
}
|
|
}
|
|
|
|
|
|
var ruleIds []int64
|
|
var ruleIds []int64
|
|
if len(cleanedData.RuleID) > 0 {
|
|
if len(cleanedData.RuleID) > 0 {
|
|
ruleIds = cleanedData.RuleID
|
|
ruleIds = cleanedData.RuleID
|
|
- }else {
|
|
|
|
|
|
+ } else {
|
|
ruleIds = []int64{int64(v.RuleId)}
|
|
ruleIds = []int64{int64(v.RuleId)}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -502,7 +425,7 @@ func (s *wafLogService) convertRawDataToExportResults(ctx context.Context, rawDa
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
- // 构造结果,代码更清晰
|
|
|
|
|
|
+ // --- 构造结果 ---
|
|
res = append(res, adminApi.ExportWafLogRes{
|
|
res = append(res, adminApi.ExportWafLogRes{
|
|
Name: v.Name,
|
|
Name: v.Name,
|
|
RequestIp: v.RequestIp,
|
|
RequestIp: v.RequestIp,
|
|
@@ -525,4 +448,4 @@ func (s *wafLogService) convertRawDataToExportResults(ctx context.Context, rawDa
|
|
// GetApiDescriptions 获取API描述映射
|
|
// GetApiDescriptions 获取API描述映射
|
|
func (s *wafLogService) GetApiDescriptions(ctx context.Context) map[string]string {
|
|
func (s *wafLogService) GetApiDescriptions(ctx context.Context) map[string]string {
|
|
return ApiDescriptionMap
|
|
return ApiDescriptionMap
|
|
-}
|
|
|
|
|
|
+}
|