robin/waf-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

主分支代码

Browse Source
This commit is contained in:
robin
2026-02-07 20:30:31 +08:00
parent 3b042d1dad
commit bc223fd1aa
65 changed files with 1969 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
EdgeAPI/internal/rpc/services/service_http_access_log.go
View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
@@ -46,9 +47,8 @@ func (this *HTTPAccessLogService) CreateHTTPAccessLogs(ctx context.Context, req
return &pb.CreateHTTPAccessLogsResponse{}, nil
}
// ListHTTPAccessLogs 列出单页访问日志
// ListHTTPAccessLogs 列出单页访问日志（优先 ClickHouse，否则 MySQL；ClickHouse 路径下节点/集群批量查询避免 N+1）
func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *pb.ListHTTPAccessLogsRequest) (*pb.ListHTTPAccessLogsResponse, error) {
// 校验请求
_, userId, err := this.ValidateAdminAndUser(ctx, true)
if err != nil {
return nil, err
@@ -56,11 +56,8 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
var tx = this.NullTx()
// 检查服务ID
if userId > 0 {
req.UserId = userId
// 这里不用担心serverId <= 0 的情况，因为如果userId>0，则只会查询当前用户下的服务，不会产生安全问题
if req.ServerId > 0 {
err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId)
if err != nil {
@@ -69,6 +66,17 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
}
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() && req.Day != "" {
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
if listErr != nil {
return nil, listErr
}
if resp != nil {
return resp, nil
}
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil {
return nil, err
@@ -82,8 +90,6 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
if err != nil {
return nil, err
}
// 节点 & 集群
pbNode, ok := pbNodeMap[a.NodeId]
if ok {
a.Node = pbNode
@@ -94,42 +100,131 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
if node != nil {
pbNode = &pb.Node{Id: int64(node.Id), Name: node.Name}
var clusterId = int64(node.ClusterId)
pbCluster, ok := pbClusterMap[clusterId]
if ok {
pbNode.NodeCluster = pbCluster
} else {
if !ok {
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, clusterId)
if err != nil {
return nil, err
}
if cluster != nil {
pbCluster = &pb.NodeCluster{
Id: int64(cluster.Id),
Name: cluster.Name,
}
pbNode.NodeCluster = pbCluster
pbCluster = &pb.NodeCluster{Id: int64(cluster.Id), Name: cluster.Name}
pbClusterMap[clusterId] = pbCluster
}
}
if pbCluster != nil {
pbNode.NodeCluster = pbCluster
}
pbNodeMap[a.NodeId] = pbNode
a.Node = pbNode
}
}
result = append(result, a)
}
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result, // TODO 仅仅为了兼容，当用户节点版本大于0.0.8时可以删除
AccessLogs: result,
HasMore: hasMore,
RequestId: requestId,
}, nil
}
// listHTTPAccessLogsFromClickHouse 从 ClickHouse logs_ingest 查列表，并批量填充 Node/NodeCluster（避免 N+1）
func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.LogsIngestStore, req *pb.ListHTTPAccessLogsRequest, userId int64) (*pb.ListHTTPAccessLogsResponse, error) {
f := clickhouse.ListFilter{
Day: req.Day,
HourFrom: req.HourFrom,
HourTo: req.HourTo,
Size: req.Size,
Reverse: req.Reverse,
HasError: req.HasError,
HasFirewallPolicy: req.HasFirewallPolicy,
FirewallPolicyId: req.FirewallPolicyId,
NodeId: req.NodeId,
ClusterId: req.NodeClusterId,
LastRequestId: req.RequestId,
}
if req.ServerId > 0 {
f.ServerIds = []int64{req.ServerId}
} else if userId > 0 {
serverIds, err := models.SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
if err != nil {
return nil, err
}
if len(serverIds) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: nil, AccessLogs: nil, HasMore: false, RequestId: ""}, nil
}
f.ServerIds = serverIds
}
if req.NodeClusterId > 0 {
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, req.NodeClusterId)
if err != nil {
return nil, err
}
f.NodeIds = nodeIds
}
rows, nextCursor, err := store.List(ctx, f)
if err != nil {
return nil, err
}
if len(rows) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: []*pb.HTTPAccessLog{}, AccessLogs: []*pb.HTTPAccessLog{}, HasMore: false, RequestId: ""}, nil
}
result := make([]*pb.HTTPAccessLog, 0, len(rows))
nodeIdSet := make(map[int64]struct{})
for _, r := range rows {
result = append(result, clickhouse.RowToPB(r))
nodeIdSet[int64(r.NodeId)] = struct{}{}
}
nodeIds := make([]int64, 0, len(nodeIdSet))
for id := range nodeIdSet {
nodeIds = append(nodeIds, id)
}
nodes, err := models.SharedNodeDAO.FindEnabledBasicNodesWithIds(tx, nodeIds)
if err != nil {
return nil, err
}
clusterIds := make(map[int64]struct{})
for _, node := range nodes {
if node.ClusterId > 0 {
clusterIds[int64(node.ClusterId)] = struct{}{}
}
}
clusterIdList := make([]int64, 0, len(clusterIds))
for cid := range clusterIds {
clusterIdList = append(clusterIdList, cid)
}
clusters, _ := models.SharedNodeClusterDAO.FindEnabledNodeClustersWithIds(tx, clusterIdList)
clusterMap := make(map[int64]*pb.NodeCluster)
for _, c := range clusters {
clusterMap[int64(c.Id)] = &pb.NodeCluster{Id: int64(c.Id), Name: c.Name}
}
pbNodeMap := make(map[int64]*pb.Node)
for _, node := range nodes {
pbNode := &pb.Node{Id: int64(node.Id), Name: node.Name}
if c := clusterMap[int64(node.ClusterId)]; c != nil {
pbNode.NodeCluster = c
}
pbNodeMap[int64(node.Id)] = pbNode
}
for _, a := range result {
if n := pbNodeMap[a.NodeId]; n != nil {
a.Node = n
}
}
hasMore := nextCursor != ""
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result,
HasMore: hasMore,
RequestId: nextCursor,
}, nil
}
// FindHTTPAccessLog 查找单个日志
func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb.FindHTTPAccessLogRequest) (*pb.FindHTTPAccessLogResponse, error) {
// 校验请求

2
EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go
View File

@@ -11,7 +11,7 @@ import (
)
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return !accesslogs.SharedStorageManager.DisableDefaultDB()
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {

6
EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go
View File

@@ -53,6 +53,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
})
}
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -76,7 +77,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
}
// 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB)
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
if err != nil {
return nil, err
}
@@ -101,7 +102,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
}
// 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.IsOn)
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
if err != nil {
return nil, err
}
@@ -133,6 +134,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
}}, nil
}

6
EdgeAPI/internal/rpc/services/service_sys_setting.go
View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
@@ -26,6 +27,11 @@ func (this *SysSettingService) UpdateSysSetting(ctx context.Context, req *pb.Upd
return nil, err
}
// 若为 ClickHouse 配置，清空缓存使下次读取生效
if req.Code == "clickhouseConfig" {
clickhouse.ResetSharedConfig()
}
return this.Success()
}