robin/waf-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

dns clickhouse改造

Browse Source
This commit is contained in:
robin
2026-02-10 19:30:44 +08:00
parent 4812ad5aaf
commit 1bb8140a41
47 changed files with 2815 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go
View File

@@ -4,6 +4,7 @@ package nameservers
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
@@ -29,11 +30,13 @@ func (this *NSAccessLogService) CreateNSAccessLogs(ctx context.Context, req *pb.
return &pb.CreateNSAccessLogsResponse{}, nil
}
var tx = this.NullTx()
if this.canWriteNSAccessLogsToDB() {
var tx = this.NullTx()
err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
if err != nil {
return nil, err
err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
if err != nil {
return nil, err
}
}
return &pb.CreateNSAccessLogsResponse{}, nil
@@ -54,6 +57,34 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
// TODO 检查权限
}
store := clickhouse.NewNSLogsIngestStore()
canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured() && req.Day != ""
canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
if canReadFromClickHouse {
resp, listErr := this.listNSAccessLogsFromClickHouse(ctx, store, req)
if listErr == nil && resp != nil {
return resp, nil
}
if !canReadFromMySQL {
if listErr != nil {
return nil, listErr
}
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: []*pb.NSAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
}
if !canReadFromMySQL {
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: []*pb.NSAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
accessLogs, requestId, hasMore, err := models.SharedNSAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NsClusterId, req.NsNodeId, req.NsDomainId, req.NsRecordId, req.RecordType, req.Keyword, req.Reverse)
if err != nil {
return nil, err
@@ -67,23 +98,9 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
}
// 线路
if len(a.NsRouteCodes) > 0 {
for _, routeCode := range a.NsRouteCodes {
route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
if err != nil {
return nil, err
}
if route != nil {
a.NsRoutes = append(a.NsRoutes, &pb.NSRoute{
Id: types.Int64(route.Id),
IsOn: route.IsOn,
Name: route.Name,
Code: routeCode,
NsCluster: nil,
NsDomain: nil,
})
}
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
result = append(result, a)
@@ -104,6 +121,31 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
return nil, err
}
store := clickhouse.NewNSLogsIngestStore()
canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured()
canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
if canReadFromClickHouse {
row, findErr := store.FindByRequestId(ctx, req.RequestId)
if findErr != nil {
if !canReadFromMySQL {
return nil, findErr
}
} else if row != nil {
a := clickhouse.NSRowToPB(row)
if a != nil {
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
}
return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
}
}
if !canReadFromMySQL {
return &pb.FindNSAccessLogResponse{NsAccessLog: nil}, nil
}
var tx = this.NullTx()
accessLog, err := models.SharedNSAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
@@ -123,5 +165,70 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
if err != nil {
return nil, err
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
}
func (this *NSAccessLogService) listNSAccessLogsFromClickHouse(ctx context.Context, store *clickhouse.NSLogsIngestStore, req *pb.ListNSAccessLogsRequest) (*pb.ListNSAccessLogsResponse, error) {
rows, nextCursor, hasMore, err := store.List(ctx, clickhouse.NSListFilter{
Day: req.Day,
Size: req.Size,
Reverse: req.Reverse,
LastRequestId: req.RequestId,
NSClusterId: req.NsClusterId,
NSNodeId: req.NsNodeId,
NSDomainId: req.NsDomainId,
NSRecordId: req.NsRecordId,
RecordType: req.RecordType,
Keyword: req.Keyword,
})
if err != nil {
return nil, err
}
result := make([]*pb.NSAccessLog, 0, len(rows))
for _, row := range rows {
a := clickhouse.NSRowToPB(row)
if a == nil {
continue
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
result = append(result, a)
}
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: result,
HasMore: hasMore,
RequestId: nextCursor,
}, nil
}
func (this *NSAccessLogService) fillNSRoutes(accessLog *pb.NSAccessLog) error {
if accessLog == nil || len(accessLog.NsRouteCodes) == 0 {
return nil
}
for _, routeCode := range accessLog.NsRouteCodes {
route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
if err != nil {
return err
}
if route != nil {
accessLog.NsRoutes = append(accessLog.NsRoutes, &pb.NSRoute{
Id: types.Int64(route.Id),
IsOn: route.IsOn,
Name: route.Name,
Code: routeCode,
NsCluster: nil,
NsDomain: nil,
})
}
}
return nil
}

17
EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log_ext_plus.go Normal file
View File

@@ -0,0 +1,17 @@
//go:build plus
package nameservers
import "github.com/TeaOSLab/EdgeAPI/internal/accesslogs"
func (this *NSAccessLogService) canWriteNSAccessLogsToDB() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *NSAccessLogService) shouldReadNSAccessLogsFromClickHouse() bool {
return accesslogs.SharedStorageManager.WriteClickHouse()
}
func (this *NSAccessLogService) shouldReadNSAccessLogsFromMySQL() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
}

27
EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go
View File

@@ -8,12 +8,27 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/accesslogs"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
)
type HTTPAccessLogPolicyService struct {
BaseService
}
func (this *HTTPAccessLogPolicyService) normalizeStorageTypeAndTargets(policyType string, writeTargetsJSON []byte, disableDefaultDB bool) (string, []byte) {
_ = writeTargetsJSON
_ = disableDefaultDB
// 兼容旧前端/缓存可能传来的历史类型编码
switch policyType {
case "clickhouse":
policyType = serverconfigs.AccessLogStorageTypeFileClickhouse
case "mysql_clickhouse":
policyType = serverconfigs.AccessLogStorageTypeFileMySQLClickhouse
}
return policyType, nil
}
// CountAllHTTPAccessLogPolicies 计算访问日志策略数量
func (this *HTTPAccessLogPolicyService) CountAllHTTPAccessLogPolicies(ctx context.Context, req *pb.CountAllHTTPAccessLogPoliciesRequest) (*pb.RPCCountResponse, error) {
_, err := this.ValidateAdmin(ctx)
@@ -53,7 +68,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
WriteTargetsJSON: nil,
})
}
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -76,8 +91,10 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
}
}
policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB)
// 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON)
if err != nil {
return nil, err
}
@@ -101,8 +118,10 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
}
}
policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB)
// 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON, req.IsOn)
if err != nil {
return nil, err
}
@@ -134,7 +153,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
WriteTargetsJSON: nil,
}}, nil
}