dns clickhouse改造

EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go

@@ -4,6 +4,7 @@ package nameservers
 
 import (
 	"context"
+	"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
@@ -29,11 +30,13 @@ func (this *NSAccessLogService) CreateNSAccessLogs(ctx context.Context, req *pb.
 		return &pb.CreateNSAccessLogsResponse{}, nil
 	}
 
-	var tx = this.NullTx()
+	if this.canWriteNSAccessLogsToDB() {
+		var tx = this.NullTx()
 
-	err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
-	if err != nil {
-		return nil, err
+		err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return &pb.CreateNSAccessLogsResponse{}, nil
@@ -54,6 +57,34 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
 		// TODO 检查权限
 	}
 
+	store := clickhouse.NewNSLogsIngestStore()
+	canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured() && req.Day != ""
+	canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
+	if canReadFromClickHouse {
+		resp, listErr := this.listNSAccessLogsFromClickHouse(ctx, store, req)
+		if listErr == nil && resp != nil {
+			return resp, nil
+		}
+		if !canReadFromMySQL {
+			if listErr != nil {
+				return nil, listErr
+			}
+			return &pb.ListNSAccessLogsResponse{
+				NsAccessLogs: []*pb.NSAccessLog{},
+				HasMore:      false,
+				RequestId:    "",
+			}, nil
+		}
+	}
+
+	if !canReadFromMySQL {
+		return &pb.ListNSAccessLogsResponse{
+			NsAccessLogs: []*pb.NSAccessLog{},
+			HasMore:      false,
+			RequestId:    "",
+		}, nil
+	}
+
 	accessLogs, requestId, hasMore, err := models.SharedNSAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NsClusterId, req.NsNodeId, req.NsDomainId, req.NsRecordId, req.RecordType, req.Keyword, req.Reverse)
 	if err != nil {
 		return nil, err
@@ -67,23 +98,9 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
 		}
 
 		// 线路
-		if len(a.NsRouteCodes) > 0 {
-			for _, routeCode := range a.NsRouteCodes {
-				route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
-				if err != nil {
-					return nil, err
-				}
-				if route != nil {
-					a.NsRoutes = append(a.NsRoutes, &pb.NSRoute{
-						Id:        types.Int64(route.Id),
-						IsOn:      route.IsOn,
-						Name:      route.Name,
-						Code:      routeCode,
-						NsCluster: nil,
-						NsDomain:  nil,
-					})
-				}
-			}
+		err = this.fillNSRoutes(a)
+		if err != nil {
+			return nil, err
 		}
 
 		result = append(result, a)
@@ -104,6 +121,31 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
 		return nil, err
 	}
 
+	store := clickhouse.NewNSLogsIngestStore()
+	canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured()
+	canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
+	if canReadFromClickHouse {
+		row, findErr := store.FindByRequestId(ctx, req.RequestId)
+		if findErr != nil {
+			if !canReadFromMySQL {
+				return nil, findErr
+			}
+		} else if row != nil {
+			a := clickhouse.NSRowToPB(row)
+			if a != nil {
+				err = this.fillNSRoutes(a)
+				if err != nil {
+					return nil, err
+				}
+			}
+			return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
+		}
+	}
+
+	if !canReadFromMySQL {
+		return &pb.FindNSAccessLogResponse{NsAccessLog: nil}, nil
+	}
+
 	var tx = this.NullTx()
 
 	accessLog, err := models.SharedNSAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
@@ -123,5 +165,70 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
 	if err != nil {
 		return nil, err
 	}
+	err = this.fillNSRoutes(a)
+	if err != nil {
+		return nil, err
+	}
 	return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
 }
+
+func (this *NSAccessLogService) listNSAccessLogsFromClickHouse(ctx context.Context, store *clickhouse.NSLogsIngestStore, req *pb.ListNSAccessLogsRequest) (*pb.ListNSAccessLogsResponse, error) {
+	rows, nextCursor, hasMore, err := store.List(ctx, clickhouse.NSListFilter{
+		Day:           req.Day,
+		Size:          req.Size,
+		Reverse:       req.Reverse,
+		LastRequestId: req.RequestId,
+		NSClusterId:   req.NsClusterId,
+		NSNodeId:      req.NsNodeId,
+		NSDomainId:    req.NsDomainId,
+		NSRecordId:    req.NsRecordId,
+		RecordType:    req.RecordType,
+		Keyword:       req.Keyword,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result := make([]*pb.NSAccessLog, 0, len(rows))
+	for _, row := range rows {
+		a := clickhouse.NSRowToPB(row)
+		if a == nil {
+			continue
+		}
+		err = this.fillNSRoutes(a)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, a)
+	}
+
+	return &pb.ListNSAccessLogsResponse{
+		NsAccessLogs: result,
+		HasMore:      hasMore,
+		RequestId:    nextCursor,
+	}, nil
+}
+
+func (this *NSAccessLogService) fillNSRoutes(accessLog *pb.NSAccessLog) error {
+	if accessLog == nil || len(accessLog.NsRouteCodes) == 0 {
+		return nil
+	}
+
+	for _, routeCode := range accessLog.NsRouteCodes {
+		route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
+		if err != nil {
+			return err
+		}
+		if route != nil {
+			accessLog.NsRoutes = append(accessLog.NsRoutes, &pb.NSRoute{
+				Id:        types.Int64(route.Id),
+				IsOn:      route.IsOn,
+				Name:      route.Name,
+				Code:      routeCode,
+				NsCluster: nil,
+				NsDomain:  nil,
+			})
+		}
+	}
+	return nil
+}

EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log_ext_plus.go

@@ -0,0 +1,17 @@
+//go:build plus
+
+package nameservers
+
+import "github.com/TeaOSLab/EdgeAPI/internal/accesslogs"
+
+func (this *NSAccessLogService) canWriteNSAccessLogsToDB() bool {
+	return accesslogs.SharedStorageManager.WriteMySQL()
+}
+
+func (this *NSAccessLogService) shouldReadNSAccessLogsFromClickHouse() bool {
+	return accesslogs.SharedStorageManager.WriteClickHouse()
+}
+
+func (this *NSAccessLogService) shouldReadNSAccessLogsFromMySQL() bool {
+	return accesslogs.SharedStorageManager.WriteMySQL()
+}