diff --git a/EdgeAPI/internal/accesslogs/storage_manager.go b/EdgeAPI/internal/accesslogs/storage_manager.go index 53b1078..7639c47 100644 --- a/EdgeAPI/internal/accesslogs/storage_manager.go +++ b/EdgeAPI/internal/accesslogs/storage_manager.go @@ -80,7 +80,7 @@ func (this *StorageManager) Loop() error { if int64(policy.Id) == publicPolicyId { this.disableDefaultDB = policy.DisableDefaultDB - this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB) + this.writeTargets = serverconfigs.ResolveWriteTargetsByType(policy.Type, policy.DisableDefaultDB) foundPolicy = true } } @@ -185,6 +185,9 @@ func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets } func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) { + if serverconfigs.IsFileBasedStorageType(storageType) { + storageType = serverconfigs.AccessLogStorageTypeFile + } switch storageType { case serverconfigs.AccessLogStorageTypeFile: var config = &serverconfigs.AccessLogFileStorageConfig{} diff --git a/EdgeAPI/internal/clickhouse/logs_ingest_store.go b/EdgeAPI/internal/clickhouse/logs_ingest_store.go index 7c2359a..dde2623 100644 --- a/EdgeAPI/internal/clickhouse/logs_ingest_store.go +++ b/EdgeAPI/internal/clickhouse/logs_ingest_store.go @@ -85,6 +85,10 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI if f.Day == "" { return nil, "", fmt.Errorf("clickhouse: day required") } + dayNumber, err := normalizeDayNumber(f.Day) + if err != nil { + return nil, "", err + } table := "logs_ingest" if s.client.cfg.Database != "" && s.client.cfg.Database != "default" { table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest") @@ -92,7 +96,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI table = quoteIdent(table) } - conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"} + conditions := []string{"toYYYYMMDD(timestamp) = " + strconv.Itoa(dayNumber)} if f.HourFrom != "" { if _, err := strconv.Atoi(f.HourFrom); err == nil { conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom) @@ -240,6 +244,22 @@ func escapeString(s string) string { return strings.ReplaceAll(s, "'", "''") } +func normalizeDayNumber(day string) (int, error) { + normalized := strings.TrimSpace(day) + if normalized == "" { + return 0, fmt.Errorf("clickhouse: day required") + } + normalized = strings.ReplaceAll(normalized, "-", "") + if len(normalized) != 8 { + return 0, fmt.Errorf("clickhouse: invalid day '%s'", day) + } + dayNumber, err := strconv.Atoi(normalized) + if err != nil { + return 0, fmt.Errorf("clickhouse: invalid day '%s'", day) + } + return dayNumber, nil +} + func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow { r := &LogsIngestRow{} u64 := func(key string) uint64 { diff --git a/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go b/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go new file mode 100644 index 0000000..a8a38c6 --- /dev/null +++ b/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go @@ -0,0 +1,375 @@ +package clickhouse + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" +) + +// NSLogsIngestRow 对应 dns_logs_ingest 表一行. +type NSLogsIngestRow struct { + Timestamp time.Time + RequestId string + NodeId uint64 + ClusterId uint64 + DomainId uint64 + RecordId uint64 + RemoteAddr string + QuestionName string + QuestionType string + RecordName string + RecordType string + RecordValue string + Networking string + IsRecursive bool + Error string + NSRouteCodes []string + ContentJSON string +} + +// NSListFilter DNS 日志查询过滤条件. +type NSListFilter struct { + Day string + Size int64 + Reverse bool + LastRequestId string + NSClusterId int64 + NSNodeId int64 + NSDomainId int64 + NSRecordId int64 + RecordType string + Keyword string +} + +// NSLogsIngestStore DNS ClickHouse 查询封装. +type NSLogsIngestStore struct { + client *Client +} + +// NewNSLogsIngestStore 创建 NSLogsIngestStore. +func NewNSLogsIngestStore() *NSLogsIngestStore { + return &NSLogsIngestStore{client: NewClient()} +} + +// Client 返回底层 client. +func (s *NSLogsIngestStore) Client() *Client { + return s.client +} + +// List 列出 DNS 访问日志，返回列表、下一游标与是否有更多. +func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*NSLogsIngestRow, nextCursor string, hasMore bool, err error) { + if !s.client.IsConfigured() { + return nil, "", false, fmt.Errorf("clickhouse: not configured") + } + if f.Day == "" { + return nil, "", false, fmt.Errorf("clickhouse: day required") + } + dayNumber, err := normalizeDayNumber(f.Day) + if err != nil { + return nil, "", false, err + } + + table := quoteIdent("dns_logs_ingest") + if s.client.cfg.Database != "" && s.client.cfg.Database != "default" { + table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("dns_logs_ingest") + } + + conditions := []string{"toYYYYMMDD(timestamp) = " + strconv.Itoa(dayNumber)} + if f.NSClusterId > 0 { + conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.NSClusterId, 10)) + } + if f.NSNodeId > 0 { + conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NSNodeId, 10)) + } + if f.NSDomainId > 0 { + conditions = append(conditions, "domain_id = "+strconv.FormatInt(f.NSDomainId, 10)) + } + if f.NSRecordId > 0 { + conditions = append(conditions, "record_id = "+strconv.FormatInt(f.NSRecordId, 10)) + } + if f.RecordType != "" { + conditions = append(conditions, "question_type = '"+escapeString(f.RecordType)+"'") + } + if f.Keyword != "" { + keyword := escapeString(f.Keyword) + conditions = append(conditions, fmt.Sprintf("(remote_addr LIKE '%%%s%%' OR question_name LIKE '%%%s%%' OR record_value LIKE '%%%s%%' OR error LIKE '%%%s%%')", keyword, keyword, keyword, keyword)) + } + + // 游标分页：reverse=false 查更旧，reverse=true 查更新。 + if f.LastRequestId != "" { + if f.Reverse { + conditions = append(conditions, "request_id > '"+escapeString(f.LastRequestId)+"'") + } else { + conditions = append(conditions, "request_id < '"+escapeString(f.LastRequestId)+"'") + } + } + + orderDir := "DESC" + if f.Reverse { + orderDir = "ASC" + } + + limit := f.Size + if limit <= 0 { + limit = 20 + } + if limit > 1000 { + limit = 1000 + } + + query := fmt.Sprintf( + "SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d", + table, + strings.Join(conditions, " AND "), + orderDir, + orderDir, + limit+1, + ) + + var rawRows []map[string]interface{} + if err = s.client.Query(ctx, query, &rawRows); err != nil { + return nil, "", false, err + } + + rows = make([]*NSLogsIngestRow, 0, len(rawRows)) + for _, rawRow := range rawRows { + row := mapToNSLogsIngestRow(rawRow) + if row != nil { + rows = append(rows, row) + } + } + + hasMore = len(rows) > int(limit) + if hasMore { + nextCursor = rows[limit].RequestId + rows = rows[:limit] + } else if len(rows) > 0 { + nextCursor = rows[len(rows)-1].RequestId + } + + if f.Reverse { + for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 { + rows[left], rows[right] = rows[right], rows[left] + } + if len(rows) > 0 { + nextCursor = rows[0].RequestId + } + } + + return rows, nextCursor, hasMore, nil +} + +// FindByRequestId 按 request_id 查单条 DNS 日志. +func (s *NSLogsIngestStore) FindByRequestId(ctx context.Context, requestId string) (*NSLogsIngestRow, error) { + if !s.client.IsConfigured() { + return nil, fmt.Errorf("clickhouse: not configured") + } + if requestId == "" { + return nil, nil + } + + table := quoteIdent("dns_logs_ingest") + if s.client.cfg.Database != "" && s.client.cfg.Database != "default" { + table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("dns_logs_ingest") + } + + query := fmt.Sprintf( + "SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE request_id = '%s' LIMIT 1", + table, + escapeString(requestId), + ) + + var rawRows []map[string]interface{} + if err := s.client.Query(ctx, query, &rawRows); err != nil { + return nil, err + } + if len(rawRows) == 0 { + return nil, nil + } + return mapToNSLogsIngestRow(rawRows[0]), nil +} + +func mapToNSLogsIngestRow(row map[string]interface{}) *NSLogsIngestRow { + result := &NSLogsIngestRow{} + + u64 := func(key string) uint64 { + value, ok := row[key] + if !ok || value == nil { + return 0 + } + switch typed := value.(type) { + case float64: + return uint64(typed) + case string: + number, _ := strconv.ParseUint(typed, 10, 64) + return number + case json.Number: + number, _ := typed.Int64() + return uint64(number) + case int64: + return uint64(typed) + case uint64: + return typed + } + return 0 + } + + str := func(key string) string { + value, ok := row[key] + if !ok || value == nil { + return "" + } + switch typed := value.(type) { + case string: + return typed + case json.Number: + return typed.String() + default: + return fmt.Sprintf("%v", typed) + } + } + + ts := func(key string) time.Time { + value, ok := row[key] + if !ok || value == nil { + return time.Time{} + } + switch typed := value.(type) { + case string: + if typed == "" { + return time.Time{} + } + layouts := []string{ + "2006-01-02 15:04:05", + time.RFC3339, + "2006-01-02T15:04:05", + } + for _, layout := range layouts { + if parsed, err := time.Parse(layout, typed); err == nil { + return parsed + } + } + case float64: + return time.Unix(int64(typed), 0) + case json.Number: + number, _ := typed.Int64() + return time.Unix(number, 0) + } + return time.Time{} + } + + boolValue := func(key string) bool { + value, ok := row[key] + if !ok || value == nil { + return false + } + switch typed := value.(type) { + case bool: + return typed + case float64: + return typed > 0 + case int64: + return typed > 0 + case uint64: + return typed > 0 + case string: + switch strings.ToLower(strings.TrimSpace(typed)) { + case "1", "true", "yes": + return true + } + } + return false + } + + parseStringArray := func(key string) []string { + value, ok := row[key] + if !ok || value == nil { + return nil + } + switch typed := value.(type) { + case []string: + return typed + case []interface{}: + result := make([]string, 0, len(typed)) + for _, one := range typed { + if one == nil { + continue + } + result = append(result, fmt.Sprintf("%v", one)) + } + return result + case string: + if typed == "" { + return nil + } + var result []string + if json.Unmarshal([]byte(typed), &result) == nil { + return result + } + return []string{typed} + } + return nil + } + + result.Timestamp = ts("timestamp") + result.RequestId = str("request_id") + result.NodeId = u64("node_id") + result.ClusterId = u64("cluster_id") + result.DomainId = u64("domain_id") + result.RecordId = u64("record_id") + result.RemoteAddr = str("remote_addr") + result.QuestionName = str("question_name") + result.QuestionType = str("question_type") + result.RecordName = str("record_name") + result.RecordType = str("record_type") + result.RecordValue = str("record_value") + result.Networking = str("networking") + result.IsRecursive = boolValue("is_recursive") + result.Error = str("error") + result.NSRouteCodes = parseStringArray("ns_route_codes") + result.ContentJSON = str("content_json") + return result +} + +// NSRowToPB 将 ClickHouse 行转换为 pb.NSAccessLog. +func NSRowToPB(row *NSLogsIngestRow) *pb.NSAccessLog { + if row == nil { + return nil + } + + log := &pb.NSAccessLog{ + NsNodeId: int64(row.NodeId), + NsDomainId: int64(row.DomainId), + NsRecordId: int64(row.RecordId), + NsRouteCodes: row.NSRouteCodes, + RemoteAddr: row.RemoteAddr, + QuestionName: row.QuestionName, + QuestionType: row.QuestionType, + RecordName: row.RecordName, + RecordType: row.RecordType, + RecordValue: row.RecordValue, + Networking: row.Networking, + Timestamp: row.Timestamp.Unix(), + RequestId: row.RequestId, + Error: row.Error, + IsRecursive: row.IsRecursive, + } + if !row.Timestamp.IsZero() { + log.TimeLocal = row.Timestamp.Format("2/Jan/2006:15:04:05 -0700") + } + + if row.ContentJSON != "" { + contentLog := &pb.NSAccessLog{} + if json.Unmarshal([]byte(row.ContentJSON), contentLog) == nil { + contentLog.RequestId = row.RequestId + return contentLog + } + } + + return log +} diff --git a/EdgeAPI/internal/db/models/http_access_log_policy_dao.go b/EdgeAPI/internal/db/models/http_access_log_policy_dao.go index 2bf011a..57ab440 100644 --- a/EdgeAPI/internal/db/models/http_access_log_policy_dao.go +++ b/EdgeAPI/internal/db/models/http_access_log_policy_dao.go @@ -108,6 +108,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res // CreatePolicy 创建策略 func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) { + _ = writeTargetsJSON var op = NewHTTPAccessLogPolicyOperator() op.Name = name op.Type = policyType @@ -121,15 +122,14 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy op.IsOn = true op.FirewallOnly = firewallOnly op.DisableDefaultDB = disableDefaultDB - if len(writeTargetsJSON) > 0 { - op.WriteTargets = writeTargetsJSON - } + op.WriteTargets = dbs.SQL("NULL") op.State = HTTPAccessLogPolicyStateEnabled return this.SaveInt64(tx, op) } // UpdatePolicy 修改策略 func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error { + _ = writeTargetsJSON if policyId <= 0 { return errors.New("invalid policyId") } @@ -167,9 +167,7 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam op.IsPublic = isPublic op.FirewallOnly = firewallOnly op.DisableDefaultDB = disableDefaultDB - if len(writeTargetsJSON) > 0 { - op.WriteTargets = writeTargetsJSON - } + op.WriteTargets = dbs.SQL("NULL") op.IsOn = isOn return this.Save(tx, op) } diff --git a/EdgeAPI/internal/db/models/http_access_log_policy_utils.go b/EdgeAPI/internal/db/models/http_access_log_policy_utils.go new file mode 100644 index 0000000..cfa151e --- /dev/null +++ b/EdgeAPI/internal/db/models/http_access_log_policy_utils.go @@ -0,0 +1,22 @@ +package models + +import ( + "encoding/json" + "strings" + + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" +) + +// ParseHTTPAccessLogPolicyFilePath 提取访问日志策略中的文件路径（所有 file* 类型有效）。 +func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string { + if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 { + return "" + } + + config := &serverconfigs.AccessLogFileStorageConfig{} + if err := json.Unmarshal(policy.Options, config); err != nil { + return "" + } + + return strings.TrimSpace(config.Path) +} diff --git a/EdgeAPI/internal/db/models/node_dao.go b/EdgeAPI/internal/db/models/node_dao.go index e052e7d..7805a72 100644 --- a/EdgeAPI/internal/db/models/node_dao.go +++ b/EdgeAPI/internal/db/models/node_dao.go @@ -1174,7 +1174,8 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared if publicPolicyId > 0 { publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId) if publicPolicy != nil { - config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB) + config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ResolveWriteTargetsByType(publicPolicy.Type, publicPolicy.DisableDefaultDB) + config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) } } } diff --git a/EdgeAPI/internal/db/models/ns_node_dao_plus.go b/EdgeAPI/internal/db/models/ns_node_dao_plus.go index 65e6145..0e59852 100644 --- a/EdgeAPI/internal/db/models/ns_node_dao_plus.go +++ b/EdgeAPI/internal/db/models/ns_node_dao_plus.go @@ -472,6 +472,16 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs. } } } + { + publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx) + if publicPolicyId > 0 { + publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId) + if publicPolicy != nil { + config.AccessLogWriteTargets = serverconfigs.ResolveWriteTargetsByType(publicPolicy.Type, publicPolicy.DisableDefaultDB) + config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) + } + } + } // 递归DNS配置 if IsNotNull(cluster.Recursion) { diff --git a/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go b/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go index 51536a0..b4b17c5 100644 --- a/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go +++ b/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log.go @@ -4,6 +4,7 @@ package nameservers import ( "context" + "github.com/TeaOSLab/EdgeAPI/internal/clickhouse" "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers" "github.com/TeaOSLab/EdgeAPI/internal/rpc/services" @@ -29,11 +30,13 @@ func (this *NSAccessLogService) CreateNSAccessLogs(ctx context.Context, req *pb. return &pb.CreateNSAccessLogsResponse{}, nil } - var tx = this.NullTx() + if this.canWriteNSAccessLogsToDB() { + var tx = this.NullTx() - err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs) - if err != nil { - return nil, err + err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs) + if err != nil { + return nil, err + } } return &pb.CreateNSAccessLogsResponse{}, nil @@ -54,6 +57,34 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li // TODO 检查权限 } + store := clickhouse.NewNSLogsIngestStore() + canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured() && req.Day != "" + canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL() + if canReadFromClickHouse { + resp, listErr := this.listNSAccessLogsFromClickHouse(ctx, store, req) + if listErr == nil && resp != nil { + return resp, nil + } + if !canReadFromMySQL { + if listErr != nil { + return nil, listErr + } + return &pb.ListNSAccessLogsResponse{ + NsAccessLogs: []*pb.NSAccessLog{}, + HasMore: false, + RequestId: "", + }, nil + } + } + + if !canReadFromMySQL { + return &pb.ListNSAccessLogsResponse{ + NsAccessLogs: []*pb.NSAccessLog{}, + HasMore: false, + RequestId: "", + }, nil + } + accessLogs, requestId, hasMore, err := models.SharedNSAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NsClusterId, req.NsNodeId, req.NsDomainId, req.NsRecordId, req.RecordType, req.Keyword, req.Reverse) if err != nil { return nil, err @@ -67,23 +98,9 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li } // 线路 - if len(a.NsRouteCodes) > 0 { - for _, routeCode := range a.NsRouteCodes { - route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode) - if err != nil { - return nil, err - } - if route != nil { - a.NsRoutes = append(a.NsRoutes, &pb.NSRoute{ - Id: types.Int64(route.Id), - IsOn: route.IsOn, - Name: route.Name, - Code: routeCode, - NsCluster: nil, - NsDomain: nil, - }) - } - } + err = this.fillNSRoutes(a) + if err != nil { + return nil, err } result = append(result, a) @@ -104,6 +121,31 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin return nil, err } + store := clickhouse.NewNSLogsIngestStore() + canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured() + canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL() + if canReadFromClickHouse { + row, findErr := store.FindByRequestId(ctx, req.RequestId) + if findErr != nil { + if !canReadFromMySQL { + return nil, findErr + } + } else if row != nil { + a := clickhouse.NSRowToPB(row) + if a != nil { + err = this.fillNSRoutes(a) + if err != nil { + return nil, err + } + } + return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil + } + } + + if !canReadFromMySQL { + return &pb.FindNSAccessLogResponse{NsAccessLog: nil}, nil + } + var tx = this.NullTx() accessLog, err := models.SharedNSAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId) @@ -123,5 +165,70 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin if err != nil { return nil, err } + err = this.fillNSRoutes(a) + if err != nil { + return nil, err + } return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil } + +func (this *NSAccessLogService) listNSAccessLogsFromClickHouse(ctx context.Context, store *clickhouse.NSLogsIngestStore, req *pb.ListNSAccessLogsRequest) (*pb.ListNSAccessLogsResponse, error) { + rows, nextCursor, hasMore, err := store.List(ctx, clickhouse.NSListFilter{ + Day: req.Day, + Size: req.Size, + Reverse: req.Reverse, + LastRequestId: req.RequestId, + NSClusterId: req.NsClusterId, + NSNodeId: req.NsNodeId, + NSDomainId: req.NsDomainId, + NSRecordId: req.NsRecordId, + RecordType: req.RecordType, + Keyword: req.Keyword, + }) + if err != nil { + return nil, err + } + + result := make([]*pb.NSAccessLog, 0, len(rows)) + for _, row := range rows { + a := clickhouse.NSRowToPB(row) + if a == nil { + continue + } + err = this.fillNSRoutes(a) + if err != nil { + return nil, err + } + result = append(result, a) + } + + return &pb.ListNSAccessLogsResponse{ + NsAccessLogs: result, + HasMore: hasMore, + RequestId: nextCursor, + }, nil +} + +func (this *NSAccessLogService) fillNSRoutes(accessLog *pb.NSAccessLog) error { + if accessLog == nil || len(accessLog.NsRouteCodes) == 0 { + return nil + } + + for _, routeCode := range accessLog.NsRouteCodes { + route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode) + if err != nil { + return err + } + if route != nil { + accessLog.NsRoutes = append(accessLog.NsRoutes, &pb.NSRoute{ + Id: types.Int64(route.Id), + IsOn: route.IsOn, + Name: route.Name, + Code: routeCode, + NsCluster: nil, + NsDomain: nil, + }) + } + } + return nil +} diff --git a/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log_ext_plus.go b/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log_ext_plus.go new file mode 100644 index 0000000..fbe6774 --- /dev/null +++ b/EdgeAPI/internal/rpc/services/nameservers/service_ns_access_log_ext_plus.go @@ -0,0 +1,17 @@ +//go:build plus + +package nameservers + +import "github.com/TeaOSLab/EdgeAPI/internal/accesslogs" + +func (this *NSAccessLogService) canWriteNSAccessLogsToDB() bool { + return accesslogs.SharedStorageManager.WriteMySQL() +} + +func (this *NSAccessLogService) shouldReadNSAccessLogsFromClickHouse() bool { + return accesslogs.SharedStorageManager.WriteClickHouse() +} + +func (this *NSAccessLogService) shouldReadNSAccessLogsFromMySQL() bool { + return accesslogs.SharedStorageManager.WriteMySQL() +} diff --git a/EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go b/EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go index 1823c26..ab37ae6 100644 --- a/EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go +++ b/EdgeAPI/internal/rpc/services/service_http_access_log_policy_plus.go @@ -8,12 +8,27 @@ import ( "github.com/TeaOSLab/EdgeAPI/internal/accesslogs" "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" ) type HTTPAccessLogPolicyService struct { BaseService } +func (this *HTTPAccessLogPolicyService) normalizeStorageTypeAndTargets(policyType string, writeTargetsJSON []byte, disableDefaultDB bool) (string, []byte) { + _ = writeTargetsJSON + _ = disableDefaultDB + + // 兼容旧前端/缓存可能传来的历史类型编码 + switch policyType { + case "clickhouse": + policyType = serverconfigs.AccessLogStorageTypeFileClickhouse + case "mysql_clickhouse": + policyType = serverconfigs.AccessLogStorageTypeFileMySQLClickhouse + } + return policyType, nil +} + // CountAllHTTPAccessLogPolicies 计算访问日志策略数量 func (this *HTTPAccessLogPolicyService) CountAllHTTPAccessLogPolicies(ctx context.Context, req *pb.CountAllHTTPAccessLogPoliciesRequest) (*pb.RPCCountResponse, error) { _, err := this.ValidateAdmin(ctx) @@ -53,7 +68,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co IsPublic: policy.IsPublic, FirewallOnly: policy.FirewallOnly == 1, DisableDefaultDB: policy.DisableDefaultDB, - WriteTargetsJSON: policy.WriteTargets, + WriteTargetsJSON: nil, }) } return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil @@ -76,8 +91,10 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co } } + policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB) + // 创建 - policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON) + policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON) if err != nil { return nil, err } @@ -101,8 +118,10 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co } } + policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB) + // 保存修改 - err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn) + err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON, req.IsOn) if err != nil { return nil, err } @@ -134,7 +153,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont IsPublic: policy.IsPublic, FirewallOnly: policy.FirewallOnly == 1, DisableDefaultDB: policy.DisableDefaultDB, - WriteTargetsJSON: policy.WriteTargets, + WriteTargetsJSON: nil, }}, nil } diff --git a/EdgeAdmin/build/build-all-plus.sh b/EdgeAdmin/build/build-all-plus.sh index 7d4c8f5..2c3b9b9 100644 --- a/EdgeAdmin/build/build-all-plus.sh +++ b/EdgeAdmin/build/build-all-plus.sh @@ -18,7 +18,7 @@ echo "build all edge-admin" echo "==============================" ./build.sh linux amd64 plus #./build.sh linux 386 plus -./build.sh linux arm64 plus +#./build.sh linux arm64 plus #./build.sh linux mips64 plus #./build.sh linux mips64le plus #./build.sh darwin amd64 plus @@ -26,4 +26,3 @@ echo "==============================" - diff --git a/EdgeAdmin/build/build-all.sh b/EdgeAdmin/build/build-all.sh index ad11dad..26019fc 100644 --- a/EdgeAdmin/build/build-all.sh +++ b/EdgeAdmin/build/build-all.sh @@ -14,8 +14,8 @@ fi ./build.sh linux amd64 ./build.sh linux 386 -./build.sh linux arm64 +#./build.sh linux arm64 ./build.sh linux mips64 ./build.sh linux mips64le ./build.sh darwin amd64 -./build.sh darwin arm64 +#./build.sh darwin arm64 diff --git a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/createPopup.go b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/createPopup.go index fd53d4a..228c5ef 100644 --- a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/createPopup.go +++ b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/createPopup.go @@ -87,17 +87,20 @@ func (this *CreatePopupAction) RunPost(params struct { Field("type", params.Type). Require("请选择存储类型") - baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type) - if writeTargets == nil { - writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true} + baseType, _ := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type) + storedType := baseType + if serverconfigs.IsFileBasedStorageType(params.Type) { + storedType = params.Type } var options any = nil switch baseType { case serverconfigs.AccessLogStorageTypeFile: - params.Must. - Field("filePath", params.FilePath). - Require("请输入日志文件路径") + if !serverconfigs.IsFileBasedStorageType(params.Type) || (params.Type != serverconfigs.AccessLogStorageTypeFileClickhouse && params.Type != serverconfigs.AccessLogStorageTypeFileMySQLClickhouse) { + params.Must. + Field("filePath", params.FilePath). + Require("请输入日志文件路径") + } var storage = new(serverconfigs.AccessLogFileStorageConfig) storage.Path = params.FilePath @@ -175,21 +178,15 @@ func (this *CreatePopupAction) RunPost(params struct { this.ErrorPage(err) return } - writeTargetsMap := map[string]bool{ - "file": writeTargets.File, - "mysql": writeTargets.MySQL, - "clickhouse": writeTargets.ClickHouse, - } - writeTargetsJSON, _ := json.Marshal(writeTargetsMap) createResp, err := this.RPC().HTTPAccessLogPolicyRPC().CreateHTTPAccessLogPolicy(this.AdminContext(), &pb.CreateHTTPAccessLogPolicyRequest{ Name: params.Name, - Type: baseType, + Type: storedType, OptionsJSON: optionsJSON, CondsJSON: nil, // TODO IsPublic: params.IsPublic, FirewallOnly: params.FirewallOnly, DisableDefaultDB: params.DisableDefaultDB, - WriteTargetsJSON: writeTargetsJSON, + WriteTargetsJSON: nil, }) if err != nil { this.ErrorPage(err) diff --git a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/index.go b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/index.go index 4cded29..d71a87a 100644 --- a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/index.go +++ b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/index.go @@ -46,11 +46,7 @@ func (this *IndexAction) RunGet(params struct{}) { return } } - writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB) - typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets) - if typeDisplay == "" { - typeDisplay = policy.Type - } + typeDisplay := policy.Type policyMaps = append(policyMaps, maps.Map{ "id": policy.Id, "name": policy.Name, diff --git a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/policyutils/utils.go b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/policyutils/utils.go index bf27397..69b74ba 100644 --- a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/policyutils/utils.go +++ b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/policyutils/utils.go @@ -36,11 +36,7 @@ func InitPolicy(parent *actionutils.ParentAction, policyId int64) error { } } - writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB) - typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets) - if typeDisplay == "" { - typeDisplay = policy.Type - } + typeDisplay := policy.Type parent.Data["policy"] = maps.Map{ "id": policy.Id, diff --git a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/update.go b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/update.go index 6a7ca06..fe6f192 100644 --- a/EdgeAdmin/internal/web/actions/default/servers/accesslogs/update.go +++ b/EdgeAdmin/internal/web/actions/default/servers/accesslogs/update.go @@ -106,21 +106,30 @@ func (this *UpdateAction) RunPost(params struct { Field("type", params.Type). Require("请选择存储类型") - baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type) - if writeTargets == nil { - writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true} + baseType, _ := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type) + storedType := baseType + if serverconfigs.IsFileBasedStorageType(params.Type) { + storedType = params.Type } var options interface{} = nil switch baseType { case serverconfigs.AccessLogStorageTypeFile: - params.Must. - Field("filePath", params.FilePath). - Require("请输入日志文件路径") - var storage = new(serverconfigs.AccessLogFileStorageConfig) - storage.Path = params.FilePath - storage.AutoCreate = params.FileAutoCreate + if params.Type == serverconfigs.AccessLogStorageTypeFileClickhouse || params.Type == serverconfigs.AccessLogStorageTypeFileMySQLClickhouse { + if len(policy.OptionsJSON) > 0 { + _ = json.Unmarshal(policy.OptionsJSON, storage) + } + if len(params.FilePath) > 0 { + storage.Path = params.FilePath + } + } else { + params.Must. + Field("filePath", params.FilePath). + Require("请输入日志文件路径") + storage.Path = params.FilePath + storage.AutoCreate = params.FileAutoCreate + } options = storage case serverconfigs.AccessLogStorageTypeES: params.Must. @@ -195,23 +204,17 @@ func (this *UpdateAction) RunPost(params struct { this.ErrorPage(err) return } - writeTargetsMap := map[string]bool{ - "file": writeTargets.File, - "mysql": writeTargets.MySQL, - "clickhouse": writeTargets.ClickHouse, - } - writeTargetsJSON, _ := json.Marshal(writeTargetsMap) _, err = this.RPC().HTTPAccessLogPolicyRPC().UpdateHTTPAccessLogPolicy(this.AdminContext(), &pb.UpdateHTTPAccessLogPolicyRequest{ HttpAccessLogPolicyId: params.PolicyId, Name: params.Name, - Type: baseType, + Type: storedType, OptionsJSON: optionsJSON, CondsJSON: nil, // TODO IsOn: params.IsOn, IsPublic: params.IsPublic, FirewallOnly: params.FirewallOnly, DisableDefaultDB: params.DisableDefaultDB, - WriteTargetsJSON: writeTargetsJSON, + WriteTargetsJSON: nil, }) if err != nil { this.ErrorPage(err) diff --git a/EdgeAdmin/web/views/@default/servers/accesslogs/createPopup.html b/EdgeAdmin/web/views/@default/servers/accesslogs/createPopup.html index 3653918..001b802 100644 --- a/EdgeAdmin/web/views/@default/servers/accesslogs/createPopup.html +++ b/EdgeAdmin/web/views/@default/servers/accesslogs/createPopup.html @@ -21,8 +21,8 @@ - - + + 日志文件路径 * @@ -36,7 +36,9 @@ 时：${hour} 分：${minute} 秒：${second} - 年月日：${date}，比如/var/log/web-access-${date}.log，此文件会在API节点上写入。 + 年月日：${date}，比如/var/log/web-access-${date}.log。 + 当存储类型包含 ClickHouse 时，此文件会在节点侧写入，并由 Fluent Bit 采集后写入 ClickHouse。 + 此文件会在API节点上写入。

@@ -51,6 +53,15 @@ + + + + 日志文件路径 + +

当前类型包含 ClickHouse，日志文件路径将由节点侧按公用策略自动复用，或回退到默认日志目录，无需手动输入。

+ + + @@ -242,4 +253,4 @@ - \ No newline at end of file + diff --git a/EdgeAdmin/web/views/@default/servers/accesslogs/update.html b/EdgeAdmin/web/views/@default/servers/accesslogs/update.html index 0353a07..4e32dc5 100644 --- a/EdgeAdmin/web/views/@default/servers/accesslogs/update.html +++ b/EdgeAdmin/web/views/@default/servers/accesslogs/update.html @@ -22,8 +22,8 @@ - - + + 日志文件路径 * @@ -37,7 +37,9 @@ 时：${hour} 分：${minute} 秒：${second} - 年月日：${date}，比如/var/log/web-access-${date}.log，此文件会在API节点上写入。 + 年月日：${date}，比如/var/log/web-access-${date}.log。 + 当存储类型包含 ClickHouse 时，此文件会在节点侧写入，并由 Fluent Bit 采集后写入 ClickHouse。 + 此文件会在API节点上写入。

@@ -52,6 +54,15 @@ + + + + 日志文件路径 + +

当前类型包含 ClickHouse，日志文件路径将由节点侧按公用策略自动复用，或回退到默认日志目录，无需手动输入。

+ + + @@ -247,4 +258,4 @@ - \ No newline at end of file + diff --git a/EdgeCommon/pkg/dnsconfigs/ns_node_config.go b/EdgeCommon/pkg/dnsconfigs/ns_node_config.go index 5427b47..64d4188 100644 --- a/EdgeCommon/pkg/dnsconfigs/ns_node_config.go +++ b/EdgeCommon/pkg/dnsconfigs/ns_node_config.go @@ -11,21 +11,23 @@ import ( ) type NSNodeConfig struct { - Id int64 `yaml:"id" json:"id"` - IsPlus bool `yaml:"isPlus" json:"isPlus"` - NodeId string `yaml:"nodeId" json:"nodeId"` - Secret string `yaml:"secret" json:"secret"` - ClusterId int64 `yaml:"clusterId" json:"clusterId"` - AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"` - RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"` - DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"` - AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"` - TimeZone string `yaml:"timeZone" json:"timeZone"` // 自动设置时区 - Hosts []string `yaml:"hosts" json:"hosts"` // 主机名 - Email string `yaml:"email" json:"email"` - SOA *NSSOAConfig `yaml:"soa" json:"soa"` // SOA配置 - SOASerial uint32 `yaml:"soaSerial" json:"soaSerial"` - DetectAgents bool `yaml:"detectAgents" json:"detectAgents"` // 是否实时监测Agents + Id int64 `yaml:"id" json:"id"` + IsPlus bool `yaml:"isPlus" json:"isPlus"` + NodeId string `yaml:"nodeId" json:"nodeId"` + Secret string `yaml:"secret" json:"secret"` + ClusterId int64 `yaml:"clusterId" json:"clusterId"` + AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"` + AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"` + AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"` + RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"` + DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"` + AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"` + TimeZone string `yaml:"timeZone" json:"timeZone"` // 自动设置时区 + Hosts []string `yaml:"hosts" json:"hosts"` // 主机名 + Email string `yaml:"email" json:"email"` + SOA *NSSOAConfig `yaml:"soa" json:"soa"` // SOA配置 + SOASerial uint32 `yaml:"soaSerial" json:"soaSerial"` + DetectAgents bool `yaml:"detectAgents" json:"detectAgents"` // 是否实时监测Agents TCP *serverconfigs.TCPProtocolConfig `yaml:"tcp" json:"tcp"` // TCP配置 TLS *serverconfigs.TLSProtocolConfig `yaml:"tls" json:"tls"` // TLS配置 diff --git a/EdgeCommon/pkg/serverconfigs/access_log_storages.go b/EdgeCommon/pkg/serverconfigs/access_log_storages.go index 7bf6673..275f318 100644 --- a/EdgeCommon/pkg/serverconfigs/access_log_storages.go +++ b/EdgeCommon/pkg/serverconfigs/access_log_storages.go @@ -84,6 +84,33 @@ func ParseStorageTypeAndWriteTargets(selectedType string) (baseType string, writ return baseType, writeTargets } +// ResolveWriteTargetsByType 仅根据策略类型与 disableDefaultDB 计算写入目标（不依赖 writeTargets 字段） +func ResolveWriteTargetsByType(policyType string, disableDefaultDB bool) *AccessLogWriteTargets { + t := &AccessLogWriteTargets{} + switch policyType { + case AccessLogStorageTypeFileMySQL: + t.File = true + t.MySQL = true + case AccessLogStorageTypeFileClickhouse: + t.File = true + t.ClickHouse = true + case AccessLogStorageTypeFileMySQLClickhouse: + t.File = true + t.MySQL = true + t.ClickHouse = true + case AccessLogStorageTypeFile: + t.File = true + t.MySQL = !disableDefaultDB + default: + t.MySQL = !disableDefaultDB + } + if !t.File && !t.MySQL && !t.ClickHouse { + t.File = true + t.MySQL = true + } + return t +} + // ComposeStorageTypeDisplay 根据策略的 Type + WriteTargets 得到下拉框显示用的类型 code（用于编辑页回显） func ComposeStorageTypeDisplay(policyType string, writeTargets *AccessLogWriteTargets) string { if policyType != AccessLogStorageTypeFile { diff --git a/EdgeCommon/pkg/serverconfigs/access_log_write_targets.go b/EdgeCommon/pkg/serverconfigs/access_log_write_targets.go index 1e2b03b..be19ade 100644 --- a/EdgeCommon/pkg/serverconfigs/access_log_write_targets.go +++ b/EdgeCommon/pkg/serverconfigs/access_log_write_targets.go @@ -2,8 +2,6 @@ package serverconfigs -import "encoding/json" - // AccessLogWriteTargets 访问日志写入目标（双写/单写：文件、MySQL、ClickHouse） type AccessLogWriteTargets struct { File bool `yaml:"file" json:"file"` // 写本地 JSON 文件（供 Fluent Bit → ClickHouse 或自用） @@ -27,23 +25,8 @@ func (t *AccessLogWriteTargets) NeedWriteFile() bool { return t.File } -// ParseWriteTargetsFromPolicy 从策略的 writeTargets JSON 与旧字段解析；无 writeTargets 时按 type + disableDefaultDB 推断 +// ParseWriteTargetsFromPolicy 兼容入口：当前统一按 type 推导写入目标，不再依赖 writeTargets 字段 func ParseWriteTargetsFromPolicy(writeTargetsJSON []byte, policyType string, disableDefaultDB bool) *AccessLogWriteTargets { - if len(writeTargetsJSON) > 0 { - var t AccessLogWriteTargets - if err := json.Unmarshal(writeTargetsJSON, &t); err == nil { - return &t - } - } - // 兼容旧策略：type=file 视为写文件，!disableDefaultDB 视为写 MySQL - t := &AccessLogWriteTargets{ - File: policyType == AccessLogStorageTypeFile, - MySQL: !disableDefaultDB, - ClickHouse: false, - } - if !t.File && !t.MySQL && !t.ClickHouse { - t.File = true - t.MySQL = true - } - return t + _ = writeTargetsJSON + return ResolveWriteTargetsByType(policyType, disableDefaultDB) } diff --git a/EdgeCommon/pkg/serverconfigs/global_server_config.go b/EdgeCommon/pkg/serverconfigs/global_server_config.go index f2e23bd..3cb65e5 100644 --- a/EdgeCommon/pkg/serverconfigs/global_server_config.go +++ b/EdgeCommon/pkg/serverconfigs/global_server_config.go @@ -78,6 +78,7 @@ type GlobalServerConfig struct { EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志 WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标：文件/MySQL/ClickHouse（双写/单写） + FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径（用于节点侧复用） } `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置 Stat struct { diff --git a/EdgeDNS/build/build-all.sh b/EdgeDNS/build/build-all.sh index bcc2e23..9ee68d4 100644 --- a/EdgeDNS/build/build-all.sh +++ b/EdgeDNS/build/build-all.sh @@ -2,8 +2,8 @@ ./build.sh linux amd64 #./build.sh linux 386 -./build.sh linux arm64 +#./build.sh linux arm64 #./build.sh linux mips64 #./build.sh linux mips64le #./build.sh darwin amd64 -#./build.sh darwin arm64 \ No newline at end of file +#./build.sh darwin arm64 diff --git a/EdgeDNS/go.mod b/EdgeDNS/go.mod index b857817..b470d94 100644 --- a/EdgeDNS/go.mod +++ b/EdgeDNS/go.mod @@ -24,6 +24,8 @@ require ( github.com/josharian/native v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mdlayher/socket v0.5.0 // indirect + github.com/oschwald/geoip2-golang v1.13.0 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.13 // indirect diff --git a/EdgeDNS/go.sum b/EdgeDNS/go.sum index c2c985b..ce34f7c 100644 --- a/EdgeDNS/go.sum +++ b/EdgeDNS/go.sum @@ -1,15 +1,23 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8= github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/iwind/TeaGo v0.0.0-20240128112714-6bcd0529d0ea h1:o0QCF6tMJ9E6OgU1c0L+rYDshKsTu7mEk+7KCGLbnpI= github.com/iwind/TeaGo v0.0.0-20240128112714-6bcd0529d0ea/go.mod h1:Ng3xWekHSVy0E/6/jYqJ7Htydm/H+mWIl0AS+Eg3H2M= github.com/iwind/gosock v0.0.0-20220505115348-f88412125a62 h1:HJH6RDheAY156DnIfJSD/bEvqyXzsZuE2gzs8PuUjoo= @@ -30,6 +38,10 @@ github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= +github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -45,8 +57,9 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -57,39 +70,43 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrB github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/EdgeDNS/internal/accesslogs/dns_file_writer.go b/EdgeDNS/internal/accesslogs/dns_file_writer.go new file mode 100644 index 0000000..35b5500 --- /dev/null +++ b/EdgeDNS/internal/accesslogs/dns_file_writer.go @@ -0,0 +1,199 @@ +package accesslogs + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeDNS/internal/remotelogs" +) + +const ( + defaultDNSLogDir = "/var/log/edge/edge-dns" + envDNSLogDir = "EDGE_DNS_LOG_DIR" +) + +var ( + sharedDNSFileWriter *DNSFileWriter + sharedDNSFileWriterOnce sync.Once +) + +// SharedDNSFileWriter 返回 DNS 本地日志写入器（单例）. +func SharedDNSFileWriter() *DNSFileWriter { + sharedDNSFileWriterOnce.Do(func() { + sharedDNSFileWriter = NewDNSFileWriter() + }) + return sharedDNSFileWriter +} + +// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集. +type DNSFileWriter struct { + dir string + mu sync.Mutex + file *os.File + inited bool +} + +// NewDNSFileWriter 创建 DNS 本地日志写入器. +func NewDNSFileWriter() *DNSFileWriter { + logDir := resolveDefaultDNSLogDir() + return &DNSFileWriter{dir: logDir} +} + +func resolveDefaultDNSLogDir() string { + logDir := strings.TrimSpace(os.Getenv(envDNSLogDir)) + if logDir == "" { + return defaultDNSLogDir + } + return logDir +} + +func resolveDNSDirFromPolicyPath(policyPath string) string { + policyPath = strings.TrimSpace(policyPath) + if policyPath == "" { + return "" + } + + if strings.HasSuffix(policyPath, "/") || strings.HasSuffix(policyPath, "\\") { + return filepath.Clean(policyPath) + } + + baseName := filepath.Base(policyPath) + if strings.Contains(baseName, ".") || strings.Contains(baseName, "${") { + return filepath.Clean(filepath.Dir(policyPath)) + } + + return filepath.Clean(policyPath) +} + +// Dir 返回当前日志目录. +func (w *DNSFileWriter) Dir() string { + return w.dir +} + +// SetDirByPolicyPath 使用公用日志策略 path 更新目录，空值时回退到 EDGE_DNS_LOG_DIR/default。 +func (w *DNSFileWriter) SetDirByPolicyPath(policyPath string) { + dir := resolveDNSDirFromPolicyPath(policyPath) + w.SetDir(dir) +} + +// SetDir 更新目录并重置文件句柄。 +func (w *DNSFileWriter) SetDir(dir string) { + if strings.TrimSpace(dir) == "" { + dir = resolveDefaultDNSLogDir() + } + + w.mu.Lock() + defer w.mu.Unlock() + + if dir == w.dir { + return + } + + if w.file != nil { + _ = w.file.Close() + w.file = nil + } + w.inited = false + w.dir = dir +} + +// EnsureInit 在启动时预创建目录与 access.log. +func (w *DNSFileWriter) EnsureInit() error { + if w.dir == "" { + return nil + } + return w.init() +} + +func (w *DNSFileWriter) init() error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.inited && w.file != nil { + return nil + } + + if w.dir == "" { + return nil + } + + if err := os.MkdirAll(w.dir, 0755); err != nil { + remotelogs.Error("DNS_ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error()) + return err + } + + fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error()) + return err + } + + w.file = fp + w.inited = true + return nil +} + +// WriteBatch 批量写入 DNS 访问日志. +func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) { + if len(logs) == 0 || w.dir == "" { + return + } + if err := w.init(); err != nil { + return + } + + w.mu.Lock() + fp := w.file + w.mu.Unlock() + if fp == nil { + return + } + + for _, log := range logs { + ingestLog := FromNSAccessLog(log, clusterId) + if ingestLog == nil { + continue + } + line, err := json.Marshal(ingestLog) + if err != nil { + continue + } + _, _ = fp.Write(append(line, '\n')) + } +} + +// Reopen 关闭并重新打开日志文件（配合 logrotate）. +func (w *DNSFileWriter) Reopen() error { + w.mu.Lock() + if w.file != nil { + _ = w.file.Close() + w.file = nil + } + w.inited = false + w.mu.Unlock() + return w.init() +} + +// Close 关闭日志文件. +func (w *DNSFileWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.file == nil { + return nil + } + + err := w.file.Close() + w.file = nil + w.inited = false + if err != nil { + remotelogs.Error("DNS_ACCESS_LOG_FILE", fmt.Sprintf("close access.log failed: %v", err)) + return err + } + return nil +} diff --git a/EdgeDNS/internal/accesslogs/dns_ingest_log.go b/EdgeDNS/internal/accesslogs/dns_ingest_log.go new file mode 100644 index 0000000..55cdf6f --- /dev/null +++ b/EdgeDNS/internal/accesslogs/dns_ingest_log.go @@ -0,0 +1,57 @@ +package accesslogs + +import ( + "encoding/json" + + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" +) + +// DNSIngestLog DNS 访问日志单行结构（JSONEachRow）. +type DNSIngestLog struct { + Timestamp int64 `json:"timestamp"` + RequestId string `json:"request_id"` + NodeId int64 `json:"node_id"` + ClusterId int64 `json:"cluster_id"` + DomainId int64 `json:"domain_id"` + RecordId int64 `json:"record_id"` + RemoteAddr string `json:"remote_addr"` + QuestionName string `json:"question_name"` + QuestionType string `json:"question_type"` + RecordName string `json:"record_name"` + RecordType string `json:"record_type"` + RecordValue string `json:"record_value"` + Networking string `json:"networking"` + IsRecursive bool `json:"is_recursive"` + Error string `json:"error"` + NSRouteCodes []string `json:"ns_route_codes,omitempty"` + ContentJSON string `json:"content_json,omitempty"` +} + +// FromNSAccessLog 将 pb.NSAccessLog 转为 DNSIngestLog. +func FromNSAccessLog(log *pb.NSAccessLog, clusterId int64) *DNSIngestLog { + if log == nil { + return nil + } + + contentBytes, _ := json.Marshal(log) + + return &DNSIngestLog{ + Timestamp: log.GetTimestamp(), + RequestId: log.GetRequestId(), + NodeId: log.GetNsNodeId(), + ClusterId: clusterId, + DomainId: log.GetNsDomainId(), + RecordId: log.GetNsRecordId(), + RemoteAddr: log.GetRemoteAddr(), + QuestionName: log.GetQuestionName(), + QuestionType: log.GetQuestionType(), + RecordName: log.GetRecordName(), + RecordType: log.GetRecordType(), + RecordValue: log.GetRecordValue(), + Networking: log.GetNetworking(), + IsRecursive: log.GetIsRecursive(), + Error: log.GetError(), + NSRouteCodes: log.GetNsRouteCodes(), + ContentJSON: string(contentBytes), + } +} diff --git a/EdgeDNS/internal/nodes/manager_node_config.go b/EdgeDNS/internal/nodes/manager_node_config.go index fa990eb..5066b07 100644 --- a/EdgeDNS/internal/nodes/manager_node_config.go +++ b/EdgeDNS/internal/nodes/manager_node_config.go @@ -10,6 +10,7 @@ import ( "github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" + "github.com/TeaOSLab/EdgeDNS/internal/accesslogs" "github.com/TeaOSLab/EdgeDNS/internal/configs" teaconst "github.com/TeaOSLab/EdgeDNS/internal/const" "github.com/TeaOSLab/EdgeDNS/internal/events" @@ -108,6 +109,15 @@ func (this *NodeConfigManager) NotifyChange() { func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) { teaconst.IsPlus = config.IsPlus + accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath) + + needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse + if needWriteFile { + _ = accesslogs.SharedDNSFileWriter().EnsureInit() + } else { + _ = accesslogs.SharedDNSFileWriter().Close() + } + // timezone var timeZone = config.TimeZone if len(timeZone) == 0 { diff --git a/EdgeDNS/internal/nodes/ns_access_log_queue.go b/EdgeDNS/internal/nodes/ns_access_log_queue.go index f350d16..a541151 100644 --- a/EdgeDNS/internal/nodes/ns_access_log_queue.go +++ b/EdgeDNS/internal/nodes/ns_access_log_queue.go @@ -2,6 +2,7 @@ package nodes import ( "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeDNS/internal/accesslogs" "github.com/TeaOSLab/EdgeDNS/internal/remotelogs" "github.com/TeaOSLab/EdgeDNS/internal/rpc" "strconv" @@ -89,6 +90,26 @@ Loop: return nil } + var clusterId int64 + var needWriteFile = true + var needReportAPI = true + if sharedNodeConfig != nil { + clusterId = sharedNodeConfig.ClusterId + if sharedNodeConfig.AccessLogWriteTargets != nil { + targets := sharedNodeConfig.AccessLogWriteTargets + needWriteFile = targets.File || targets.ClickHouse + needReportAPI = targets.MySQL + } + } + + if needWriteFile { + accesslogs.SharedDNSFileWriter().WriteBatch(accessLogs, clusterId) + } + + if !needReportAPI { + return nil + } + // 发送到API client, err := rpc.SharedRPC() if err != nil { diff --git a/EdgeNode/build/build-all-plus.sh b/EdgeNode/build/build-all-plus.sh index 2f16a4a..d91ba94 100644 --- a/EdgeNode/build/build-all-plus.sh +++ b/EdgeNode/build/build-all-plus.sh @@ -2,8 +2,8 @@ ./build.sh linux amd64 plus #./build.sh linux 386 plus -./build.sh linux arm64 plus +#./build.sh linux arm64 plus #./build.sh linux mips64 plus #./build.sh linux mips64le plus #./build.sh darwin amd64 plus -#./build.sh darwin arm64 plus \ No newline at end of file +#./build.sh darwin arm64 plus diff --git a/EdgeNode/build/build-all.sh b/EdgeNode/build/build-all.sh index bcc2e23..9ee68d4 100644 --- a/EdgeNode/build/build-all.sh +++ b/EdgeNode/build/build-all.sh @@ -2,8 +2,8 @@ ./build.sh linux amd64 #./build.sh linux 386 -./build.sh linux arm64 +#./build.sh linux arm64 #./build.sh linux mips64 #./build.sh linux mips64le #./build.sh darwin amd64 -#./build.sh darwin arm64 \ No newline at end of file +#./build.sh darwin arm64 diff --git a/EdgeNode/build/data/disk.speed.json b/EdgeNode/build/data/disk.speed.json index 8d3ae39..8ed61c3 100644 --- a/EdgeNode/build/data/disk.speed.json +++ b/EdgeNode/build/data/disk.speed.json @@ -1 +1 @@ -{"speed":2,"speedMB":670,"countTests":2} \ No newline at end of file +{"speed":1,"speedMB":1400,"countTests":3} \ No newline at end of file diff --git a/EdgeNode/internal/accesslogs/file_writer.go b/EdgeNode/internal/accesslogs/file_writer.go index da177b5..264df8f 100644 --- a/EdgeNode/internal/accesslogs/file_writer.go +++ b/EdgeNode/internal/accesslogs/file_writer.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" @@ -39,21 +40,73 @@ type FileWriter struct { // NewFileWriter 创建本地日志文件写入器 func NewFileWriter() *FileWriter { - dir := os.Getenv(envLogDir) - if dir == "" { - dir = defaultLogDir - } + dir := resolveDefaultLogDir() return &FileWriter{ dir: dir, files: make(map[string]*os.File), } } +func resolveDefaultLogDir() string { + dir := strings.TrimSpace(os.Getenv(envLogDir)) + if dir == "" { + return defaultLogDir + } + return dir +} + +func resolveDirFromPolicyPath(policyPath string) string { + policyPath = strings.TrimSpace(policyPath) + if policyPath == "" { + return "" + } + + if strings.HasSuffix(policyPath, "/") || strings.HasSuffix(policyPath, "\\") { + return filepath.Clean(policyPath) + } + + baseName := filepath.Base(policyPath) + if strings.Contains(baseName, ".") || strings.Contains(baseName, "${") { + return filepath.Clean(filepath.Dir(policyPath)) + } + + return filepath.Clean(policyPath) +} + // Dir 返回当前配置的日志目录 func (w *FileWriter) Dir() string { return w.dir } +// SetDirByPolicyPath 使用公用日志策略 path 更新目录，空值时回退到 EDGE_LOG_DIR/default。 +func (w *FileWriter) SetDirByPolicyPath(policyPath string) { + dir := resolveDirFromPolicyPath(policyPath) + w.SetDir(dir) +} + +// SetDir 更新日志目录并重置文件句柄。 +func (w *FileWriter) SetDir(dir string) { + if strings.TrimSpace(dir) == "" { + dir = resolveDefaultLogDir() + } + + w.mu.Lock() + defer w.mu.Unlock() + + if dir == w.dir { + return + } + + for name, f := range w.files { + if f != nil { + _ = f.Close() + } + w.files[name] = nil + } + w.inited = false + w.dir = dir +} + // IsEnabled 是否启用落盘（目录非空即视为启用） func (w *FileWriter) IsEnabled() bool { return w.dir != "" diff --git a/EdgeNode/internal/nodes/node.go b/EdgeNode/internal/nodes/node.go index 0d0ed36..77180c8 100644 --- a/EdgeNode/internal/nodes/node.go +++ b/EdgeNode/internal/nodes/node.go @@ -887,6 +887,12 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) { nodeconfigs.ResetNodeConfig(config) sharedNodeConfig = config + var accessLogFilePath string + if config != nil && config.GlobalServerConfig != nil { + accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath + } + accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath) + // 并发读写数 fsutils.ReaderLimiter.SetThreads(config.MaxConcurrentReads) fsutils.WriterLimiter.SetThreads(config.MaxConcurrentWrites) diff --git a/EdgeReporter/build/build-all.sh b/EdgeReporter/build/build-all.sh index f1a5ceb..ed976be 100644 --- a/EdgeReporter/build/build-all.sh +++ b/EdgeReporter/build/build-all.sh @@ -2,10 +2,10 @@ ./build.sh linux amd64 ./build.sh linux 386 -./build.sh linux arm64 +#./build.sh linux arm64 ./build.sh linux mips64 ./build.sh linux mips64le ./build.sh darwin amd64 -./build.sh darwin arm64 +#./build.sh darwin arm64 ./build.sh windows amd64 -./build.sh windows 386 \ No newline at end of file +./build.sh windows 386 diff --git a/EdgeUser/build/build-all.sh b/EdgeUser/build/build-all.sh index bcc2e23..9ee68d4 100644 --- a/EdgeUser/build/build-all.sh +++ b/EdgeUser/build/build-all.sh @@ -2,8 +2,8 @@ ./build.sh linux amd64 #./build.sh linux 386 -./build.sh linux arm64 +#./build.sh linux arm64 #./build.sh linux mips64 #./build.sh linux mips64le #./build.sh darwin amd64 -#./build.sh darwin arm64 \ No newline at end of file +#./build.sh darwin arm64 diff --git a/GoEdge HTTPDNS 需求文档 v2.0.docx b/GoEdge HTTPDNS 需求文档 v2.0.docx new file mode 100644 index 0000000..2b583de Binary files /dev/null and b/GoEdge HTTPDNS 需求文档 v2.0.docx differ diff --git a/HTTPDNS_技术实施方案.md b/HTTPDNS_技术实施方案.md new file mode 100644 index 0000000..dbc44b3 --- /dev/null +++ b/HTTPDNS_技术实施方案.md @@ -0,0 +1,1290 @@ +# GoEdge HTTPDNS 技术实施方案 + +> 版本: 1.0 | 作者: AI Assistant | 日期: 2026-02-09 + +--- + +## 一、项目概述 + +### 1.1 目标 +在 GoEdge 平台实现完整的 HTTPDNS 服务，包括： +- 基于 HTTPS 的 DNS 解析接口 +- 动态指纹校验（WAF） +- App 管理后台 +- 移动端 SDK 示例 + +### 1.2 设计决策 +| 决策项 | 选择 | 理由 | +|--------|------|------| +| WAF 指纹校验 | 必须 | 防止非法请求绕过解析直接攻击源站 | +| App 管理界面 | 必须 | 标准产品功能，支持多租户 | +| SDK 示例 | 提供 | 降低客户接入成本 | +| 部署位置 | 复用 Edge-DNS | 减少运维复杂度 | +| 接口路径 | 新增 `/httpdns/resolve` | 保持向后兼容 | + +--- + +## 二、系统架构 + +### 2.1 整体架构图 + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ 移动端 App │ +│ ┌───────────────────────────────────────────────────────────────────┐ │ +│ │ HTTPDNS SDK │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ +│ │ │ Resolver │ │CacheManager │ │NetworkMonitor│ │ │ +│ │ │ 解析引擎 │ │ 缓存管理 │ │ 网络感知 │ │ │ +│ │ │ -多节点容错 │ │ -内存LRU │ │ -切换监听 │ │ │ +│ │ │ -超时降级 │ │ -持久化 │ │ -自动清缓存 │ │ │ +│ │ └─────────────┘ │ -软过期 │ │ -IPv6检测 │ │ │ +│ │ └─────────────┘ └─────────────┘ │ │ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ +│ │ │ Signer │ │ Reporter │ │ Prefetch │ │ │ +│ │ │ 签名模块 │ │ 监控上报 │ │ 预解析 │ │ │ +│ │ │ -HMAC-SHA256│ │ -成功率 │ │ -冷启动优化 │ │ │ +│ │ │ -防重放 │ │ -耗时统计 │ │ -批量解析 │ │ │ +│ │ └─────────────┘ │ -缓存命中率│ └─────────────┘ │ │ +│ │ └─────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + │ HTTPS + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ GoEdge 平台 │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Edge-DNS │ │ Edge-Node │ │ Edge-Admin │ │ +│ │ │ │ │ │ │ │ +│ │ /httpdns/ │ │ WAF 校验 │ │ App 管理 │ │ +│ │ resolve │ │ 指纹验证 │ │ AppID/Secret│ │ +│ │ │ │ │ │ │ │ +│ │ 智能调度 │ │ 流量转发 │ │ SDK统计接收 │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ │ │ │ │ +│ └───────────────────┴───────────────────┘ │ +│ │ │ +│ ┌──────────────┐ │ +│ │ Edge-API │ │ +│ │ 数据服务 │ │ +│ └──────────────┘ │ +│ │ │ +│ ┌──────────────┐ │ +│ │ MySQL │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + │ HTTPS (IP 直连) + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ 源站服务器 │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### 2.2 请求流程（带缓存与降级） + +```mermaid +sequenceDiagram + participant App as 移动 App + participant Cache as SDK缓存 + participant Resolver as SDK解析器 + participant DNS as Edge-DNS + participant SysDNS as 系统DNS + participant Node as Edge-Node + participant API as Edge-API + + Note over App,API: 阶段一：DNS 解析（含缓存与降级） + + App->>Resolver: resolve("api.example.com") + + Resolver->>Cache: 查询缓存 + alt 缓存有效 + Cache-->>Resolver: 返回 IP (命中) + Resolver-->>App: ["1.2.3.4"] + else 缓存软过期 + Cache-->>Resolver: 返回 IP + 标记需刷新 + Resolver-->>App: ["1.2.3.4"] (先返回) + Resolver->>DNS: 后台异步刷新 + DNS-->>Resolver: 新 IP + Resolver->>Cache: 更新缓存 + else 缓存完全过期或不存在 + Resolver->>DNS: GET /httpdns/resolve + alt HTTPDNS 正常 + DNS-->>Resolver: {"ips": ["1.2.3.4"], "ttl": 600} + Resolver->>Cache: 写入缓存 + Resolver-->>App: ["1.2.3.4"] + else HTTPDNS 超时/失败 + Resolver->>SysDNS: 降级到系统 DNS + SysDNS-->>Resolver: 1.2.3.4 + Resolver-->>App: ["1.2.3.4"] (降级) + end + end + + Note over App,API: 阶段二：业务请求（带签名） + App->>Node: HTTPS://1.2.3.4/v1/user + 签名Header + Node->>API: 查询 AppSecret + API-->>Node: AppSecret + Node->>Node: HMAC-SHA256 验证 + alt 验证失败 + Node-->>App: 403 Forbidden + else 验证成功 + Node-->>App: 200 OK + 响应数据 + end + + Note over App,API: 阶段三：监控上报（异步） + Resolver-->>API: 定时上报统计数据 +``` + +### 2.3 网络切换处理流程 + +```mermaid +flowchart LR + A[WiFi] -->|切换| B[4G/5G] + B --> C{NetworkMonitor检测} + C --> D[清空所有缓存] + D --> E[下次请求重新解析] + E --> F[获取新网络最优IP] +``` + +--- + +## 三、模块详细设计 + +### 3.1 Edge-DNS: HTTPDNS 解析接口 + +#### 3.1.1 接口定义 + +| 项目 | 说明 | +|------|------| +| **Endpoint** | `GET /httpdns/resolve` | +| **协议** | HTTPS (443) | +| **认证** | 无（解析接口公开，业务请求才需要签名） | + +#### 3.1.2 请求参数 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `host` | string | 是 | 待解析的域名 | +| `type` | string | 否 | 记录类型，默认 `A,AAAA`（同时返回） | +| `ip` | string | 否 | 客户端 IP（用于调试/代理场景） | + +#### 3.1.3 响应格式 + +```json +{ + "status": "ok", + "dns_server_time": 1700000000, + "client_ip": "114.114.114.114", + "data": [ + { + "host": "api.example.com", + "type": "A", + "ips": ["1.1.1.1", "1.1.1.2"], + "ips_v6": ["240e:xxx::1"], + "ttl": 600 + } + ] +} +``` + +#### 3.1.4 代码实现 + +**文件**: `EdgeDNS/internal/nodes/httpdns.go` (新建) + +```go +package nodes + +import ( + "encoding/json" + "net" + "net/http" + "strings" + "time" +) + +// HTTPDNSResponse HTTPDNS 响应结构 +type HTTPDNSResponse struct { + Status string `json:"status"` + DNSServerTime int64 `json:"dns_server_time"` + ClientIP string `json:"client_ip"` + Data []HTTPDNSRecord `json:"data"` + Error string `json:"error,omitempty"` +} + +// HTTPDNSRecord 单条解析记录 +type HTTPDNSRecord struct { + Host string `json:"host"` + Type string `json:"type"` + IPs []string `json:"ips"` + IPsV6 []string `json:"ips_v6"` + TTL int `json:"ttl"` +} + +// handleHTTPDNSResolve 处理 HTTPDNS 解析请求 +func (this *Server) handleHTTPDNSResolve(writer http.ResponseWriter, req *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writer.Header().Set("Access-Control-Allow-Origin", "*") + + // 1. 解析参数 + query := req.URL.Query() + host := strings.TrimSpace(query.Get("host")) + if host == "" { + this.writeHTTPDNSError(writer, "missing 'host' parameter") + return + } + + // 2. 获取客户端 IP + clientIP := query.Get("ip") + if clientIP == "" { + clientIP = this.extractClientIP(req) + } + + // 3. 查询 A 记录 + ipsV4 := this.resolveRecords(host, "A", clientIP) + + // 4. 查询 AAAA 记录 + ipsV6 := this.resolveRecords(host, "AAAA", clientIP) + + // 5. 获取 TTL + ttl := this.getRecordTTL(host, clientIP) + if ttl == 0 { + ttl = 600 + } + + // 6. 构造响应 + resp := HTTPDNSResponse{ + Status: "ok", + DNSServerTime: time.Now().Unix(), + ClientIP: clientIP, + Data: []HTTPDNSRecord{{ + Host: host, + Type: "A", + IPs: ipsV4, + IPsV6: ipsV6, + TTL: ttl, + }}, + } + + json.NewEncoder(writer).Encode(resp) +} + +// extractClientIP 从请求中提取客户端真实 IP +func (this *Server) extractClientIP(req *http.Request) string { + xff := req.Header.Get("X-Forwarded-For") + if xff != "" { + parts := strings.Split(xff, ",") + return strings.TrimSpace(parts[0]) + } + xri := req.Header.Get("X-Real-IP") + if xri != "" { + return xri + } + host, _, _ := net.SplitHostPort(req.RemoteAddr) + return host +} + +// resolveRecords 解析指定类型的记录 +func (this *Server) resolveRecords(host, recordType, clientIP string) []string { + var result []string + if !strings.HasSuffix(host, ".") { + host += "." + } + domain, recordName := sharedDomainManager.SplitDomain(host) + if domain == nil { + return result + } + routeCodes := sharedRouteManager.FindRouteCodes(clientIP, domain.UserId) + records, _ := sharedRecordManager.FindRecords(domain.Id, routeCodes, recordName, recordType, false) + for _, record := range records { + if record.Value != "" { + result = append(result, record.Value) + } + } + return result +} + +// getRecordTTL 获取记录 TTL +func (this *Server) getRecordTTL(host, clientIP string) int { + if !strings.HasSuffix(host, ".") { + host += "." + } + domain, recordName := sharedDomainManager.SplitDomain(host) + if domain == nil { + return 0 + } + routeCodes := sharedRouteManager.FindRouteCodes(clientIP, domain.UserId) + records, _ := sharedRecordManager.FindRecords(domain.Id, routeCodes, recordName, "A", false) + if len(records) > 0 { + return int(records[0].Ttl) + } + return 0 +} + +// writeHTTPDNSError 写入错误响应 +func (this *Server) writeHTTPDNSError(writer http.ResponseWriter, errMsg string) { + writer.WriteHeader(http.StatusBadRequest) + resp := HTTPDNSResponse{ + Status: "error", + DNSServerTime: time.Now().Unix(), + Error: errMsg, + } + json.NewEncoder(writer).Encode(resp) +} +``` + +**修改**: `EdgeDNS/internal/nodes/server.go` 第735行附近 + +```go +func (this *Server) handleHTTP(writer http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/dns-query" { + this.handleHTTPDNSMessage(writer, req) + return + } + // 新增 HTTPDNS JSON API + if req.URL.Path == "/httpdns/resolve" { + this.handleHTTPDNSResolve(writer, req) + return + } + if req.URL.Path == "/resolve" { + this.handleHTTPJSONAPI(writer, req) + return + } + writer.WriteHeader(http.StatusNotFound) +} +``` + +--- + +### 3.2 Edge-Node: WAF 指纹校验 + +#### 3.2.1 校验逻辑 + +| 步骤 | 说明 | +|------|------| +| 1 | 提取 `X-GE-AppID`, `X-GE-Timestamp`, `X-GE-Token` | +| 2 | 根据 AppID 查询 AppSecret | +| 3 | 校验时间戳（±300 秒内有效） | +| 4 | 计算 HMAC-SHA256 并比对 | + +#### 3.2.2 代码实现 + +**文件**: `EdgeNode/internal/waf/checkpoints/checkpoint_httpdns_fingerprint.go` (新建) + +```go +package checkpoints + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "math" + "net/http" + "strconv" + "time" + + "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeNode/internal/rpc" +) + +type CheckpointHTTPDNSFingerprint struct { + Checkpoint +} + +func (this *CheckpointHTTPDNSFingerprint) RequestValue( + req CheckpointRequest, param string, options map[string]string, ruleId int64, +) (value any, hasRequestBody bool, sysErr error, userErr error) { + httpReq, ok := req.WAFRaw().(*http.Request) + if !ok { + return "INVALID_REQUEST", false, nil, nil + } + + appID := httpReq.Header.Get("X-GE-AppID") + token := httpReq.Header.Get("X-GE-Token") + tsStr := httpReq.Header.Get("X-GE-Timestamp") + + if appID == "" && token == "" && tsStr == "" { + return "", false, nil, nil + } + if appID == "" { + return "MISSING_APPID", false, nil, nil + } + if token == "" { + return "MISSING_TOKEN", false, nil, nil + } + if tsStr == "" { + return "MISSING_TIMESTAMP", false, nil, nil + } + + appSecret, err := this.getAppSecret(appID) + if err != nil || appSecret == "" { + return "INVALID_APPID", false, nil, nil + } + + ts, _ := strconv.ParseInt(tsStr, 10, 64) + if math.Abs(float64(time.Now().Unix()-ts)) > 300 { + return "TIMESTAMP_EXPIRED", false, nil, nil + } + + mac := hmac.New(sha256.New, []byte(appSecret)) + mac.Write([]byte(appID + tsStr + httpReq.URL.Path)) + expected := hex.EncodeToString(mac.Sum(nil)) + + if token != expected { + return "SIGNATURE_MISMATCH", false, nil, nil + } + return "OK", false, nil, nil +} + +func (this *CheckpointHTTPDNSFingerprint) ResponseValue( + req CheckpointRequest, param string, options map[string]string, ruleId int64, +) (value any, hasRequestBody bool, sysErr error, userErr error) { + return "", false, nil, nil +} + +func (this *CheckpointHTTPDNSFingerprint) getAppSecret(appID string) (string, error) { + client, err := rpc.SharedRPC() + if err != nil { + return "", err + } + resp, err := client.HTTPDNSAppRPC.FindHTTPDNSAppSecret( + client.Context(), &pb.FindHTTPDNSAppSecretRequest{AppId: appID}, + ) + if err != nil { + return "", err + } + return resp.AppSecret, nil +} +``` + +--- + +### 3.3 Edge-API: App 管理服务 + +#### 3.3.1 数据库表 + +```sql +CREATE TABLE IF NOT EXISTS edgeHTTPDNSApps ( + id BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT, + appId VARCHAR(64) NOT NULL UNIQUE COMMENT 'App标识', + appSecret VARCHAR(128) NOT NULL COMMENT 'App密钥', + name VARCHAR(255) NOT NULL DEFAULT '' COMMENT '应用名称', + description TEXT COMMENT '描述', + userId BIGINT UNSIGNED DEFAULT 0 COMMENT '关联用户ID', + isOn TINYINT(1) DEFAULT 1 COMMENT '是否启用', + createdAt INT UNSIGNED DEFAULT 0, + state TINYINT(1) DEFAULT 1, + UNIQUE KEY uk_appId (appId), + KEY idx_userId (userId) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +``` + +#### 3.3.2 gRPC Proto + +**文件**: `EdgeCommon/pkg/rpc/protos/service_httpdns_app.proto` (新建) + +```protobuf +syntax = "proto3"; +option go_package = "./pb"; +package pb; + +service HTTPDNSAppService { + rpc createHTTPDNSApp(CreateHTTPDNSAppRequest) returns (CreateHTTPDNSAppResponse); + rpc findHTTPDNSAppSecret(FindHTTPDNSAppSecretRequest) returns (FindHTTPDNSAppSecretResponse); + rpc listHTTPDNSApps(ListHTTPDNSAppsRequest) returns (ListHTTPDNSAppsResponse); + rpc deleteHTTPDNSApp(DeleteHTTPDNSAppRequest) returns (RPCSuccess); +} + +message CreateHTTPDNSAppRequest { + string name = 1; + string description = 2; + int64 userId = 3; +} +message CreateHTTPDNSAppResponse { + int64 httpdnsAppId = 1; + string appId = 2; + string appSecret = 3; +} +message FindHTTPDNSAppSecretRequest { + string appId = 1; +} +message FindHTTPDNSAppSecretResponse { + string appSecret = 1; +} +message ListHTTPDNSAppsRequest { + int64 userId = 1; + int64 offset = 2; + int64 size = 3; +} +message ListHTTPDNSAppsResponse { + repeated HTTPDNSApp httpdnsApps = 1; +} +message DeleteHTTPDNSAppRequest { + int64 httpdnsAppId = 1; +} +message HTTPDNSApp { + int64 id = 1; + string appId = 2; + string appSecret = 3; + string name = 4; + int64 userId = 5; + bool isOn = 6; + int64 createdAt = 7; +} +``` + +--- + +### 3.4 SDK 完整设计 + +#### 3.4.1 SDK 架构概览 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ HTTPDNS SDK │ +├─────────────────────────────────────────────────────────────┤ +│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ +│ │ 解析引擎 │ │ 缓存管理 │ │ 网络感知 │ │ +│ │ Resolver │ │ CacheManager │ │ NetworkMonitor│ │ +│ └───────────────┘ └───────────────┘ └───────────────┘ │ +│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ +│ │ 容错降级 │ │ 签名模块 │ │ 监控上报 │ │ +│ │ Failover │ │ Signer │ │ Reporter │ │ +│ └───────────────┘ └───────────────┘ └───────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +#### 3.4.2 缓存管理模块 (CacheManager) + +**设计原则**： +- 两级缓存：内存 + 持久化 +- 软过期策略：过期后先返回旧数据，后台异步更新 +- LRU 淘汰：防止内存溢出 + +**Android 实现**： + +```kotlin +class HTTPDNSCacheManager(private val context: Context) { + // 内存缓存 (LRU) + private val memoryCache = object : LinkedHashMap(100, 0.75f, true) { + override fun removeEldestEntry(eldest: Map.Entry): Boolean { + return size > MAX_CACHE_SIZE + } + } + + // 持久化缓存 (MMKV) + private val mmkv = MMKV.mmkvWithID("httpdns_cache", MMKV.MULTI_PROCESS_MODE) + + companion object { + const val MAX_CACHE_SIZE = 100 + const val SOFT_EXPIRE_SECONDS = 60 // 软过期：TTL 到期后仍可用 60s + } + + data class CacheEntry( + val ips: List, + val ipsV6: List, + val expireAt: Long, // 硬过期时间 + val softExpireAt: Long, // 软过期时间 = expireAt + SOFT_EXPIRE_SECONDS + val createAt: Long = System.currentTimeMillis() + ) { + fun isExpired(): Boolean = System.currentTimeMillis() > expireAt + fun isSoftExpired(): Boolean = System.currentTimeMillis() > softExpireAt + fun toJson(): String = Gson().toJson(this) + } + + /** + * 获取缓存（支持软过期） + * @return Pair<缓存结果, 是否需要后台刷新> + */ + @Synchronized + fun get(host: String): Pair { + // 1. 先查内存 + memoryCache[host]?.let { entry -> + return when { + !entry.isExpired() -> Pair(entry, false) // 未过期 + !entry.isSoftExpired() -> Pair(entry, true) // 软过期，需刷新 + else -> Pair(null, true) // 完全过期 + } + } + + // 2. 查持久化 + val json = mmkv.decodeString(host) ?: return Pair(null, true) + val entry = Gson().fromJson(json, CacheEntry::class.java) + + // 回填内存 + memoryCache[host] = entry + + return when { + !entry.isExpired() -> Pair(entry, false) + !entry.isSoftExpired() -> Pair(entry, true) + else -> Pair(null, true) + } + } + + /** + * 写入缓存 + */ + @Synchronized + fun put(host: String, ips: List, ipsV6: List, ttl: Int) { + val now = System.currentTimeMillis() + val entry = CacheEntry( + ips = ips, + ipsV6 = ipsV6, + expireAt = now + ttl * 1000L, + softExpireAt = now + (ttl + SOFT_EXPIRE_SECONDS) * 1000L + ) + memoryCache[host] = entry + mmkv.encode(host, entry.toJson()) + } + + /** + * 清空所有缓存（网络切换时调用） + */ + @Synchronized + fun clear() { + memoryCache.clear() + mmkv.clearAll() + } +} +``` + +**iOS 实现**： + +```swift +class HTTPDNSCacheManager { + static let shared = HTTPDNSCacheManager() + + private var memoryCache: [String: CacheEntry] = [:] + private let defaults = UserDefaults(suiteName: "com.goedge.httpdns")! + private let queue = DispatchQueue(label: "httpdns.cache", attributes: .concurrent) + + private let maxCacheSize = 100 + private let softExpireSeconds: TimeInterval = 60 + + struct CacheEntry: Codable { + let ips: [String] + let ipsV6: [String] + let expireAt: Date + let softExpireAt: Date + let createAt: Date + + func isExpired() -> Bool { Date() > expireAt } + func isSoftExpired() -> Bool { Date() > softExpireAt } + } + + func get(host: String) -> (entry: CacheEntry?, needRefresh: Bool) { + return queue.sync { + // 查内存 + if let entry = memoryCache[host] { + if !entry.isExpired() { return (entry, false) } + if !entry.isSoftExpired() { return (entry, true) } + } + + // 查持久化 + guard let data = defaults.data(forKey: host), + let entry = try? JSONDecoder().decode(CacheEntry.self, from: data) else { + return (nil, true) + } + + // 回填内存 + memoryCache[host] = entry + + if !entry.isExpired() { return (entry, false) } + if !entry.isSoftExpired() { return (entry, true) } + return (nil, true) + } + } + + func put(host: String, ips: [String], ipsV6: [String], ttl: Int) { + queue.async(flags: .barrier) { + let entry = CacheEntry( + ips: ips, + ipsV6: ipsV6, + expireAt: Date().addingTimeInterval(TimeInterval(ttl)), + softExpireAt: Date().addingTimeInterval(TimeInterval(ttl) + self.softExpireSeconds), + createAt: Date() + ) + self.memoryCache[host] = entry + + // LRU 淘汰 + if self.memoryCache.count > self.maxCacheSize { + let oldest = self.memoryCache.min { $0.value.createAt < $1.value.createAt } + if let key = oldest?.key { self.memoryCache.removeValue(forKey: key) } + } + + // 持久化 + if let data = try? JSONEncoder().encode(entry) { + self.defaults.set(data, forKey: host) + } + } + } + + func clear() { + queue.async(flags: .barrier) { + self.memoryCache.removeAll() + // 清理持久化 + for key in self.defaults.dictionaryRepresentation().keys { + self.defaults.removeObject(forKey: key) + } + } + } +} +``` + +#### 3.4.3 网络感知模块 (NetworkMonitor) + +**功能**： +- 监听网络切换（WiFi ↔ 4G/5G） +- 网络切换时清空缓存 +- 检测 IPv6 可用性 + +**Android 实现**： + +```kotlin +class NetworkMonitor(private val context: Context) { + private val connectivityManager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + private var lastNetworkType: String? = null + var onNetworkChanged: (() -> Unit)? = null + + private val networkCallback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(network: Network) { + checkNetworkChange() + } + + override fun onLost(network: Network) { + checkNetworkChange() + } + + override fun onCapabilitiesChanged(network: Network, caps: NetworkCapabilities) { + checkNetworkChange() + } + } + + fun start() { + val request = NetworkRequest.Builder() + .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + .build() + connectivityManager.registerNetworkCallback(request, networkCallback) + lastNetworkType = getCurrentNetworkType() + } + + fun stop() { + connectivityManager.unregisterNetworkCallback(networkCallback) + } + + private fun checkNetworkChange() { + val currentType = getCurrentNetworkType() + if (currentType != lastNetworkType) { + Log.d("HTTPDNS", "Network changed: $lastNetworkType -> $currentType") + lastNetworkType = currentType + onNetworkChanged?.invoke() + } + } + + private fun getCurrentNetworkType(): String { + val network = connectivityManager.activeNetwork ?: return "NONE" + val caps = connectivityManager.getNetworkCapabilities(network) ?: return "UNKNOWN" + return when { + caps.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) -> "WIFI" + caps.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR) -> "CELLULAR" + else -> "OTHER" + } + } + + /** + * 检测当前网络是否支持 IPv6 + */ + fun isIPv6Supported(): Boolean { + return try { + val addresses = NetworkInterface.getNetworkInterfaces().toList() + .flatMap { it.inetAddresses.toList() } + addresses.any { it is Inet6Address && !it.isLoopbackAddress && !it.isLinkLocalAddress } + } catch (e: Exception) { + false + } + } +} +``` + +**iOS 实现**： + +```swift +import Network + +class NetworkMonitor { + static let shared = NetworkMonitor() + + private let monitor = NWPathMonitor() + private let queue = DispatchQueue(label: "httpdns.network") + private var lastInterfaceType: NWInterface.InterfaceType? + + var onNetworkChanged: (() -> Void)? + + func start() { + monitor.pathUpdateHandler = { [weak self] path in + let currentType = path.availableInterfaces.first?.type + if currentType != self?.lastInterfaceType { + print("HTTPDNS: Network changed \(self?.lastInterfaceType?.description ?? "nil") -> \(currentType?.description ?? "nil")") + self?.lastInterfaceType = currentType + DispatchQueue.main.async { + self?.onNetworkChanged?() + } + } + } + monitor.start(queue: queue) + } + + func stop() { + monitor.cancel() + } + + func isIPv6Supported() -> Bool { + let path = monitor.currentPath + return path.supportsIPv6 + } +} +``` + +#### 3.4.4 容错降级模块 (Failover) + +**策略**： +1. 多节点轮询：主节点失败自动切备用 +2. 超时降级：HTTPDNS 超时自动降级到系统 DNS +3. 黑名单机制：特定域名强制走系统 DNS + +**Android 实现**： + +```kotlin +class HTTPDNSResolver( + private val serverUrls: List, // 多个服务节点 + private val cacheManager: HTTPDNSCacheManager, + private val timeout: Long = 3000L +) { + private val blacklist = setOf("localhost", "*.local", "*.internal") + private var currentServerIndex = 0 + + suspend fun resolve(host: String): ResolveResult { + // 1. 黑名单检查 + if (isBlacklisted(host)) { + return ResolveResult(systemResolve(host), ResolveSource.SYSTEM_DNS) + } + + // 2. 查缓存 + val (cached, needRefresh) = cacheManager.get(host) + if (cached != null) { + if (needRefresh) { + // 后台异步刷新 + CoroutineScope(Dispatchers.IO).launch { fetchFromServer(host) } + } + return ResolveResult(cached.ips, ResolveSource.CACHE) + } + + // 3. 请求服务器（带重试） + return try { + withTimeout(timeout) { + val ips = fetchFromServerWithRetry(host) + ResolveResult(ips, ResolveSource.HTTPDNS) + } + } catch (e: TimeoutCancellationException) { + // 4. 降级到系统 DNS + Log.w("HTTPDNS", "Timeout, fallback to system DNS") + ResolveResult(systemResolve(host), ResolveSource.SYSTEM_DNS) + } + } + + private suspend fun fetchFromServerWithRetry(host: String): List { + var lastError: Exception? = null + repeat(serverUrls.size) { attempt -> + try { + return fetchFromServer(host) + } catch (e: Exception) { + lastError = e + currentServerIndex = (currentServerIndex + 1) % serverUrls.size + Log.w("HTTPDNS", "Server ${serverUrls[currentServerIndex]} failed, trying next") + } + } + throw lastError ?: Exception("All servers failed") + } + + private suspend fun fetchFromServer(host: String): List { + val url = "${serverUrls[currentServerIndex]}/httpdns/resolve?host=$host" + val response = httpClient.get(url) + val result = parseResponse(response.body) + cacheManager.put(host, result.ips, result.ipsV6, result.ttl) + return result.ips + } + + private fun systemResolve(host: String): List { + return try { + InetAddress.getAllByName(host).map { it.hostAddress } + } catch (e: Exception) { + emptyList() + } + } + + private fun isBlacklisted(host: String): Boolean { + return blacklist.any { pattern -> + if (pattern.startsWith("*")) { + host.endsWith(pattern.substring(1)) + } else { + host == pattern + } + } + } + + enum class ResolveSource { CACHE, HTTPDNS, SYSTEM_DNS } + data class ResolveResult(val ips: List, val source: ResolveSource) +} +``` + +#### 3.4.5 监控上报模块 (Reporter) + +**上报指标**： +- 解析成功率 +- 解析耗时 +- 缓存命中率 +- 降级次数 + +**数据结构**： + +```kotlin +data class HTTPDNSStats( + val host: String, + val resolveCount: Int, + val successCount: Int, + val cacheHitCount: Int, + val fallbackCount: Int, + val avgLatencyMs: Long, + val timestamp: Long +) +``` + +**Android 实现**： + +```kotlin +class HTTPDNSReporter(private val reportUrl: String) { + private val stats = mutableMapOf() + private val reportInterval = 60_000L // 60秒上报一次 + + data class MutableStats( + var resolveCount: Int = 0, + var successCount: Int = 0, + var cacheHitCount: Int = 0, + var fallbackCount: Int = 0, + var totalLatencyMs: Long = 0 + ) + + fun recordResolve(host: String, source: ResolveSource, latencyMs: Long, success: Boolean) { + synchronized(stats) { + val s = stats.getOrPut(host) { MutableStats() } + s.resolveCount++ + if (success) s.successCount++ + s.totalLatencyMs += latencyMs + when (source) { + ResolveSource.CACHE -> s.cacheHitCount++ + ResolveSource.SYSTEM_DNS -> s.fallbackCount++ + else -> {} + } + } + } + + fun startPeriodicReport() { + CoroutineScope(Dispatchers.IO).launch { + while (true) { + delay(reportInterval) + report() + } + } + } + + private suspend fun report() { + val snapshot = synchronized(stats) { + val copy = stats.toMap() + stats.clear() + copy + } + + if (snapshot.isEmpty()) return + + val reports = snapshot.map { (host, s) -> + HTTPDNSStats( + host = host, + resolveCount = s.resolveCount, + successCount = s.successCount, + cacheHitCount = s.cacheHitCount, + fallbackCount = s.fallbackCount, + avgLatencyMs = if (s.resolveCount > 0) s.totalLatencyMs / s.resolveCount else 0, + timestamp = System.currentTimeMillis() + ) + } + + try { + httpClient.post(reportUrl) { + contentType(ContentType.Application.Json) + setBody(reports) + } + } catch (e: Exception) { + Log.e("HTTPDNS", "Report failed: ${e.message}") + } + } +} +``` + +#### 3.4.6 完整 SDK 集成示例 + +**Android 初始化**： + +```kotlin +class HTTPDNSManager private constructor(context: Context) { + private val cacheManager = HTTPDNSCacheManager(context) + private val networkMonitor = NetworkMonitor(context) + private val resolver = HTTPDNSResolver( + serverUrls = listOf( + "https://httpdns1.goedge.cn", + "https://httpdns2.goedge.cn" + ), + cacheManager = cacheManager + ) + private val signer = HTTPDNSSigner(appId = "ge_xxx", appSecret = "xxx") + private val reporter = HTTPDNSReporter("https://api.goedge.cn/httpdns/report") + + companion object { + @Volatile + private var instance: HTTPDNSManager? = null + + fun init(context: Context): HTTPDNSManager { + return instance ?: synchronized(this) { + instance ?: HTTPDNSManager(context.applicationContext).also { instance = it } + } + } + + fun get(): HTTPDNSManager = instance ?: throw IllegalStateException("Must call init first") + } + + init { + // 监听网络变化 + networkMonitor.onNetworkChanged = { + Log.d("HTTPDNS", "Network changed, clearing cache") + cacheManager.clear() + } + networkMonitor.start() + + // 启动监控上报 + reporter.startPeriodicReport() + } + + suspend fun resolve(host: String): List { + val startTime = System.currentTimeMillis() + val result = resolver.resolve(host) + val latency = System.currentTimeMillis() - startTime + reporter.recordResolve(host, result.source, latency, result.ips.isNotEmpty()) + return result.ips + } + + fun signRequest(request: Request): Request = signer.sign(request) + + /** + * 预解析核心域名（App 启动时调用） + */ + suspend fun prefetch(hosts: List) { + hosts.forEach { host -> + launch { resolve(host) } + } + } +} +``` + +**使用示例**： + +```kotlin +// Application.onCreate +HTTPDNSManager.init(this) + +// 预解析 +lifecycleScope.launch { + HTTPDNSManager.get().prefetch(listOf( + "api.example.com", + "cdn.example.com", + "img.example.com" + )) +} + +// 业务请求 +lifecycleScope.launch { + val ips = HTTPDNSManager.get().resolve("api.example.com") + if (ips.isNotEmpty()) { + val request = Request.Builder() + .url("https://${ips[0]}/v1/user") + .header("Host", "api.example.com") + .build() + val signedRequest = HTTPDNSManager.get().signRequest(request) + // 发起请求... + } +} +``` + +--- + +## 四、实施计划 + +### 4.1 任务清单 + +| 阶段 | 任务 | 工时 | +|------|------|------| +| **Phase 1** | 数据库 + DAO + gRPC | 3天 | +| **Phase 2** | Edge-DNS 接口 | 1天 | +| **Phase 3** | Edge-Node WAF | 1天 | +| **Phase 4** | Edge-Admin UI | 2天 | +| **Phase 5** | SDK 核心 (解析+签名) | 1天 | +| **Phase 6** | SDK 缓存模块 (内存+持久化+LRU+软过期) | 1天 | +| **Phase 7** | SDK 网络感知 (切换监听+IPv6检测) | 0.5天 | +| **Phase 8** | SDK 容错降级 (多节点重试+系统DNS降级) | 0.5天 | +| **Phase 9** | SDK 监控上报 | 0.5天 | +| **Phase 10** | SDK 集成测试 + 文档 | 0.5天 | +| **总计** | | **11天** | + +### 4.2 文件变更清单 + +| 操作 | 文件路径 | +|------|----------| +| 新建 | `EdgeDNS/internal/nodes/httpdns.go` | +| 修改 | `EdgeDNS/internal/nodes/server.go` | +| 新建 | `EdgeNode/internal/waf/checkpoints/checkpoint_httpdns_fingerprint.go` | +| 修改 | `EdgeNode/internal/waf/checkpoints/init.go` | +| 新建 | `EdgeAPI/internal/db/models/httpdns_app_dao.go` | +| 新建 | `EdgeAPI/internal/rpc/services/service_httpdns_app.go` | +| 新建 | `EdgeCommon/pkg/rpc/protos/service_httpdns_app.proto` | +| 新建 | `EdgeAdmin/internal/web/actions/httpdns/*.go` | +| 新建 | `EdgeAdmin/web/views/httpdns/apps/*.html` | + +--- + +## 五、测试验证 + +### 5.1 服务端测试 + +```bash +# 1. 测试 HTTPDNS 解析 +curl "https://httpdns.example.com/httpdns/resolve?host=api.example.com" + +# 2. 测试无签名访问业务（应被拦截） +curl "https://1.2.3.4/v1/user" -H "Host: api.example.com" + +# 3. 测试带签名访问 +curl "https://1.2.3.4/v1/user" \ + -H "Host: api.example.com" \ + -H "X-GE-AppID: ge_abc123" \ + -H "X-GE-Timestamp: 1700000000" \ + -H "X-GE-Token: " +``` + +### 5.2 SDK 测试矩阵 + +| 模块 | 测试场景 | 预期结果 | +|------|----------|----------| +| **缓存** | 首次解析 | 请求服务器，写入缓存 | +| **缓存** | 缓存有效期内再次解析 | 直接返回缓存，不发请求 | +| **缓存** | 缓存软过期 | 返回旧数据，后台异步刷新 | +| **缓存** | 缓存完全过期 | 重新请求服务器 | +| **缓存** | 缓存超过100条 | LRU 淘汰最旧条目 | +| **网络感知** | WiFi → 4G 切换 | 清空所有缓存 | +| **网络感知** | 4G → WiFi 切换 | 清空所有缓存 | +| **网络感知** | IPv6 检测（双栈网络） | 返回 true | +| **容错** | 主节点超时 | 自动切换备用节点 | +| **容错** | 所有节点失败 | 降级到系统 DNS | +| **容错** | 黑名单域名 | 直接走系统 DNS | +| **监控** | 60 秒内多次解析 | 统计聚合后上报 | +| **监控** | 上报失败 | 静默失败，不影响解析 | + +### 5.3 Android 单元测试示例 + +```kotlin +@Test +fun `cache hit returns immediately without network request`() = runTest { + // Given + cacheManager.put("api.example.com", listOf("1.2.3.4"), emptyList(), 600) + + // When + val result = resolver.resolve("api.example.com") + + // Then + assertEquals(listOf("1.2.3.4"), result.ips) + assertEquals(ResolveSource.CACHE, result.source) + verify(httpClient, never()).get(any()) +} + +@Test +fun `soft expired cache triggers background refresh`() = runTest { + // Given: cache with TTL=1s, soft expire = TTL+60s + cacheManager.put("api.example.com", listOf("1.2.3.4"), emptyList(), 1) + advanceTimeBy(2000) // TTL expired but within soft expire + + // When + val result = resolver.resolve("api.example.com") + + // Then + assertEquals(listOf("1.2.3.4"), result.ips) // Returns stale data + advanceUntilIdle() + verify(httpClient).get(any()) // Background refresh triggered +} + +@Test +fun `network change clears cache`() = runTest { + // Given + cacheManager.put("api.example.com", listOf("1.2.3.4"), emptyList(), 600) + + // When + networkMonitor.simulateNetworkChange() + + // Then + val (cached, _) = cacheManager.get("api.example.com") + assertNull(cached) +} + +@Test +fun `fallback to system DNS on timeout`() = runTest { + // Given + whenever(httpClient.get(any())).thenThrow(TimeoutException()) + + // When + val result = resolver.resolve("api.example.com") + + // Then + assertEquals(ResolveSource.SYSTEM_DNS, result.source) +} +``` + +--- + +## 六、上线清单 + +### 6.1 服务端部署 + +- [ ] 数据库迁移（edgeHTTPDNSApps 表） +- [ ] Edge-API 部署 +- [ ] Edge-DNS 部署 +- [ ] Edge-Node 部署（含 WAF 指纹校验） +- [ ] Edge-Admin 部署（App 管理 UI） +- [ ] 创建测试 App（获取 AppID/Secret） + +### 6.2 SDK 发布 + +- [ ] Android SDK 单元测试通过 +- [ ] iOS SDK 单元测试通过 +- [ ] Android SDK 集成测试（真机） +- [ ] iOS SDK 集成测试（真机） +- [ ] SDK 打包发布（Maven/CocoaPods） +- [ ] SDK 接入文档发布 + +### 6.3 SDK 功能验收 + +- [ ] 缓存命中验证 +- [ ] 软过期刷新验证 +- [ ] LRU 淘汰验证 +- [ ] 网络切换清缓存验证 +- [ ] 多节点切换验证 +- [ ] 系统 DNS 降级验证 +- [ ] 监控数据上报验证 +- [ ] 预解析功能验证 + diff --git a/HttpDNS SDK 功能设计规范.pdf b/HttpDNS SDK 功能设计规范.pdf new file mode 100644 index 0000000..99aa321 Binary files /dev/null and b/HttpDNS SDK 功能设计规范.pdf differ diff --git a/deploy/fluent-bit/README.md b/deploy/fluent-bit/README.md index 89adcfe..47e64a5 100644 --- a/deploy/fluent-bit/README.md +++ b/deploy/fluent-bit/README.md @@ -6,18 +6,20 @@ ## Fluent Bit 跑在哪台机器上？ -**Fluent Bit 应部署在每台 EdgeNode 机器上**（与 edge-node 同机），不要部署在 EdgeAPI 机器上。 +**Fluent Bit 应部署在写日志文件的节点机器上**（EdgeNode / EdgeDNS 同机），不要部署在 EdgeAPI 机器上。 -- 日志文件（`/var/log/edge/edge-node/*.log`）是 **EdgeNode** 在本机写的，只有 EdgeNode 所在机器才有这些文件。 -- Fluent Bit 使用 **tail** 插件读取本机路径，因此必须运行在 **有这些日志文件的机器** 上，即每台 EdgeNode。 -- EdgeAPI 机器上没有边缘节点日志，只负责查询 ClickHouse/MySQL，因此不需要在 EdgeAPI 上跑 Fluent Bit。 -- **多台 EdgeNode 时**：每台 EdgeNode 各跑一份 Fluent Bit，各自采集本机日志并上报到同一 ClickHouse。 +- HTTP 日志文件默认在 `/var/log/edge/edge-node/*.log`，由 **EdgeNode** 本机写入；若配置了公用访问日志策略的文件 `path`，节点会优先复用该 `path` 所在目录。 +- DNS 日志文件默认在 `/var/log/edge/edge-dns/*.log`，由 **EdgeDNS** 本机写入；若配置了公用访问日志策略的文件 `path`，节点会优先复用该 `path` 所在目录。 +- Fluent Bit 使用 **tail** 读取本机路径，因此必须运行在这些日志文件所在机器上。 +- EdgeAPI 机器主要负责查询 ClickHouse/MySQL，不需要承担日志采集。 +- 多机部署时，每台写日志节点都跑一份 Fluent Bit，上报到同一 ClickHouse 集群。 --- ## 一、前置条件 -- **边缘节点（EdgeNode）** 已开启本地日志落盘，日志目录默认 `/var/log/edge/edge-node`，会生成 `access.log`、`waf.log`、`error.log`（JSON Lines）。由环境变量 `EDGE_LOG_DIR` 控制路径。 +- **边缘节点（EdgeNode）** 已开启本地日志落盘，目录优先取“公用访问日志策略”的文件 `path`（取目录），为空时回退 `EDGE_LOG_DIR`，再回退默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`（JSON Lines）。 +- **DNS 节点（EdgeDNS）** 已开启本地日志落盘，目录优先取“公用访问日志策略”的文件 `path`（取目录），为空时回退 `EDGE_DNS_LOG_DIR`，再回退默认 `/var/log/edge/edge-dns`；生成 `access.log`（JSON Lines）。 - **ClickHouse** 已安装并可访问（单机或集群），且已创建好 `logs_ingest` 表（见下文「五、ClickHouse 建表」）。 - 若 Fluent Bit 与 ClickHouse 不在同一台机，需保证网络可达（默认 HTTP 端口 8123）。 @@ -98,7 +100,10 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/ ### 3.4 日志路径与 parsers.conf -- **日志路径**：`fluent-bit.conf` 里 INPUT 的 `Path` 已为 `/var/log/edge/edge-node/*.log`，与 EdgeNode 默认落盘路径一致；若你改了 `EDGE_LOG_DIR`，请同步改此处的 `Path`。 +- **日志路径**：`fluent-bit.conf` 里已同时配置 HTTP 与 DNS 两类路径： + - HTTP：`/var/log/edge/edge-node/*.log` + - DNS：`/var/log/edge/edge-dns/*.log` + 若你配置了公用访问日志策略的文件 `path`，或改了 `EDGE_LOG_DIR` / `EDGE_DNS_LOG_DIR`，请同步修改对应 `Path`。 - **Parsers_File**：主配置引用了 `parsers.conf`。若安装包自带（如 `/etc/fluent-bit/parsers.conf`），无需改动；若启动报错找不到文件，可： - 从 Fluent Bit 官方仓库复制 [conf/parsers.conf](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf) 到同一目录，或 - 在同一目录新建空文件 `parsers.conf`（仅当不使用任何 parser 时）。 @@ -194,7 +199,11 @@ fluent-bit -c /etc/fluent-bit/fluent-bit.conf ## 五、ClickHouse 建表 -平台（EdgeAPI）查询的是表 `logs_ingest`，需在 ClickHouse 中先建表。库名默认为 `default`，若使用其它库，需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。 +平台（EdgeAPI）会查询两张表： +- HTTP：`logs_ingest` +- DNS：`dns_logs_ingest` + +需在 ClickHouse 中先建表。库名默认为 `default`，若使用其它库，需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。 在 ClickHouse 中执行（按需改库名或引擎）： @@ -231,6 +240,34 @@ ORDER BY (timestamp, node_id, server_id, trace_id) SETTINGS index_granularity = 8192; ``` +DNS 日志建表： + +```sql +CREATE TABLE IF NOT EXISTS default.dns_logs_ingest +( + timestamp DateTime, + request_id String, + node_id UInt64, + cluster_id UInt64, + domain_id UInt64, + record_id UInt64, + remote_addr String, + question_name String, + question_type String, + record_name String, + record_type String, + record_value String, + networking String, + is_recursive UInt8, + error String, + ns_route_codes Array(String), + content_json String DEFAULT '' +) +ENGINE = MergeTree() +ORDER BY (timestamp, request_id, node_id) +SETTINGS index_granularity = 8192; +``` + - **log_type**：`access` / `waf` / `error`；攻击日志同时看 **firewall_rule_id** 或 **firewall_policy_id** 是否大于 0（与原有 MySQL 通过规则 ID 判断攻击日志一致）。 - **request_headers / response_headers**：JSON 字符串；**request_body / response_body**：请求/响应体（单条建议限制长度，如 512KB）。 - **request_body 为空**：需在管理端为该站点/服务的「访问日志」策略中勾选「请求Body」后才会落盘；默认未勾选。路径大致为：站点/服务 → 访问日志 → 策略 → 记录字段 → 勾选「请求Body」。WAF 拦截且策略开启「记录请求Body」时也会记录。 @@ -248,6 +285,7 @@ ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_headers String ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_body String DEFAULT ''; ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_headers String DEFAULT ''; ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_body String DEFAULT ''; +ALTER TABLE default.dns_logs_ingest ADD COLUMN IF NOT EXISTS content_json String DEFAULT ''; ``` Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch`，会将 JSON 中的 `timestamp`（Unix 秒）转为 DateTime。 @@ -264,13 +302,15 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch ```sql SELECT count() FROM default.logs_ingest; SELECT * FROM default.logs_ingest LIMIT 5; + SELECT count() FROM default.dns_logs_ingest; + SELECT * FROM default.dns_logs_ingest LIMIT 5; ``` 3. **常见问题** - **连接被拒**：检查 `clickhouse-upstream.conf` 的 Host/Port、防火墙、ClickHouse 的 `listen_host`。 - **认证失败**：检查 `CH_USER`、`CH_PASSWORD` 是否与 ClickHouse 用户一致，环境变量是否被 systemd 正确加载。 - **找不到 parsers.conf**：见上文 3.4。 - - **没有新数据**：确认 EdgeNode 已写日志到 `Path` 下，且 Fluent Bit 对该目录有读权限；可用 `tail -f /var/log/edge/edge-node/access.log` 观察是否有新行。 + - **没有新数据**：确认 EdgeNode/EdgeDNS 已写日志到 `Path` 下，且 Fluent Bit 对目录有读权限；可分别执行 `tail -f /var/log/edge/edge-node/access.log` 与 `tail -f /var/log/edge/edge-dns/access.log`。 - **Node 上没有 `/var/log/edge/edge-node/access.log`**：见下文「八、Node 上找不到日志文件」。 --- @@ -279,7 +319,8 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch | 组件 | 说明 | |------|------| -| **EdgeNode** | 日志落盘路径由 `EDGE_LOG_DIR` 控制，默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`；支持 SIGHUP 重开句柄，可与 logrotate 的 `copytruncate` 配合。 | +| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`（取目录）；若为空回退 `EDGE_LOG_DIR`，再回退默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`；支持 SIGHUP 重开句柄，可与 logrotate 的 `copytruncate` 配合。 | +| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`（取目录）；若为空回退 `EDGE_DNS_LOG_DIR`，再回退默认 `/var/log/edge/edge-dns`；生成 `access.log`（JSON Lines），由 Fluent Bit 采集写入 `dns_logs_ingest`。 | | **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转，避免磁盘占满。 | | **平台（EdgeAPI）** | 配置 ClickHouse 只读连接（`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`）；当请求带 `Day` 且已配置 ClickHouse 时，访问日志列表查询走 ClickHouse。 | @@ -303,6 +344,6 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch 新版本在**首次成功加载节点配置后**会调用 `EnsureInit()`，自动创建 `/var/log/edge/edge-node` 及 `access.log`、`waf.log`、`error.log`。重启一次 edge-node 后再看目录下是否已有文件。 4. **自定义路径** - 若通过环境变量 `EDGE_LOG_DIR` 指定了其它目录，则日志在该目录下；Fluent Bit 的 `Path` 需与之一致。 + 若在管理端设置了公用访问日志策略的文件 `path`，节点会优先使用该目录；否则才使用 `EDGE_LOG_DIR`。Fluent Bit 的 `Path` 需与实际目录一致。 以上完成即完成 Fluent Bit 的部署与验证。 diff --git a/deploy/fluent-bit/fluent-bit-dns.conf b/deploy/fluent-bit/fluent-bit-dns.conf new file mode 100644 index 0000000..3a101b3 --- /dev/null +++ b/deploy/fluent-bit/fluent-bit-dns.conf @@ -0,0 +1,36 @@ +# DNS 节点专用：使用 HTTP 输出写入 ClickHouse（无需 out_clickhouse 插件） +# 启动前设置：CH_USER、CH_PASSWORD；若 ClickHouse 不在本机，请修改 Host、Port +# Read_from_Head=true：首次启动会发送已有日志；若只采新日志建议改为 false + +[SERVICE] + Flush 5 + Log_Level info + Parsers_File parsers.conf + storage.path /var/lib/fluent-bit/storage + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 128MB + +[INPUT] + Name tail + Path /var/log/edge/edge-dns/*.log + Tag app.dns.logs + Parser json + Refresh_Interval 5 + Read_from_Head false + DB /var/lib/fluent-bit/dns-logs.db + Mem_Buf_Limit 128MB + Skip_Long_Lines On + +[OUTPUT] + Name http + Match app.dns.logs + Host 127.0.0.1 + Port 8123 + URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow + Format json_lines + http_user ${CH_USER} + http_passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + Retry_Limit 10 diff --git a/deploy/fluent-bit/fluent-bit.conf b/deploy/fluent-bit/fluent-bit.conf index e5340e6..06a454c 100644 --- a/deploy/fluent-bit/fluent-bit.conf +++ b/deploy/fluent-bit/fluent-bit.conf @@ -1,5 +1,6 @@ # Fluent Bit 主配置（边缘节点日志采集 → ClickHouse） -# 生产环境将 INPUT 改为 tail 采集 /var/log/edge/edge-node/*.log +# HTTP: /var/log/edge/edge-node/*.log +# DNS: /var/log/edge/edge-dns/*.log [SERVICE] Flush 5 @@ -15,16 +16,26 @@ [INPUT] Name tail Path /var/log/edge/edge-node/*.log - Tag app.logs + Tag app.http.logs Refresh_Interval 5 Read_from_Head false - DB /var/lib/fluent-bit/logs.db + DB /var/lib/fluent-bit/http-logs.db + Mem_Buf_Limit 128MB + Skip_Long_Lines On + +[INPUT] + Name tail + Path /var/log/edge/edge-dns/*.log + Tag app.dns.logs + Refresh_Interval 5 + Read_from_Head false + DB /var/lib/fluent-bit/dns-logs.db Mem_Buf_Limit 128MB Skip_Long_Lines On [OUTPUT] Name clickhouse - Match * + Match app.http.logs Upstream ch_backends Table logs_ingest Http_User ${CH_USER} @@ -32,3 +43,14 @@ json_date_key timestamp json_date_format epoch Retry_Limit 10 + +[OUTPUT] + Name clickhouse + Match app.dns.logs + Upstream ch_backends + Table dns_logs_ingest + Http_User ${CH_USER} + Http_Passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + Retry_Limit 10 diff --git a/deploy/fluent-bit/logrotate.conf b/deploy/fluent-bit/logrotate.conf index ff307f9..8daedf9 100644 --- a/deploy/fluent-bit/logrotate.conf +++ b/deploy/fluent-bit/logrotate.conf @@ -9,3 +9,12 @@ notifempty copytruncate } + +/var/log/edge/edge-dns/*.log { + daily + rotate 14 + compress + missingok + notifempty + copytruncate +} diff --git a/go1.21.6.linux-amd64.tar.gz b/go1.21.6.linux-amd64.tar.gz new file mode 100644 index 0000000..e6567ae Binary files /dev/null and b/go1.21.6.linux-amd64.tar.gz differ diff --git a/go1.25.7.linux-amd64.tar.gz b/go1.25.7.linux-amd64.tar.gz new file mode 100644 index 0000000..4a4ae60 Binary files /dev/null and b/go1.25.7.linux-amd64.tar.gz differ diff --git a/编译部署升级策略.md b/编译部署升级策略.md new file mode 100644 index 0000000..e3e0eaf --- /dev/null +++ b/编译部署升级策略.md @@ -0,0 +1,232 @@ +# waf-platform 编译、部署、升级策略（WSL Ubuntu 22.04） + +## 1. 适用范围 + +- 主基线：`E:\AI_PRODUCT\waf-platform`（不是 `waf-platform-1.4.5/1.4.6`）。 +- 本手册覆盖： + - `EdgeAdmin` / `EdgeAPI` / `EdgeNode` / `EdgeDNS` + - HTTP + DNS 访问日志策略 + - Fluent Bit + ClickHouse 日志链路 + +--- + +## 2. 关键结论（先看） + +1. 用 `EdgeAdmin/build/build.sh` 编译时，会联动编译 `EdgeAPI`，并由 `EdgeAPI` 联动编译 `EdgeNode`。 +2. `EdgeDNS` 只有在 `plus` 模式下才会被 `EdgeAPI/build/build.sh` 自动编译并放入 deploy。 +3. 当前脚本已临时关闭自动 `arm64` 编译，只保留 `amd64` 自动链路。 +3. 如果你要发布“本次所有改动”（含 DNS/ClickHouse），建议统一用： + ```bash + cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build + bash build.sh linux amd64 plus + ``` +4. DNS 节点与 Node 节点分离部署时，两边都要有 Fluent Bit（各自采集本机日志）。 + +--- + +## 3. 编译前检查 + +在 WSL Ubuntu 22.04 执行： + +```bash +cd /mnt/e/AI_PRODUCT/waf-platform +git rev-parse --short HEAD +go version +which zip unzip go find sed +``` + +建议： + +- 线上 Ubuntu 22.04，尽量也在 Ubuntu 22.04 编译，避免 `GLIBC`/`GLIBCXX` 不兼容。 +- 若 Node plus 使用 cgo/libpcap/libbrotli，请确保构建机依赖完整。 + +--- + +## 4. 一键编译（推荐） + +```bash +cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build +bash build.sh linux amd64 plus +``` + +### 4.1 此命令会做什么 + +- 编译 `EdgeAdmin` +- 自动调用 `EdgeAPI/build/build.sh` +- `EdgeAPI` 自动编译并打包 `EdgeNode`（当前仅 linux/amd64） +- `plus` 模式下，`EdgeAPI` 自动编译并打包 `EdgeDNS`（当前仅 linux/amd64） +- 把 node/dns 包放入 API 的 `deploy` 目录用于远程安装 + +### 4.2 主要产物位置 + +- Admin 包：`EdgeAdmin/dist/edge-admin-linux-amd64-v*.zip` +- API 包：`EdgeAPI/dist/edge-api-linux-amd64-v*.zip` +- Node 包：`EdgeNode/dist/edge-node-linux-*.zip` +- DNS 包：`EdgeDNS/dist/edge-dns-linux-*.zip`（plus 时） +- API deploy 安装包目录：`EdgeAPI/build/deploy/` + +--- + +## 5. 是否需要单独编译 API / DNS / Node + +### 5.1 不需要单独编译 API 的场景 + +- 你已经执行 `EdgeAdmin/build/build.sh ... plus`，且要发布整套改动。 + +### 5.2 需要单独编译的场景 + +- 只改了 API，不想重新打 Admin： + ```bash + cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAPI/build + bash build.sh linux amd64 plus + ``` +- 只改了 Node： + ```bash + cd /mnt/e/AI_PRODUCT/waf-platform/EdgeNode/build + bash build.sh linux amd64 plus + ``` +- 只改了 DNS： + ```bash + cd /mnt/e/AI_PRODUCT/waf-platform/EdgeDNS/build + bash build.sh linux amd64 + ``` + +--- + +## 6. 升级顺序（生产建议） + +## 6.1 第一步：先改 ClickHouse（DDL） + +先在 ClickHouse 建/改表，至少包含： + +- `logs_ingest`（HTTP） +- `dns_logs_ingest`（DNS） + +先做 DDL 的原因：避免新版本写入时目标表不存在。 + +## 6.2 第二步：部署 Fluent Bit 配置 + +### Node 节点（HTTP） + +- 配置文件目录一般是 `/etc/fluent-bit/` +- 至少更新： + - `fluent-bit.conf`（或你实际启用的 `fluent-bit-http.conf`） + - `clickhouse-upstream.conf` + - `parsers.conf`（通常可复用） + +### DNS 节点（DNS） + +- DNS 节点若之前没装 Fluent Bit，需要先安装并创建 service。 + - `curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh` + - `sudo apt-get update` + - `sudo apt-get install -y fluent-bit` +- 建议同样用 `/etc/fluent-bit/`，放： + - `fluent-bit.conf`（DNS 版本或含 DNS INPUT/OUTPUT 的统一版本） + - `clickhouse-upstream.conf` + - `parsers.conf` + +重启： + +```bash +sudo systemctl restart fluent-bit +sudo systemctl status fluent-bit +``` + +## 6.3 第三步：升级管理面（API + Admin） + +在管理节点更新 `edge-api`、`edge-admin` 包并重启对应服务。 +./bin/edge-api status +./bin/edge-api restart + +## 6.4 第四步：升级数据面（Node / DNS） + +- 通过 API 的远程安装/升级流程分批升级 Node、DNS +- 或手工替换二进制后重启服务 + +## 6.5 第五步：最后切换日志策略 + +在页面启用目标策略（MySQL only / ClickHouse only / 双写），并验证读写链路。 + +--- + +## 7. 日志策略与读写行为（当前实现） + +## 7.1 HTTP / DNS 共用语义 + +- `WriteMySQL=true`：写 MySQL（通过 API） +- `WriteClickHouse=true`：写本地日志文件，由 Fluent Bit 异步采集进 CH +- 两者都开：双写 +- 两者都关：不写 + +## 7.2 查询侧优先级 + +- 优先读 ClickHouse（可用且策略允许） +- ClickHouse 异常时按策略回退 MySQL +- 若两边都不可读，返回空 + +## 7.3 关于“日志文件路径” + +- 现在前端已调整：当存储类型包含 ClickHouse 时，创建/编辑页隐藏“日志文件路径”输入。 +- 但 Fluent Bit 的 `Path` 必须匹配实际日志目录；若你改了日志目录，需要同步改 Fluent Bit 配置并重启。 + +--- + +## 8. 服务检查与常用命令 + +## 8.1 检查 Fluent Bit 服务名 + +```bash +systemctl list-unit-files | grep -Ei 'fluent|td-agent-bit' +systemctl status fluent-bit.service +``` + +## 8.2 查看 Fluent Bit 实际使用的配置文件 + +```bash +systemctl status fluent-bit.service +``` + +重点看 `ExecStart`，例如： + +```text +/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf +``` + +## 8.3 验证 ClickHouse 是否有数据 + +```sql +SELECT count() FROM default.logs_ingest; +SELECT count() FROM default.dns_logs_ingest; +``` + +--- + +## 9. 回滚策略（最小影响） + +1. 先把页面日志策略切回 MySQL-only。 +2. 回滚 API/Admin 到上一版本。 +3. Node/DNS 分批回滚。 +4. Fluent Bit 保留运行不影响主业务（只停止 CH 写入即可）。 + +--- + +## 10. 一次发布的最简执行清单 + +```bash +# 1) 构建 +cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build +bash build.sh linux amd64 plus + +# 2) 上传产物 +# EdgeAdmin/dist/*.zip +# EdgeAPI/dist/*.zip +# EdgeAPI/build/deploy/* (node/dns installer zip) + +# 3) 线上先执行 CH DDL +# 4) 更新 fluent-bit 配置并重启 +sudo systemctl restart fluent-bit + +# 5) 升级 edge-api / edge-admin 并重启 +# 6) 升级 edge-node / edge-dns +# 7) 切日志策略并验证 +``` diff --git a/访问日志策略配置手册.md b/访问日志策略配置手册.md index ecf79ea..7b30086 100644 --- a/访问日志策略配置手册.md +++ b/访问日志策略配置手册.md @@ -139,3 +139,51 @@ flowchart LR - 通常 1 分钟内自动刷新生效。 - 若要立即生效：重启 `edge-api`，并在需要时重启 `edge-node`、`fluent-bit`。 +--- + +## 8. DNS 日志与 HTTP 策略联动（新增） + +从当前版本开始，DNS 访问日志与 HTTP 访问日志共享同一套公用策略语义（`writeTargets`）： + +- `WriteMySQL=true`：DNS 节点上报 API，API 写入 MySQL 分表。 +- `WriteClickHouse=true`：DNS 节点写本地 JSONL，Fluent Bit 采集写入 ClickHouse `dns_logs_ingest`。 +- 双开即双写；双关即不写（仅保留内存处理，不入库）。 + +### 8.1 DNS 写入链路 + +```mermaid +flowchart LR + A[EdgeDNS 产生日志] --> B{writeTargets} + B -->|MySQL=true| C[CreateNSAccessLogs] + C --> D[(MySQL edgeNSAccessLogs_YYYYMMDD)] + B -->|ClickHouse=true| E[/var/log/edge/edge-dns/access.log] + E --> F[Fluent Bit] + F --> G[(ClickHouse dns_logs_ingest)] +``` + +### 8.2 DNS 查询链路 + +```mermaid +flowchart TD + Q[/ns/clusters/accessLogs] --> R{策略是否启用ClickHouse且CH可用} + R -->|是| CH[(dns_logs_ingest)] + R -->|否| M{策略是否启用MySQL} + CH -->|查询失败| M + M -->|是| MY[(MySQL edgeNSAccessLogs_YYYYMMDD)] + M -->|否| E[返回空列表] +``` + +### 8.3 组合场景说明（DNS） + +| 策略 | 写入 | 读取 | +|------|------|------| +| 仅 MySQL | API -> MySQL | MySQL | +| 仅 ClickHouse | 本地文件 -> Fluent Bit -> ClickHouse | ClickHouse | +| MySQL + ClickHouse | API -> MySQL + 本地文件 -> Fluent Bit -> ClickHouse | 优先 ClickHouse，失败回退 MySQL | + +### 8.4 DNS 相关必须配置 + +1. `EdgeAPI` 配置 ClickHouse 连接（仅读 CH 时必须）。 +2. `deploy/fluent-bit/fluent-bit.conf` 已包含 DNS 输入：`/var/log/edge/edge-dns/*.log`。 +3. ClickHouse 已创建 `dns_logs_ingest` 表。 +4. EdgeDNS 运行用户对 `EDGE_DNS_LOG_DIR`（默认 `/var/log/edge/edge-dns`）有写权限。