dns clickhouse改造

This commit is contained in:
robin
2026-02-10 19:30:44 +08:00
parent 4812ad5aaf
commit 1bb8140a41
47 changed files with 2815 additions and 174 deletions

View File

@@ -80,7 +80,7 @@ func (this *StorageManager) Loop() error {
if int64(policy.Id) == publicPolicyId {
this.disableDefaultDB = policy.DisableDefaultDB
this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB)
this.writeTargets = serverconfigs.ResolveWriteTargetsByType(policy.Type, policy.DisableDefaultDB)
foundPolicy = true
}
}
@@ -185,6 +185,9 @@ func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets
}
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
if serverconfigs.IsFileBasedStorageType(storageType) {
storageType = serverconfigs.AccessLogStorageTypeFile
}
switch storageType {
case serverconfigs.AccessLogStorageTypeFile:
var config = &serverconfigs.AccessLogFileStorageConfig{}

View File

@@ -85,6 +85,10 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
if f.Day == "" {
return nil, "", fmt.Errorf("clickhouse: day required")
}
dayNumber, err := normalizeDayNumber(f.Day)
if err != nil {
return nil, "", err
}
table := "logs_ingest"
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest")
@@ -92,7 +96,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
table = quoteIdent(table)
}
conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"}
conditions := []string{"toYYYYMMDD(timestamp) = " + strconv.Itoa(dayNumber)}
if f.HourFrom != "" {
if _, err := strconv.Atoi(f.HourFrom); err == nil {
conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom)
@@ -240,6 +244,22 @@ func escapeString(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
func normalizeDayNumber(day string) (int, error) {
normalized := strings.TrimSpace(day)
if normalized == "" {
return 0, fmt.Errorf("clickhouse: day required")
}
normalized = strings.ReplaceAll(normalized, "-", "")
if len(normalized) != 8 {
return 0, fmt.Errorf("clickhouse: invalid day '%s'", day)
}
dayNumber, err := strconv.Atoi(normalized)
if err != nil {
return 0, fmt.Errorf("clickhouse: invalid day '%s'", day)
}
return dayNumber, nil
}
func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
r := &LogsIngestRow{}
u64 := func(key string) uint64 {

View File

@@ -0,0 +1,375 @@
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// NSLogsIngestRow 对应 dns_logs_ingest 表一行.
type NSLogsIngestRow struct {
Timestamp time.Time
RequestId string
NodeId uint64
ClusterId uint64
DomainId uint64
RecordId uint64
RemoteAddr string
QuestionName string
QuestionType string
RecordName string
RecordType string
RecordValue string
Networking string
IsRecursive bool
Error string
NSRouteCodes []string
ContentJSON string
}
// NSListFilter DNS 日志查询过滤条件.
type NSListFilter struct {
Day string
Size int64
Reverse bool
LastRequestId string
NSClusterId int64
NSNodeId int64
NSDomainId int64
NSRecordId int64
RecordType string
Keyword string
}
// NSLogsIngestStore DNS ClickHouse 查询封装.
type NSLogsIngestStore struct {
client *Client
}
// NewNSLogsIngestStore 创建 NSLogsIngestStore.
func NewNSLogsIngestStore() *NSLogsIngestStore {
return &NSLogsIngestStore{client: NewClient()}
}
// Client 返回底层 client.
func (s *NSLogsIngestStore) Client() *Client {
return s.client
}
// List 列出 DNS 访问日志,返回列表、下一游标与是否有更多.
func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*NSLogsIngestRow, nextCursor string, hasMore bool, err error) {
if !s.client.IsConfigured() {
return nil, "", false, fmt.Errorf("clickhouse: not configured")
}
if f.Day == "" {
return nil, "", false, fmt.Errorf("clickhouse: day required")
}
dayNumber, err := normalizeDayNumber(f.Day)
if err != nil {
return nil, "", false, err
}
table := quoteIdent("dns_logs_ingest")
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("dns_logs_ingest")
}
conditions := []string{"toYYYYMMDD(timestamp) = " + strconv.Itoa(dayNumber)}
if f.NSClusterId > 0 {
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.NSClusterId, 10))
}
if f.NSNodeId > 0 {
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NSNodeId, 10))
}
if f.NSDomainId > 0 {
conditions = append(conditions, "domain_id = "+strconv.FormatInt(f.NSDomainId, 10))
}
if f.NSRecordId > 0 {
conditions = append(conditions, "record_id = "+strconv.FormatInt(f.NSRecordId, 10))
}
if f.RecordType != "" {
conditions = append(conditions, "question_type = '"+escapeString(f.RecordType)+"'")
}
if f.Keyword != "" {
keyword := escapeString(f.Keyword)
conditions = append(conditions, fmt.Sprintf("(remote_addr LIKE '%%%s%%' OR question_name LIKE '%%%s%%' OR record_value LIKE '%%%s%%' OR error LIKE '%%%s%%')", keyword, keyword, keyword, keyword))
}
// 游标分页reverse=false 查更旧reverse=true 查更新。
if f.LastRequestId != "" {
if f.Reverse {
conditions = append(conditions, "request_id > '"+escapeString(f.LastRequestId)+"'")
} else {
conditions = append(conditions, "request_id < '"+escapeString(f.LastRequestId)+"'")
}
}
orderDir := "DESC"
if f.Reverse {
orderDir = "ASC"
}
limit := f.Size
if limit <= 0 {
limit = 20
}
if limit > 1000 {
limit = 1000
}
query := fmt.Sprintf(
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
table,
strings.Join(conditions, " AND "),
orderDir,
orderDir,
limit+1,
)
var rawRows []map[string]interface{}
if err = s.client.Query(ctx, query, &rawRows); err != nil {
return nil, "", false, err
}
rows = make([]*NSLogsIngestRow, 0, len(rawRows))
for _, rawRow := range rawRows {
row := mapToNSLogsIngestRow(rawRow)
if row != nil {
rows = append(rows, row)
}
}
hasMore = len(rows) > int(limit)
if hasMore {
nextCursor = rows[limit].RequestId
rows = rows[:limit]
} else if len(rows) > 0 {
nextCursor = rows[len(rows)-1].RequestId
}
if f.Reverse {
for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 {
rows[left], rows[right] = rows[right], rows[left]
}
if len(rows) > 0 {
nextCursor = rows[0].RequestId
}
}
return rows, nextCursor, hasMore, nil
}
// FindByRequestId 按 request_id 查单条 DNS 日志.
func (s *NSLogsIngestStore) FindByRequestId(ctx context.Context, requestId string) (*NSLogsIngestRow, error) {
if !s.client.IsConfigured() {
return nil, fmt.Errorf("clickhouse: not configured")
}
if requestId == "" {
return nil, nil
}
table := quoteIdent("dns_logs_ingest")
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("dns_logs_ingest")
}
query := fmt.Sprintf(
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE request_id = '%s' LIMIT 1",
table,
escapeString(requestId),
)
var rawRows []map[string]interface{}
if err := s.client.Query(ctx, query, &rawRows); err != nil {
return nil, err
}
if len(rawRows) == 0 {
return nil, nil
}
return mapToNSLogsIngestRow(rawRows[0]), nil
}
func mapToNSLogsIngestRow(row map[string]interface{}) *NSLogsIngestRow {
result := &NSLogsIngestRow{}
u64 := func(key string) uint64 {
value, ok := row[key]
if !ok || value == nil {
return 0
}
switch typed := value.(type) {
case float64:
return uint64(typed)
case string:
number, _ := strconv.ParseUint(typed, 10, 64)
return number
case json.Number:
number, _ := typed.Int64()
return uint64(number)
case int64:
return uint64(typed)
case uint64:
return typed
}
return 0
}
str := func(key string) string {
value, ok := row[key]
if !ok || value == nil {
return ""
}
switch typed := value.(type) {
case string:
return typed
case json.Number:
return typed.String()
default:
return fmt.Sprintf("%v", typed)
}
}
ts := func(key string) time.Time {
value, ok := row[key]
if !ok || value == nil {
return time.Time{}
}
switch typed := value.(type) {
case string:
if typed == "" {
return time.Time{}
}
layouts := []string{
"2006-01-02 15:04:05",
time.RFC3339,
"2006-01-02T15:04:05",
}
for _, layout := range layouts {
if parsed, err := time.Parse(layout, typed); err == nil {
return parsed
}
}
case float64:
return time.Unix(int64(typed), 0)
case json.Number:
number, _ := typed.Int64()
return time.Unix(number, 0)
}
return time.Time{}
}
boolValue := func(key string) bool {
value, ok := row[key]
if !ok || value == nil {
return false
}
switch typed := value.(type) {
case bool:
return typed
case float64:
return typed > 0
case int64:
return typed > 0
case uint64:
return typed > 0
case string:
switch strings.ToLower(strings.TrimSpace(typed)) {
case "1", "true", "yes":
return true
}
}
return false
}
parseStringArray := func(key string) []string {
value, ok := row[key]
if !ok || value == nil {
return nil
}
switch typed := value.(type) {
case []string:
return typed
case []interface{}:
result := make([]string, 0, len(typed))
for _, one := range typed {
if one == nil {
continue
}
result = append(result, fmt.Sprintf("%v", one))
}
return result
case string:
if typed == "" {
return nil
}
var result []string
if json.Unmarshal([]byte(typed), &result) == nil {
return result
}
return []string{typed}
}
return nil
}
result.Timestamp = ts("timestamp")
result.RequestId = str("request_id")
result.NodeId = u64("node_id")
result.ClusterId = u64("cluster_id")
result.DomainId = u64("domain_id")
result.RecordId = u64("record_id")
result.RemoteAddr = str("remote_addr")
result.QuestionName = str("question_name")
result.QuestionType = str("question_type")
result.RecordName = str("record_name")
result.RecordType = str("record_type")
result.RecordValue = str("record_value")
result.Networking = str("networking")
result.IsRecursive = boolValue("is_recursive")
result.Error = str("error")
result.NSRouteCodes = parseStringArray("ns_route_codes")
result.ContentJSON = str("content_json")
return result
}
// NSRowToPB 将 ClickHouse 行转换为 pb.NSAccessLog.
func NSRowToPB(row *NSLogsIngestRow) *pb.NSAccessLog {
if row == nil {
return nil
}
log := &pb.NSAccessLog{
NsNodeId: int64(row.NodeId),
NsDomainId: int64(row.DomainId),
NsRecordId: int64(row.RecordId),
NsRouteCodes: row.NSRouteCodes,
RemoteAddr: row.RemoteAddr,
QuestionName: row.QuestionName,
QuestionType: row.QuestionType,
RecordName: row.RecordName,
RecordType: row.RecordType,
RecordValue: row.RecordValue,
Networking: row.Networking,
Timestamp: row.Timestamp.Unix(),
RequestId: row.RequestId,
Error: row.Error,
IsRecursive: row.IsRecursive,
}
if !row.Timestamp.IsZero() {
log.TimeLocal = row.Timestamp.Format("2/Jan/2006:15:04:05 -0700")
}
if row.ContentJSON != "" {
contentLog := &pb.NSAccessLog{}
if json.Unmarshal([]byte(row.ContentJSON), contentLog) == nil {
contentLog.RequestId = row.RequestId
return contentLog
}
}
return log
}

View File

@@ -108,6 +108,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
// CreatePolicy 创建策略
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) {
_ = writeTargetsJSON
var op = NewHTTPAccessLogPolicyOperator()
op.Name = name
op.Type = policyType
@@ -121,15 +122,14 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
op.IsOn = true
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.WriteTargets = dbs.SQL("NULL")
op.State = HTTPAccessLogPolicyStateEnabled
return this.SaveInt64(tx, op)
}
// UpdatePolicy 修改策略
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error {
_ = writeTargetsJSON
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -167,9 +167,7 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
op.IsPublic = isPublic
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.WriteTargets = dbs.SQL("NULL")
op.IsOn = isOn
return this.Save(tx, op)
}

View File

@@ -0,0 +1,22 @@
package models
import (
"encoding/json"
"strings"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
)
// ParseHTTPAccessLogPolicyFilePath 提取访问日志策略中的文件路径(所有 file* 类型有效)。
func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string {
if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 {
return ""
}
config := &serverconfigs.AccessLogFileStorageConfig{}
if err := json.Unmarshal(policy.Options, config); err != nil {
return ""
}
return strings.TrimSpace(config.Path)
}

View File

@@ -1174,7 +1174,8 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
if publicPolicyId > 0 {
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
if publicPolicy != nil {
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ResolveWriteTargetsByType(publicPolicy.Type, publicPolicy.DisableDefaultDB)
config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
}
}
}

View File

@@ -472,6 +472,16 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.
}
}
}
{
publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx)
if publicPolicyId > 0 {
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
if publicPolicy != nil {
config.AccessLogWriteTargets = serverconfigs.ResolveWriteTargetsByType(publicPolicy.Type, publicPolicy.DisableDefaultDB)
config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
}
}
}
// 递归DNS配置
if IsNotNull(cluster.Recursion) {

View File

@@ -4,6 +4,7 @@ package nameservers
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
@@ -29,11 +30,13 @@ func (this *NSAccessLogService) CreateNSAccessLogs(ctx context.Context, req *pb.
return &pb.CreateNSAccessLogsResponse{}, nil
}
var tx = this.NullTx()
if this.canWriteNSAccessLogsToDB() {
var tx = this.NullTx()
err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
if err != nil {
return nil, err
err = models.SharedNSAccessLogDAO.CreateNSAccessLogs(tx, req.NsAccessLogs)
if err != nil {
return nil, err
}
}
return &pb.CreateNSAccessLogsResponse{}, nil
@@ -54,6 +57,34 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
// TODO 检查权限
}
store := clickhouse.NewNSLogsIngestStore()
canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured() && req.Day != ""
canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
if canReadFromClickHouse {
resp, listErr := this.listNSAccessLogsFromClickHouse(ctx, store, req)
if listErr == nil && resp != nil {
return resp, nil
}
if !canReadFromMySQL {
if listErr != nil {
return nil, listErr
}
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: []*pb.NSAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
}
if !canReadFromMySQL {
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: []*pb.NSAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
accessLogs, requestId, hasMore, err := models.SharedNSAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NsClusterId, req.NsNodeId, req.NsDomainId, req.NsRecordId, req.RecordType, req.Keyword, req.Reverse)
if err != nil {
return nil, err
@@ -67,23 +98,9 @@ func (this *NSAccessLogService) ListNSAccessLogs(ctx context.Context, req *pb.Li
}
// 线路
if len(a.NsRouteCodes) > 0 {
for _, routeCode := range a.NsRouteCodes {
route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
if err != nil {
return nil, err
}
if route != nil {
a.NsRoutes = append(a.NsRoutes, &pb.NSRoute{
Id: types.Int64(route.Id),
IsOn: route.IsOn,
Name: route.Name,
Code: routeCode,
NsCluster: nil,
NsDomain: nil,
})
}
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
result = append(result, a)
@@ -104,6 +121,31 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
return nil, err
}
store := clickhouse.NewNSLogsIngestStore()
canReadFromClickHouse := this.shouldReadNSAccessLogsFromClickHouse() && store.Client().IsConfigured()
canReadFromMySQL := this.shouldReadNSAccessLogsFromMySQL()
if canReadFromClickHouse {
row, findErr := store.FindByRequestId(ctx, req.RequestId)
if findErr != nil {
if !canReadFromMySQL {
return nil, findErr
}
} else if row != nil {
a := clickhouse.NSRowToPB(row)
if a != nil {
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
}
return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
}
}
if !canReadFromMySQL {
return &pb.FindNSAccessLogResponse{NsAccessLog: nil}, nil
}
var tx = this.NullTx()
accessLog, err := models.SharedNSAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
@@ -123,5 +165,70 @@ func (this *NSAccessLogService) FindNSAccessLog(ctx context.Context, req *pb.Fin
if err != nil {
return nil, err
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
return &pb.FindNSAccessLogResponse{NsAccessLog: a}, nil
}
func (this *NSAccessLogService) listNSAccessLogsFromClickHouse(ctx context.Context, store *clickhouse.NSLogsIngestStore, req *pb.ListNSAccessLogsRequest) (*pb.ListNSAccessLogsResponse, error) {
rows, nextCursor, hasMore, err := store.List(ctx, clickhouse.NSListFilter{
Day: req.Day,
Size: req.Size,
Reverse: req.Reverse,
LastRequestId: req.RequestId,
NSClusterId: req.NsClusterId,
NSNodeId: req.NsNodeId,
NSDomainId: req.NsDomainId,
NSRecordId: req.NsRecordId,
RecordType: req.RecordType,
Keyword: req.Keyword,
})
if err != nil {
return nil, err
}
result := make([]*pb.NSAccessLog, 0, len(rows))
for _, row := range rows {
a := clickhouse.NSRowToPB(row)
if a == nil {
continue
}
err = this.fillNSRoutes(a)
if err != nil {
return nil, err
}
result = append(result, a)
}
return &pb.ListNSAccessLogsResponse{
NsAccessLogs: result,
HasMore: hasMore,
RequestId: nextCursor,
}, nil
}
func (this *NSAccessLogService) fillNSRoutes(accessLog *pb.NSAccessLog) error {
if accessLog == nil || len(accessLog.NsRouteCodes) == 0 {
return nil
}
for _, routeCode := range accessLog.NsRouteCodes {
route, err := nameservers.SharedNSRouteDAO.FindEnabledRouteWithCode(nil, routeCode)
if err != nil {
return err
}
if route != nil {
accessLog.NsRoutes = append(accessLog.NsRoutes, &pb.NSRoute{
Id: types.Int64(route.Id),
IsOn: route.IsOn,
Name: route.Name,
Code: routeCode,
NsCluster: nil,
NsDomain: nil,
})
}
}
return nil
}

View File

@@ -0,0 +1,17 @@
//go:build plus
package nameservers
import "github.com/TeaOSLab/EdgeAPI/internal/accesslogs"
func (this *NSAccessLogService) canWriteNSAccessLogsToDB() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *NSAccessLogService) shouldReadNSAccessLogsFromClickHouse() bool {
return accesslogs.SharedStorageManager.WriteClickHouse()
}
func (this *NSAccessLogService) shouldReadNSAccessLogsFromMySQL() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
}

View File

@@ -8,12 +8,27 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/accesslogs"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
)
type HTTPAccessLogPolicyService struct {
BaseService
}
func (this *HTTPAccessLogPolicyService) normalizeStorageTypeAndTargets(policyType string, writeTargetsJSON []byte, disableDefaultDB bool) (string, []byte) {
_ = writeTargetsJSON
_ = disableDefaultDB
// 兼容旧前端/缓存可能传来的历史类型编码
switch policyType {
case "clickhouse":
policyType = serverconfigs.AccessLogStorageTypeFileClickhouse
case "mysql_clickhouse":
policyType = serverconfigs.AccessLogStorageTypeFileMySQLClickhouse
}
return policyType, nil
}
// CountAllHTTPAccessLogPolicies 计算访问日志策略数量
func (this *HTTPAccessLogPolicyService) CountAllHTTPAccessLogPolicies(ctx context.Context, req *pb.CountAllHTTPAccessLogPoliciesRequest) (*pb.RPCCountResponse, error) {
_, err := this.ValidateAdmin(ctx)
@@ -53,7 +68,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
WriteTargetsJSON: nil,
})
}
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -76,8 +91,10 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
}
}
policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB)
// 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON)
if err != nil {
return nil, err
}
@@ -101,8 +118,10 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
}
}
policyType, writeTargetsJSON := this.normalizeStorageTypeAndTargets(req.Type, req.WriteTargetsJSON, req.DisableDefaultDB)
// 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, policyType, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, writeTargetsJSON, req.IsOn)
if err != nil {
return nil, err
}
@@ -134,7 +153,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
WriteTargetsJSON: nil,
}}, nil
}

View File

@@ -18,7 +18,7 @@ echo "build all edge-admin"
echo "=============================="
./build.sh linux amd64 plus
#./build.sh linux 386 plus
./build.sh linux arm64 plus
#./build.sh linux arm64 plus
#./build.sh linux mips64 plus
#./build.sh linux mips64le plus
#./build.sh darwin amd64 plus
@@ -26,4 +26,3 @@ echo "=============================="

View File

@@ -14,8 +14,8 @@ fi
./build.sh linux amd64
./build.sh linux 386
./build.sh linux arm64
#./build.sh linux arm64
./build.sh linux mips64
./build.sh linux mips64le
./build.sh darwin amd64
./build.sh darwin arm64
#./build.sh darwin arm64

View File

@@ -87,17 +87,20 @@ func (this *CreatePopupAction) RunPost(params struct {
Field("type", params.Type).
Require("请选择存储类型")
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
if writeTargets == nil {
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
baseType, _ := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
storedType := baseType
if serverconfigs.IsFileBasedStorageType(params.Type) {
storedType = params.Type
}
var options any = nil
switch baseType {
case serverconfigs.AccessLogStorageTypeFile:
params.Must.
Field("filePath", params.FilePath).
Require("请输入日志文件路径")
if !serverconfigs.IsFileBasedStorageType(params.Type) || (params.Type != serverconfigs.AccessLogStorageTypeFileClickhouse && params.Type != serverconfigs.AccessLogStorageTypeFileMySQLClickhouse) {
params.Must.
Field("filePath", params.FilePath).
Require("请输入日志文件路径")
}
var storage = new(serverconfigs.AccessLogFileStorageConfig)
storage.Path = params.FilePath
@@ -175,21 +178,15 @@ func (this *CreatePopupAction) RunPost(params struct {
this.ErrorPage(err)
return
}
writeTargetsMap := map[string]bool{
"file": writeTargets.File,
"mysql": writeTargets.MySQL,
"clickhouse": writeTargets.ClickHouse,
}
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
createResp, err := this.RPC().HTTPAccessLogPolicyRPC().CreateHTTPAccessLogPolicy(this.AdminContext(), &pb.CreateHTTPAccessLogPolicyRequest{
Name: params.Name,
Type: baseType,
Type: storedType,
OptionsJSON: optionsJSON,
CondsJSON: nil, // TODO
IsPublic: params.IsPublic,
FirewallOnly: params.FirewallOnly,
DisableDefaultDB: params.DisableDefaultDB,
WriteTargetsJSON: writeTargetsJSON,
WriteTargetsJSON: nil,
})
if err != nil {
this.ErrorPage(err)

View File

@@ -46,11 +46,7 @@ func (this *IndexAction) RunGet(params struct{}) {
return
}
}
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
if typeDisplay == "" {
typeDisplay = policy.Type
}
typeDisplay := policy.Type
policyMaps = append(policyMaps, maps.Map{
"id": policy.Id,
"name": policy.Name,

View File

@@ -36,11 +36,7 @@ func InitPolicy(parent *actionutils.ParentAction, policyId int64) error {
}
}
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
if typeDisplay == "" {
typeDisplay = policy.Type
}
typeDisplay := policy.Type
parent.Data["policy"] = maps.Map{
"id": policy.Id,

View File

@@ -106,21 +106,30 @@ func (this *UpdateAction) RunPost(params struct {
Field("type", params.Type).
Require("请选择存储类型")
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
if writeTargets == nil {
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
baseType, _ := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
storedType := baseType
if serverconfigs.IsFileBasedStorageType(params.Type) {
storedType = params.Type
}
var options interface{} = nil
switch baseType {
case serverconfigs.AccessLogStorageTypeFile:
params.Must.
Field("filePath", params.FilePath).
Require("请输入日志文件路径")
var storage = new(serverconfigs.AccessLogFileStorageConfig)
storage.Path = params.FilePath
storage.AutoCreate = params.FileAutoCreate
if params.Type == serverconfigs.AccessLogStorageTypeFileClickhouse || params.Type == serverconfigs.AccessLogStorageTypeFileMySQLClickhouse {
if len(policy.OptionsJSON) > 0 {
_ = json.Unmarshal(policy.OptionsJSON, storage)
}
if len(params.FilePath) > 0 {
storage.Path = params.FilePath
}
} else {
params.Must.
Field("filePath", params.FilePath).
Require("请输入日志文件路径")
storage.Path = params.FilePath
storage.AutoCreate = params.FileAutoCreate
}
options = storage
case serverconfigs.AccessLogStorageTypeES:
params.Must.
@@ -195,23 +204,17 @@ func (this *UpdateAction) RunPost(params struct {
this.ErrorPage(err)
return
}
writeTargetsMap := map[string]bool{
"file": writeTargets.File,
"mysql": writeTargets.MySQL,
"clickhouse": writeTargets.ClickHouse,
}
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
_, err = this.RPC().HTTPAccessLogPolicyRPC().UpdateHTTPAccessLogPolicy(this.AdminContext(), &pb.UpdateHTTPAccessLogPolicyRequest{
HttpAccessLogPolicyId: params.PolicyId,
Name: params.Name,
Type: baseType,
Type: storedType,
OptionsJSON: optionsJSON,
CondsJSON: nil, // TODO
IsOn: params.IsOn,
IsPublic: params.IsPublic,
FirewallOnly: params.FirewallOnly,
DisableDefaultDB: params.DisableDefaultDB,
WriteTargetsJSON: writeTargetsJSON,
WriteTargetsJSON: nil,
})
if err != nil {
this.ErrorPage(err)

View File

@@ -21,8 +21,8 @@
</td>
</tr>
<!-- 文件(文件 / 文件+MySQL / 文件+ClickHouse / 文件+MySQL+ClickHouse -->
<tbody v-show="type == 'file' || type == 'file_mysql' || type == 'file_clickhouse' || type == 'file_mysql_clickhouse'">
<!-- 文件(文件 / 文件+MySQL -->
<tbody v-show="type == 'file' || type == 'file_mysql'">
<tr>
<td>日志文件路径 *</td>
<td>
@@ -36,7 +36,9 @@
<span class="ui label tiny basic">时:${hour}</span>
<span class="ui label tiny basic">分:${minute}</span>
<span class="ui label tiny basic">秒:${second}</span>
<span class="ui label tiny basic">年月日:${date}</span>,比如<span class="ui label tiny basic">/var/log/web-access-${date}.log</span>此文件会在API节点上写入
<span class="ui label tiny basic">年月日:${date}</span>,比如<span class="ui label tiny basic">/var/log/web-access-${date}.log</span>
<span v-if="type == 'file_clickhouse' || type == 'file_mysql_clickhouse'">当存储类型包含 ClickHouse 时,此文件会在节点侧写入,并由 Fluent Bit 采集后写入 ClickHouse。</span>
<span v-else>此文件会在API节点上写入。</span>
</p>
</td>
</tr>
@@ -51,6 +53,15 @@
</td>
</tr>
</tbody>
<!-- 文件+ClickHouse / 文件+MySQL+ClickHouse -->
<tbody v-show="type == 'file_clickhouse' || type == 'file_mysql_clickhouse'">
<tr>
<td>日志文件路径</td>
<td>
<p class="comment">当前类型包含 ClickHouse日志文件路径将由节点侧按公用策略自动复用或回退到默认日志目录无需手动输入。</p>
</td>
</tr>
</tbody>
<!-- Elastic Search -->
<tbody v-show="type == 'es'">

View File

@@ -22,8 +22,8 @@
</td>
</tr>
<!-- 文件 -->
<tbody v-show="policy.typeDisplay == 'file' || policy.typeDisplay == 'file_mysql' || policy.typeDisplay == 'file_clickhouse' || policy.typeDisplay == 'file_mysql_clickhouse'">
<!-- 文件(文件 / 文件+MySQL -->
<tbody v-show="policy.typeDisplay == 'file' || policy.typeDisplay == 'file_mysql'">
<tr>
<td>日志文件路径 *</td>
<td>
@@ -37,7 +37,9 @@
<span class="ui label tiny basic">时:${hour}</span>
<span class="ui label tiny basic">分:${minute}</span>
<span class="ui label tiny basic">秒:${second}</span>
<span class="ui label tiny basic">年月日:${date}</span>,比如<span class="ui label tin basic">/var/log/web-access-${date}.log</span>此文件会在API节点上写入
<span class="ui label tiny basic">年月日:${date}</span>,比如<span class="ui label tiny basic">/var/log/web-access-${date}.log</span>
<span v-if="policy.typeDisplay == 'file_clickhouse' || policy.typeDisplay == 'file_mysql_clickhouse'">当存储类型包含 ClickHouse 时,此文件会在节点侧写入,并由 Fluent Bit 采集后写入 ClickHouse。</span>
<span v-else>此文件会在API节点上写入。</span>
</p>
</td>
</tr>
@@ -52,6 +54,15 @@
</td>
</tr>
</tbody>
<!-- 文件+ClickHouse / 文件+MySQL+ClickHouse -->
<tbody v-show="policy.typeDisplay == 'file_clickhouse' || policy.typeDisplay == 'file_mysql_clickhouse'">
<tr>
<td>日志文件路径</td>
<td>
<p class="comment">当前类型包含 ClickHouse日志文件路径将由节点侧按公用策略自动复用或回退到默认日志目录无需手动输入。</p>
</td>
</tr>
</tbody>
<!-- Elastic Search -->
<tbody v-show="policy.typeDisplay == 'es'">

View File

@@ -11,21 +11,23 @@ import (
)
type NSNodeConfig struct {
Id int64 `yaml:"id" json:"id"`
IsPlus bool `yaml:"isPlus" json:"isPlus"`
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
ClusterId int64 `yaml:"clusterId" json:"clusterId"`
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`
TimeZone string `yaml:"timeZone" json:"timeZone"` // 自动设置时区
Hosts []string `yaml:"hosts" json:"hosts"` // 主机名
Email string `yaml:"email" json:"email"`
SOA *NSSOAConfig `yaml:"soa" json:"soa"` // SOA配置
SOASerial uint32 `yaml:"soaSerial" json:"soaSerial"`
DetectAgents bool `yaml:"detectAgents" json:"detectAgents"` // 是否实时监测Agents
Id int64 `yaml:"id" json:"id"`
IsPlus bool `yaml:"isPlus" json:"isPlus"`
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
ClusterId int64 `yaml:"clusterId" json:"clusterId"`
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"`
AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"`
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`
TimeZone string `yaml:"timeZone" json:"timeZone"` // 自动设置时区
Hosts []string `yaml:"hosts" json:"hosts"` // 主机名
Email string `yaml:"email" json:"email"`
SOA *NSSOAConfig `yaml:"soa" json:"soa"` // SOA配置
SOASerial uint32 `yaml:"soaSerial" json:"soaSerial"`
DetectAgents bool `yaml:"detectAgents" json:"detectAgents"` // 是否实时监测Agents
TCP *serverconfigs.TCPProtocolConfig `yaml:"tcp" json:"tcp"` // TCP配置
TLS *serverconfigs.TLSProtocolConfig `yaml:"tls" json:"tls"` // TLS配置

View File

@@ -84,6 +84,33 @@ func ParseStorageTypeAndWriteTargets(selectedType string) (baseType string, writ
return baseType, writeTargets
}
// ResolveWriteTargetsByType 仅根据策略类型与 disableDefaultDB 计算写入目标(不依赖 writeTargets 字段)
func ResolveWriteTargetsByType(policyType string, disableDefaultDB bool) *AccessLogWriteTargets {
t := &AccessLogWriteTargets{}
switch policyType {
case AccessLogStorageTypeFileMySQL:
t.File = true
t.MySQL = true
case AccessLogStorageTypeFileClickhouse:
t.File = true
t.ClickHouse = true
case AccessLogStorageTypeFileMySQLClickhouse:
t.File = true
t.MySQL = true
t.ClickHouse = true
case AccessLogStorageTypeFile:
t.File = true
t.MySQL = !disableDefaultDB
default:
t.MySQL = !disableDefaultDB
}
if !t.File && !t.MySQL && !t.ClickHouse {
t.File = true
t.MySQL = true
}
return t
}
// ComposeStorageTypeDisplay 根据策略的 Type + WriteTargets 得到下拉框显示用的类型 code用于编辑页回显
func ComposeStorageTypeDisplay(policyType string, writeTargets *AccessLogWriteTargets) string {
if policyType != AccessLogStorageTypeFile {

View File

@@ -2,8 +2,6 @@
package serverconfigs
import "encoding/json"
// AccessLogWriteTargets 访问日志写入目标(双写/单写文件、MySQL、ClickHouse
type AccessLogWriteTargets struct {
File bool `yaml:"file" json:"file"` // 写本地 JSON 文件(供 Fluent Bit → ClickHouse 或自用)
@@ -27,23 +25,8 @@ func (t *AccessLogWriteTargets) NeedWriteFile() bool {
return t.File
}
// ParseWriteTargetsFromPolicy 从策略的 writeTargets JSON 与旧字段解析;无 writeTargets 时按 type + disableDefaultDB 推断
// ParseWriteTargetsFromPolicy 兼容入口:当前统一按 type 推导写入目标,不再依赖 writeTargets 字段
func ParseWriteTargetsFromPolicy(writeTargetsJSON []byte, policyType string, disableDefaultDB bool) *AccessLogWriteTargets {
if len(writeTargetsJSON) > 0 {
var t AccessLogWriteTargets
if err := json.Unmarshal(writeTargetsJSON, &t); err == nil {
return &t
}
}
// 兼容旧策略type=file 视为写文件,!disableDefaultDB 视为写 MySQL
t := &AccessLogWriteTargets{
File: policyType == AccessLogStorageTypeFile,
MySQL: !disableDefaultDB,
ClickHouse: false,
}
if !t.File && !t.MySQL && !t.ClickHouse {
t.File = true
t.MySQL = true
}
return t
_ = writeTargetsJSON
return ResolveWriteTargetsByType(policyType, disableDefaultDB)
}

View File

@@ -78,6 +78,7 @@ type GlobalServerConfig struct {
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse双写/单写)
FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用)
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
Stat struct {

View File

@@ -2,7 +2,7 @@
./build.sh linux amd64
#./build.sh linux 386
./build.sh linux arm64
#./build.sh linux arm64
#./build.sh linux mips64
#./build.sh linux mips64le
#./build.sh darwin amd64

View File

@@ -24,6 +24,8 @@ require (
github.com/josharian/native v1.1.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mdlayher/socket v0.5.0 // indirect
github.com/oschwald/geoip2-golang v1.13.0 // indirect
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect

View File

@@ -1,15 +1,23 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8=
github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/iwind/TeaGo v0.0.0-20240128112714-6bcd0529d0ea h1:o0QCF6tMJ9E6OgU1c0L+rYDshKsTu7mEk+7KCGLbnpI=
github.com/iwind/TeaGo v0.0.0-20240128112714-6bcd0529d0ea/go.mod h1:Ng3xWekHSVy0E/6/jYqJ7Htydm/H+mWIl0AS+Eg3H2M=
github.com/iwind/gosock v0.0.0-20220505115348-f88412125a62 h1:HJH6RDheAY156DnIfJSD/bEvqyXzsZuE2gzs8PuUjoo=
@@ -30,6 +38,10 @@ github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI
github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
@@ -45,8 +57,9 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4=
github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0=
@@ -57,39 +70,43 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrB
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=

View File

@@ -0,0 +1,199 @@
package accesslogs
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
)
const (
defaultDNSLogDir = "/var/log/edge/edge-dns"
envDNSLogDir = "EDGE_DNS_LOG_DIR"
)
var (
sharedDNSFileWriter *DNSFileWriter
sharedDNSFileWriterOnce sync.Once
)
// SharedDNSFileWriter 返回 DNS 本地日志写入器(单例).
func SharedDNSFileWriter() *DNSFileWriter {
sharedDNSFileWriterOnce.Do(func() {
sharedDNSFileWriter = NewDNSFileWriter()
})
return sharedDNSFileWriter
}
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集.
type DNSFileWriter struct {
dir string
mu sync.Mutex
file *os.File
inited bool
}
// NewDNSFileWriter 创建 DNS 本地日志写入器.
func NewDNSFileWriter() *DNSFileWriter {
logDir := resolveDefaultDNSLogDir()
return &DNSFileWriter{dir: logDir}
}
func resolveDefaultDNSLogDir() string {
logDir := strings.TrimSpace(os.Getenv(envDNSLogDir))
if logDir == "" {
return defaultDNSLogDir
}
return logDir
}
func resolveDNSDirFromPolicyPath(policyPath string) string {
policyPath = strings.TrimSpace(policyPath)
if policyPath == "" {
return ""
}
if strings.HasSuffix(policyPath, "/") || strings.HasSuffix(policyPath, "\\") {
return filepath.Clean(policyPath)
}
baseName := filepath.Base(policyPath)
if strings.Contains(baseName, ".") || strings.Contains(baseName, "${") {
return filepath.Clean(filepath.Dir(policyPath))
}
return filepath.Clean(policyPath)
}
// Dir 返回当前日志目录.
func (w *DNSFileWriter) Dir() string {
return w.dir
}
// SetDirByPolicyPath 使用公用日志策略 path 更新目录,空值时回退到 EDGE_DNS_LOG_DIR/default。
func (w *DNSFileWriter) SetDirByPolicyPath(policyPath string) {
dir := resolveDNSDirFromPolicyPath(policyPath)
w.SetDir(dir)
}
// SetDir 更新目录并重置文件句柄。
func (w *DNSFileWriter) SetDir(dir string) {
if strings.TrimSpace(dir) == "" {
dir = resolveDefaultDNSLogDir()
}
w.mu.Lock()
defer w.mu.Unlock()
if dir == w.dir {
return
}
if w.file != nil {
_ = w.file.Close()
w.file = nil
}
w.inited = false
w.dir = dir
}
// EnsureInit 在启动时预创建目录与 access.log.
func (w *DNSFileWriter) EnsureInit() error {
if w.dir == "" {
return nil
}
return w.init()
}
func (w *DNSFileWriter) init() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.inited && w.file != nil {
return nil
}
if w.dir == "" {
return nil
}
if err := os.MkdirAll(w.dir, 0755); err != nil {
remotelogs.Error("DNS_ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error())
return err
}
fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error())
return err
}
w.file = fp
w.inited = true
return nil
}
// WriteBatch 批量写入 DNS 访问日志.
func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
if len(logs) == 0 || w.dir == "" {
return
}
if err := w.init(); err != nil {
return
}
w.mu.Lock()
fp := w.file
w.mu.Unlock()
if fp == nil {
return
}
for _, log := range logs {
ingestLog := FromNSAccessLog(log, clusterId)
if ingestLog == nil {
continue
}
line, err := json.Marshal(ingestLog)
if err != nil {
continue
}
_, _ = fp.Write(append(line, '\n'))
}
}
// Reopen 关闭并重新打开日志文件(配合 logrotate.
func (w *DNSFileWriter) Reopen() error {
w.mu.Lock()
if w.file != nil {
_ = w.file.Close()
w.file = nil
}
w.inited = false
w.mu.Unlock()
return w.init()
}
// Close 关闭日志文件.
func (w *DNSFileWriter) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.file == nil {
return nil
}
err := w.file.Close()
w.file = nil
w.inited = false
if err != nil {
remotelogs.Error("DNS_ACCESS_LOG_FILE", fmt.Sprintf("close access.log failed: %v", err))
return err
}
return nil
}

View File

@@ -0,0 +1,57 @@
package accesslogs
import (
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// DNSIngestLog DNS 访问日志单行结构JSONEachRow.
type DNSIngestLog struct {
Timestamp int64 `json:"timestamp"`
RequestId string `json:"request_id"`
NodeId int64 `json:"node_id"`
ClusterId int64 `json:"cluster_id"`
DomainId int64 `json:"domain_id"`
RecordId int64 `json:"record_id"`
RemoteAddr string `json:"remote_addr"`
QuestionName string `json:"question_name"`
QuestionType string `json:"question_type"`
RecordName string `json:"record_name"`
RecordType string `json:"record_type"`
RecordValue string `json:"record_value"`
Networking string `json:"networking"`
IsRecursive bool `json:"is_recursive"`
Error string `json:"error"`
NSRouteCodes []string `json:"ns_route_codes,omitempty"`
ContentJSON string `json:"content_json,omitempty"`
}
// FromNSAccessLog 将 pb.NSAccessLog 转为 DNSIngestLog.
func FromNSAccessLog(log *pb.NSAccessLog, clusterId int64) *DNSIngestLog {
if log == nil {
return nil
}
contentBytes, _ := json.Marshal(log)
return &DNSIngestLog{
Timestamp: log.GetTimestamp(),
RequestId: log.GetRequestId(),
NodeId: log.GetNsNodeId(),
ClusterId: clusterId,
DomainId: log.GetNsDomainId(),
RecordId: log.GetNsRecordId(),
RemoteAddr: log.GetRemoteAddr(),
QuestionName: log.GetQuestionName(),
QuestionType: log.GetQuestionType(),
RecordName: log.GetRecordName(),
RecordType: log.GetRecordType(),
RecordValue: log.GetRecordValue(),
Networking: log.GetNetworking(),
IsRecursive: log.GetIsRecursive(),
Error: log.GetError(),
NSRouteCodes: log.GetNsRouteCodes(),
ContentJSON: string(contentBytes),
}
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeDNS/internal/accesslogs"
"github.com/TeaOSLab/EdgeDNS/internal/configs"
teaconst "github.com/TeaOSLab/EdgeDNS/internal/const"
"github.com/TeaOSLab/EdgeDNS/internal/events"
@@ -108,6 +109,15 @@ func (this *NodeConfigManager) NotifyChange() {
func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) {
teaconst.IsPlus = config.IsPlus
accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
if needWriteFile {
_ = accesslogs.SharedDNSFileWriter().EnsureInit()
} else {
_ = accesslogs.SharedDNSFileWriter().Close()
}
// timezone
var timeZone = config.TimeZone
if len(timeZone) == 0 {

View File

@@ -2,6 +2,7 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeDNS/internal/accesslogs"
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
"github.com/TeaOSLab/EdgeDNS/internal/rpc"
"strconv"
@@ -89,6 +90,26 @@ Loop:
return nil
}
var clusterId int64
var needWriteFile = true
var needReportAPI = true
if sharedNodeConfig != nil {
clusterId = sharedNodeConfig.ClusterId
if sharedNodeConfig.AccessLogWriteTargets != nil {
targets := sharedNodeConfig.AccessLogWriteTargets
needWriteFile = targets.File || targets.ClickHouse
needReportAPI = targets.MySQL
}
}
if needWriteFile {
accesslogs.SharedDNSFileWriter().WriteBatch(accessLogs, clusterId)
}
if !needReportAPI {
return nil
}
// 发送到API
client, err := rpc.SharedRPC()
if err != nil {

View File

@@ -2,7 +2,7 @@
./build.sh linux amd64 plus
#./build.sh linux 386 plus
./build.sh linux arm64 plus
#./build.sh linux arm64 plus
#./build.sh linux mips64 plus
#./build.sh linux mips64le plus
#./build.sh darwin amd64 plus

View File

@@ -2,7 +2,7 @@
./build.sh linux amd64
#./build.sh linux 386
./build.sh linux arm64
#./build.sh linux arm64
#./build.sh linux mips64
#./build.sh linux mips64le
#./build.sh darwin amd64

View File

@@ -1 +1 @@
{"speed":2,"speedMB":670,"countTests":2}
{"speed":1,"speedMB":1400,"countTests":3}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
@@ -39,21 +40,73 @@ type FileWriter struct {
// NewFileWriter 创建本地日志文件写入器
func NewFileWriter() *FileWriter {
dir := os.Getenv(envLogDir)
if dir == "" {
dir = defaultLogDir
}
dir := resolveDefaultLogDir()
return &FileWriter{
dir: dir,
files: make(map[string]*os.File),
}
}
func resolveDefaultLogDir() string {
dir := strings.TrimSpace(os.Getenv(envLogDir))
if dir == "" {
return defaultLogDir
}
return dir
}
func resolveDirFromPolicyPath(policyPath string) string {
policyPath = strings.TrimSpace(policyPath)
if policyPath == "" {
return ""
}
if strings.HasSuffix(policyPath, "/") || strings.HasSuffix(policyPath, "\\") {
return filepath.Clean(policyPath)
}
baseName := filepath.Base(policyPath)
if strings.Contains(baseName, ".") || strings.Contains(baseName, "${") {
return filepath.Clean(filepath.Dir(policyPath))
}
return filepath.Clean(policyPath)
}
// Dir 返回当前配置的日志目录
func (w *FileWriter) Dir() string {
return w.dir
}
// SetDirByPolicyPath 使用公用日志策略 path 更新目录,空值时回退到 EDGE_LOG_DIR/default。
func (w *FileWriter) SetDirByPolicyPath(policyPath string) {
dir := resolveDirFromPolicyPath(policyPath)
w.SetDir(dir)
}
// SetDir 更新日志目录并重置文件句柄。
func (w *FileWriter) SetDir(dir string) {
if strings.TrimSpace(dir) == "" {
dir = resolveDefaultLogDir()
}
w.mu.Lock()
defer w.mu.Unlock()
if dir == w.dir {
return
}
for name, f := range w.files {
if f != nil {
_ = f.Close()
}
w.files[name] = nil
}
w.inited = false
w.dir = dir
}
// IsEnabled 是否启用落盘(目录非空即视为启用)
func (w *FileWriter) IsEnabled() bool {
return w.dir != ""

View File

@@ -887,6 +887,12 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) {
nodeconfigs.ResetNodeConfig(config)
sharedNodeConfig = config
var accessLogFilePath string
if config != nil && config.GlobalServerConfig != nil {
accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
}
accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)
// 并发读写数
fsutils.ReaderLimiter.SetThreads(config.MaxConcurrentReads)
fsutils.WriterLimiter.SetThreads(config.MaxConcurrentWrites)

View File

@@ -2,10 +2,10 @@
./build.sh linux amd64
./build.sh linux 386
./build.sh linux arm64
#./build.sh linux arm64
./build.sh linux mips64
./build.sh linux mips64le
./build.sh darwin amd64
./build.sh darwin arm64
#./build.sh darwin arm64
./build.sh windows amd64
./build.sh windows 386

View File

@@ -2,7 +2,7 @@
./build.sh linux amd64
#./build.sh linux 386
./build.sh linux arm64
#./build.sh linux arm64
#./build.sh linux mips64
#./build.sh linux mips64le
#./build.sh darwin amd64

Binary file not shown.

File diff suppressed because it is too large Load Diff

Binary file not shown.

View File

@@ -6,18 +6,20 @@
## Fluent Bit 跑在哪台机器上?
**Fluent Bit 应部署在每台 EdgeNode 机器上**(与 edge-node 同机),不要部署在 EdgeAPI 机器上。
**Fluent Bit 应部署在写日志文件的节点机器上**EdgeNode / EdgeDNS 同机),不要部署在 EdgeAPI 机器上。
- 日志文件(`/var/log/edge/edge-node/*.log`)是 **EdgeNode** 本机写的,只有 EdgeNode 所在机器才有这些文件
- Fluent Bit 使用 **tail** 插件读取本机路径,因此必须运行在 **有这些日志文件的机器** 上,即每台 EdgeNode
- EdgeAPI 机器上没有边缘节点日志,只负责查询 ClickHouse/MySQL因此不需要在 EdgeAPI 上跑 Fluent Bit
- **多台 EdgeNode 时**:每台 EdgeNode 各跑一份 Fluent Bit各自采集本机日志并上报到同一 ClickHouse
- HTTP 日志文件默认在 `/var/log/edge/edge-node/*.log`,由 **EdgeNode** 本机写入;若配置了公用访问日志策略的文件 `path`,节点会优先复用该 `path` 所在目录
- DNS 日志文件默认在 `/var/log/edge/edge-dns/*.log`,由 **EdgeDNS** 本机写入;若配置了公用访问日志策略的文件 `path`,节点会优先复用该 `path` 所在目录
- Fluent Bit 使用 **tail** 读取本机路径,因此必须运行在这些日志文件所在机器上
- EdgeAPI 机器主要负责查询 ClickHouse/MySQL不需要承担日志采集
- 多机部署时,每台写日志节点都跑一份 Fluent Bit上报到同一 ClickHouse 集群。
---
## 一、前置条件
- **边缘节点EdgeNode** 已开启本地日志落盘,日志目录默认 `/var/log/edge/edge-node`,会生成 `access.log``waf.log``error.log`JSON Lines由环境变量 `EDGE_LOG_DIR` 控制路径。
- **边缘节点EdgeNode** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`生成 `access.log``waf.log``error.log`JSON Lines
- **DNS 节点EdgeDNS** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest`见下文「五、ClickHouse 建表」)。
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123
@@ -98,7 +100,10 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
### 3.4 日志路径与 parsers.conf
- **日志路径**`fluent-bit.conf` INPUT 的 `Path` 已为 `/var/log/edge/edge-node/*.log`,与 EdgeNode 默认落盘路径一致;若你改了 `EDGE_LOG_DIR`,请同步改此处的 `Path`
- **日志路径**`fluent-bit.conf`已同时配置 HTTP 与 DNS 两类路径:
- HTTP`/var/log/edge/edge-node/*.log`
- DNS`/var/log/edge/edge-dns/*.log`
若你配置了公用访问日志策略的文件 `path`,或改了 `EDGE_LOG_DIR` / `EDGE_DNS_LOG_DIR`,请同步修改对应 `Path`
- **Parsers_File**:主配置引用了 `parsers.conf`。若安装包自带(如 `/etc/fluent-bit/parsers.conf`),无需改动;若启动报错找不到文件,可:
- 从 Fluent Bit 官方仓库复制 [conf/parsers.conf](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf) 到同一目录,或
- 在同一目录新建空文件 `parsers.conf`(仅当不使用任何 parser 时)。
@@ -194,7 +199,11 @@ fluent-bit -c /etc/fluent-bit/fluent-bit.conf
## 五、ClickHouse 建表
平台EdgeAPI查询的是表 `logs_ingest`,需在 ClickHouse 中先建表。库名默认为 `default`,若使用其它库,需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。
平台EdgeAPI查询两张表:
- HTTP`logs_ingest`
- DNS`dns_logs_ingest`
需在 ClickHouse 中先建表。库名默认为 `default`,若使用其它库,需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。
在 ClickHouse 中执行(按需改库名或引擎):
@@ -231,6 +240,34 @@ ORDER BY (timestamp, node_id, server_id, trace_id)
SETTINGS index_granularity = 8192;
```
DNS 日志建表:
```sql
CREATE TABLE IF NOT EXISTS default.dns_logs_ingest
(
timestamp DateTime,
request_id String,
node_id UInt64,
cluster_id UInt64,
domain_id UInt64,
record_id UInt64,
remote_addr String,
question_name String,
question_type String,
record_name String,
record_type String,
record_value String,
networking String,
is_recursive UInt8,
error String,
ns_route_codes Array(String),
content_json String DEFAULT ''
)
ENGINE = MergeTree()
ORDER BY (timestamp, request_id, node_id)
SETTINGS index_granularity = 8192;
```
- **log_type**`access` / `waf` / `error`;攻击日志同时看 **firewall_rule_id****firewall_policy_id** 是否大于 0与原有 MySQL 通过规则 ID 判断攻击日志一致)。
- **request_headers / response_headers**JSON 字符串;**request_body / response_body**:请求/响应体(单条建议限制长度,如 512KB
- **request_body 为空**:需在管理端为该站点/服务的「访问日志」策略中勾选「请求Body」后才会落盘默认未勾选。路径大致为站点/服务 → 访问日志 → 策略 → 记录字段 → 勾选「请求Body」。WAF 拦截且策略开启「记录请求Body」时也会记录。
@@ -248,6 +285,7 @@ ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_headers String
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_body String DEFAULT '';
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_headers String DEFAULT '';
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_body String DEFAULT '';
ALTER TABLE default.dns_logs_ingest ADD COLUMN IF NOT EXISTS content_json String DEFAULT '';
```
Fluent Bit 写入时使用 `json_date_key timestamp``json_date_format epoch`,会将 JSON 中的 `timestamp`Unix 秒)转为 DateTime。
@@ -264,13 +302,15 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
```sql
SELECT count() FROM default.logs_ingest;
SELECT * FROM default.logs_ingest LIMIT 5;
SELECT count() FROM default.dns_logs_ingest;
SELECT * FROM default.dns_logs_ingest LIMIT 5;
```
3. **常见问题**
- **连接被拒**:检查 `clickhouse-upstream.conf` 的 Host/Port、防火墙、ClickHouse 的 `listen_host`。
- **认证失败**:检查 `CH_USER`、`CH_PASSWORD` 是否与 ClickHouse 用户一致,环境变量是否被 systemd 正确加载。
- **找不到 parsers.conf**:见上文 3.4。
- **没有新数据**:确认 EdgeNode 已写日志到 `Path` 下,且 Fluent Bit 对目录有读权限;可 `tail -f /var/log/edge/edge-node/access.log` 观察是否有新行
- **没有新数据**:确认 EdgeNode/EdgeDNS 已写日志到 `Path` 下,且 Fluent Bit 对目录有读权限;可分别执行 `tail -f /var/log/edge/edge-node/access.log` 与 `tail -f /var/log/edge/edge-dns/access.log`
- **Node 上没有 `/var/log/edge/edge-node/access.log`**见下文「八、Node 上找不到日志文件」。
---
@@ -279,7 +319,8 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
| 组件 | 说明 |
|------|------|
| **EdgeNode** | 日志落盘路径 `EDGE_LOG_DIR` 控制,默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines由 Fluent Bit 采集写入 `dns_logs_ingest`。 |
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 |
| **平台EdgeAPI** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
@@ -303,6 +344,6 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
新版本在**首次成功加载节点配置后**会调用 `EnsureInit()`,自动创建 `/var/log/edge/edge-node` 及 `access.log`、`waf.log`、`error.log`。重启一次 edge-node 后再看目录下是否已有文件。
4. **自定义路径**
通过环境变量 `EDGE_LOG_DIR` 指定了其它目录,则日志在该目录下;Fluent Bit 的 `Path` 需与一致。
在管理端设置了公用访问日志策略的文件 `path`,节点会优先使用该目录;否则才使用 `EDGE_LOG_DIR`。Fluent Bit 的 `Path` 需与实际目录一致。
以上完成即完成 Fluent Bit 的部署与验证。

View File

@@ -0,0 +1,36 @@
# DNS 节点专用:使用 HTTP 输出写入 ClickHouse无需 out_clickhouse 插件)
# 启动前设置CH_USER、CH_PASSWORD若 ClickHouse 不在本机,请修改 Host、Port
# Read_from_Head=true首次启动会发送已有日志若只采新日志建议改为 false
[SERVICE]
Flush 5
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 128MB
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Parser json
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8123
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
Retry_Limit 10

View File

@@ -1,5 +1,6 @@
# Fluent Bit 主配置(边缘节点日志采集 → ClickHouse
# 生产环境将 INPUT 改为 tail 采集 /var/log/edge/edge-node/*.log
# HTTP: /var/log/edge/edge-node/*.log
# DNS: /var/log/edge/edge-dns/*.log
[SERVICE]
Flush 5
@@ -15,16 +16,26 @@
[INPUT]
Name tail
Path /var/log/edge/edge-node/*.log
Tag app.logs
Tag app.http.logs
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/logs.db
DB /var/lib/fluent-bit/http-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name clickhouse
Match *
Match app.http.logs
Upstream ch_backends
Table logs_ingest
Http_User ${CH_USER}
@@ -32,3 +43,14 @@
json_date_key timestamp
json_date_format epoch
Retry_Limit 10
[OUTPUT]
Name clickhouse
Match app.dns.logs
Upstream ch_backends
Table dns_logs_ingest
Http_User ${CH_USER}
Http_Passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
Retry_Limit 10

View File

@@ -9,3 +9,12 @@
notifempty
copytruncate
}
/var/log/edge/edge-dns/*.log {
daily
rotate 14
compress
missingok
notifempty
copytruncate
}

BIN
go1.21.6.linux-amd64.tar.gz Normal file

Binary file not shown.

BIN
go1.25.7.linux-amd64.tar.gz Normal file

Binary file not shown.

232
编译部署升级策略.md Normal file
View File

@@ -0,0 +1,232 @@
# waf-platform 编译、部署、升级策略WSL Ubuntu 22.04
## 1. 适用范围
- 主基线:`E:\AI_PRODUCT\waf-platform`(不是 `waf-platform-1.4.5/1.4.6`)。
- 本手册覆盖:
- `EdgeAdmin` / `EdgeAPI` / `EdgeNode` / `EdgeDNS`
- HTTP + DNS 访问日志策略
- Fluent Bit + ClickHouse 日志链路
---
## 2. 关键结论(先看)
1.`EdgeAdmin/build/build.sh` 编译时,会联动编译 `EdgeAPI`,并由 `EdgeAPI` 联动编译 `EdgeNode`
2. `EdgeDNS` 只有在 `plus` 模式下才会被 `EdgeAPI/build/build.sh` 自动编译并放入 deploy。
3. 当前脚本已临时关闭自动 `arm64` 编译,只保留 `amd64` 自动链路。
3. 如果你要发布“本次所有改动”(含 DNS/ClickHouse建议统一用
```bash
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build
bash build.sh linux amd64 plus
```
4. DNS 节点与 Node 节点分离部署时,两边都要有 Fluent Bit各自采集本机日志
---
## 3. 编译前检查
在 WSL Ubuntu 22.04 执行:
```bash
cd /mnt/e/AI_PRODUCT/waf-platform
git rev-parse --short HEAD
go version
which zip unzip go find sed
```
建议:
- 线上 Ubuntu 22.04,尽量也在 Ubuntu 22.04 编译,避免 `GLIBC`/`GLIBCXX` 不兼容。
- 若 Node plus 使用 cgo/libpcap/libbrotli请确保构建机依赖完整。
---
## 4. 一键编译(推荐)
```bash
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build
bash build.sh linux amd64 plus
```
### 4.1 此命令会做什么
- 编译 `EdgeAdmin`
- 自动调用 `EdgeAPI/build/build.sh`
- `EdgeAPI` 自动编译并打包 `EdgeNode`(当前仅 linux/amd64
- `plus` 模式下,`EdgeAPI` 自动编译并打包 `EdgeDNS`(当前仅 linux/amd64
- 把 node/dns 包放入 API 的 `deploy` 目录用于远程安装
### 4.2 主要产物位置
- Admin 包:`EdgeAdmin/dist/edge-admin-linux-amd64-v*.zip`
- API 包:`EdgeAPI/dist/edge-api-linux-amd64-v*.zip`
- Node 包:`EdgeNode/dist/edge-node-linux-*.zip`
- DNS 包:`EdgeDNS/dist/edge-dns-linux-*.zip`plus 时)
- API deploy 安装包目录:`EdgeAPI/build/deploy/`
---
## 5. 是否需要单独编译 API / DNS / Node
### 5.1 不需要单独编译 API 的场景
- 你已经执行 `EdgeAdmin/build/build.sh ... plus`,且要发布整套改动。
### 5.2 需要单独编译的场景
- 只改了 API不想重新打 Admin
```bash
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAPI/build
bash build.sh linux amd64 plus
```
- 只改了 Node
```bash
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeNode/build
bash build.sh linux amd64 plus
```
- 只改了 DNS
```bash
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeDNS/build
bash build.sh linux amd64
```
---
## 6. 升级顺序(生产建议)
## 6.1 第一步:先改 ClickHouseDDL
先在 ClickHouse 建/改表,至少包含:
- `logs_ingest`HTTP
- `dns_logs_ingest`DNS
先做 DDL 的原因:避免新版本写入时目标表不存在。
## 6.2 第二步:部署 Fluent Bit 配置
### Node 节点HTTP
- 配置文件目录一般是 `/etc/fluent-bit/`
- 至少更新:
- `fluent-bit.conf`(或你实际启用的 `fluent-bit-http.conf`
- `clickhouse-upstream.conf`
- `parsers.conf`(通常可复用)
### DNS 节点DNS
- DNS 节点若之前没装 Fluent Bit需要先安装并创建 service。
- `curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh`
- `sudo apt-get update`
- `sudo apt-get install -y fluent-bit`
- 建议同样用 `/etc/fluent-bit/`,放:
- `fluent-bit.conf`DNS 版本或含 DNS INPUT/OUTPUT 的统一版本)
- `clickhouse-upstream.conf`
- `parsers.conf`
重启:
```bash
sudo systemctl restart fluent-bit
sudo systemctl status fluent-bit
```
## 6.3 第三步升级管理面API + Admin
在管理节点更新 `edge-api`、`edge-admin` 包并重启对应服务。
./bin/edge-api status
./bin/edge-api restart
## 6.4 第四步升级数据面Node / DNS
- 通过 API 的远程安装/升级流程分批升级 Node、DNS
- 或手工替换二进制后重启服务
## 6.5 第五步:最后切换日志策略
在页面启用目标策略MySQL only / ClickHouse only / 双写),并验证读写链路。
---
## 7. 日志策略与读写行为(当前实现)
## 7.1 HTTP / DNS 共用语义
- `WriteMySQL=true`:写 MySQL通过 API
- `WriteClickHouse=true`:写本地日志文件,由 Fluent Bit 异步采集进 CH
- 两者都开:双写
- 两者都关:不写
## 7.2 查询侧优先级
- 优先读 ClickHouse可用且策略允许
- ClickHouse 异常时按策略回退 MySQL
- 若两边都不可读,返回空
## 7.3 关于“日志文件路径”
- 现在前端已调整:当存储类型包含 ClickHouse 时,创建/编辑页隐藏“日志文件路径”输入。
- 但 Fluent Bit 的 `Path` 必须匹配实际日志目录;若你改了日志目录,需要同步改 Fluent Bit 配置并重启。
---
## 8. 服务检查与常用命令
## 8.1 检查 Fluent Bit 服务名
```bash
systemctl list-unit-files | grep -Ei 'fluent|td-agent-bit'
systemctl status fluent-bit.service
```
## 8.2 查看 Fluent Bit 实际使用的配置文件
```bash
systemctl status fluent-bit.service
```
重点看 `ExecStart`,例如:
```text
/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf
```
## 8.3 验证 ClickHouse 是否有数据
```sql
SELECT count() FROM default.logs_ingest;
SELECT count() FROM default.dns_logs_ingest;
```
---
## 9. 回滚策略(最小影响)
1. 先把页面日志策略切回 MySQL-only。
2. 回滚 API/Admin 到上一版本。
3. Node/DNS 分批回滚。
4. Fluent Bit 保留运行不影响主业务(只停止 CH 写入即可)。
---
## 10. 一次发布的最简执行清单
```bash
# 1) 构建
cd /mnt/e/AI_PRODUCT/waf-platform/EdgeAdmin/build
bash build.sh linux amd64 plus
# 2) 上传产物
# EdgeAdmin/dist/*.zip
# EdgeAPI/dist/*.zip
# EdgeAPI/build/deploy/* (node/dns installer zip)
# 3) 线上先执行 CH DDL
# 4) 更新 fluent-bit 配置并重启
sudo systemctl restart fluent-bit
# 5) 升级 edge-api / edge-admin 并重启
# 6) 升级 edge-node / edge-dns
# 7) 切日志策略并验证
```

View File

@@ -139,3 +139,51 @@ flowchart LR
- 通常 1 分钟内自动刷新生效。
- 若要立即生效:重启 `edge-api`,并在需要时重启 `edge-node`、`fluent-bit`。
---
## 8. DNS 日志与 HTTP 策略联动(新增)
从当前版本开始DNS 访问日志与 HTTP 访问日志共享同一套公用策略语义(`writeTargets`
- `WriteMySQL=true`DNS 节点上报 APIAPI 写入 MySQL 分表。
- `WriteClickHouse=true`DNS 节点写本地 JSONLFluent Bit 采集写入 ClickHouse `dns_logs_ingest`。
- 双开即双写;双关即不写(仅保留内存处理,不入库)。
### 8.1 DNS 写入链路
```mermaid
flowchart LR
A[EdgeDNS 产生日志] --> B{writeTargets}
B -->|MySQL=true| C[CreateNSAccessLogs]
C --> D[(MySQL edgeNSAccessLogs_YYYYMMDD)]
B -->|ClickHouse=true| E[/var/log/edge/edge-dns/access.log]
E --> F[Fluent Bit]
F --> G[(ClickHouse dns_logs_ingest)]
```
### 8.2 DNS 查询链路
```mermaid
flowchart TD
Q[/ns/clusters/accessLogs] --> R{策略是否启用ClickHouse且CH可用}
R -->|是| CH[(dns_logs_ingest)]
R -->|否| M{策略是否启用MySQL}
CH -->|查询失败| M
M -->|是| MY[(MySQL edgeNSAccessLogs_YYYYMMDD)]
M -->|否| E[返回空列表]
```
### 8.3 组合场景说明DNS
| 策略 | 写入 | 读取 |
|------|------|------|
| 仅 MySQL | API -> MySQL | MySQL |
| 仅 ClickHouse | 本地文件 -> Fluent Bit -> ClickHouse | ClickHouse |
| MySQL + ClickHouse | API -> MySQL + 本地文件 -> Fluent Bit -> ClickHouse | 优先 ClickHouse失败回退 MySQL |
### 8.4 DNS 相关必须配置
1. `EdgeAPI` 配置 ClickHouse 连接(仅读 CH 时必须)。
2. `deploy/fluent-bit/fluent-bit.conf` 已包含 DNS 输入:`/var/log/edge/edge-dns/*.log`。
3. ClickHouse 已创建 `dns_logs_ingest` 表。
4. EdgeDNS 运行用户对 `EDGE_DNS_LOG_DIR`(默认 `/var/log/edge/edge-dns`)有写权限。