换成单集群模式

EdgeHttpDNS/internal/accesslogs/httpdns_file_writer.go

@@ -0,0 +1,172 @@
+package accesslogs
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+const (
+	defaultHTTPDNSLogDir = "/var/log/edge/edge-httpdns"
+	envHTTPDNSLogDir     = "EDGE_HTTPDNS_LOG_DIR"
+)
+
+var (
+	sharedHTTPDNSFileWriter     *HTTPDNSFileWriter
+	sharedHTTPDNSFileWriterOnce sync.Once
+)
+
+// SharedHTTPDNSFileWriter 返回 HTTPDNS 本地日志写入器（单例）.
+func SharedHTTPDNSFileWriter() *HTTPDNSFileWriter {
+	sharedHTTPDNSFileWriterOnce.Do(func() {
+		sharedHTTPDNSFileWriter = NewHTTPDNSFileWriter()
+	})
+	return sharedHTTPDNSFileWriter
+}
+
+// HTTPDNSFileWriter 将 HTTPDNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集。
+type HTTPDNSFileWriter struct {
+	dir          string
+	mu           sync.Mutex
+	file         *lumberjack.Logger
+	rotateConfig *serverconfigs.AccessLogRotateConfig
+	inited       bool
+}
+
+// NewHTTPDNSFileWriter 创建 HTTPDNS 本地日志写入器.
+func NewHTTPDNSFileWriter() *HTTPDNSFileWriter {
+	logDir := resolveDefaultHTTPDNSLogDir()
+	return &HTTPDNSFileWriter{
+		dir:          logDir,
+		rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
+	}
+}
+
+func resolveDefaultHTTPDNSLogDir() string {
+	logDir := strings.TrimSpace(os.Getenv(envHTTPDNSLogDir))
+	if logDir == "" {
+		return defaultHTTPDNSLogDir
+	}
+	return logDir
+}
+
+// SetDir 更新日志目录并重置文件句柄。
+func (w *HTTPDNSFileWriter) SetDir(dir string) {
+	if strings.TrimSpace(dir) == "" {
+		dir = resolveDefaultHTTPDNSLogDir()
+	}
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if dir == w.dir {
+		return
+	}
+
+	if w.file != nil {
+		_ = w.file.Close()
+		w.file = nil
+	}
+	w.inited = false
+	w.dir = dir
+}
+
+// Dir 返回当前日志目录.
+func (w *HTTPDNSFileWriter) Dir() string {
+	return w.dir
+}
+
+// EnsureInit 在启动时预创建目录与 access.log.
+func (w *HTTPDNSFileWriter) EnsureInit() error {
+	if w.dir == "" {
+		return nil
+	}
+	return w.init()
+}
+
+func (w *HTTPDNSFileWriter) init() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.inited && w.file != nil {
+		return nil
+	}
+
+	if w.dir == "" {
+		return nil
+	}
+
+	if err := os.MkdirAll(w.dir, 0755); err != nil {
+		log.Println("[HTTPDNS_ACCESS_LOG]mkdir log dir failed:", err.Error())
+		return err
+	}
+
+	rotateConfig := w.rotateConfig.Normalize()
+	w.file = &lumberjack.Logger{
+		Filename:   filepath.Join(w.dir, "access.log"),
+		MaxSize:    rotateConfig.MaxSizeMB,
+		MaxBackups: rotateConfig.MaxBackups,
+		MaxAge:     rotateConfig.MaxAgeDays,
+		Compress:   *rotateConfig.Compress,
+		LocalTime:  *rotateConfig.LocalTime,
+	}
+
+	w.inited = true
+	return nil
+}
+
+// WriteBatch 批量写入 HTTPDNS 访问日志.
+func (w *HTTPDNSFileWriter) WriteBatch(logs []*pb.HTTPDNSAccessLog) {
+	if len(logs) == 0 || w.dir == "" {
+		return
+	}
+	if err := w.init(); err != nil {
+		return
+	}
+
+	w.mu.Lock()
+	file := w.file
+	w.mu.Unlock()
+	if file == nil {
+		return
+	}
+
+	for _, logItem := range logs {
+		ingestLog := FromPBAccessLog(logItem)
+		if ingestLog == nil {
+			continue
+		}
+		line, err := json.Marshal(ingestLog)
+		if err != nil {
+			continue
+		}
+		_, _ = file.Write(append(line, '\n'))
+	}
+}
+
+// Close 关闭日志文件.
+func (w *HTTPDNSFileWriter) Close() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.file == nil {
+		return nil
+	}
+
+	err := w.file.Close()
+	w.file = nil
+	w.inited = false
+	if err != nil {
+		log.Println(fmt.Sprintf("[HTTPDNS_ACCESS_LOG]close access.log failed: %v", err))
+		return err
+	}
+	return nil
+}

EdgeHttpDNS/internal/accesslogs/httpdns_ingest_log.go

@@ -0,0 +1,57 @@
+package accesslogs
+
+import "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+
+// HTTPDNSIngestLog HTTPDNS 访问日志单行结构（JSONEachRow），字段与
+// ClickHouse httpdns_access_logs_ingest 表一一对应。
+type HTTPDNSIngestLog struct {
+	Timestamp    int64  `json:"timestamp"`
+	RequestId    string `json:"request_id"`
+	ClusterId    int64  `json:"cluster_id"`
+	NodeId       int64  `json:"node_id"`
+	AppId        string `json:"app_id"`
+	AppName      string `json:"app_name"`
+	Domain       string `json:"domain"`
+	QType        string `json:"qtype"`
+	ClientIP     string `json:"client_ip"`
+	ClientRegion string `json:"client_region"`
+	Carrier      string `json:"carrier"`
+	SDKVersion   string `json:"sdk_version"`
+	OS           string `json:"os"`
+	ResultIPs    string `json:"result_ips"`
+	Status       string `json:"status"`
+	ErrorCode    string `json:"error_code"`
+	CostMs       int32  `json:"cost_ms"`
+	CreatedAt    int64  `json:"created_at"`
+	Day          string `json:"day"`
+	Summary      string `json:"summary"`
+}
+
+// FromPBAccessLog 将 pb.HTTPDNSAccessLog 转为本地 ingest 结构。
+func FromPBAccessLog(log *pb.HTTPDNSAccessLog) *HTTPDNSIngestLog {
+	if log == nil {
+		return nil
+	}
+	return &HTTPDNSIngestLog{
+		Timestamp:    log.GetCreatedAt(),
+		RequestId:    log.GetRequestId(),
+		ClusterId:    log.GetClusterId(),
+		NodeId:       log.GetNodeId(),
+		AppId:        log.GetAppId(),
+		AppName:      log.GetAppName(),
+		Domain:       log.GetDomain(),
+		QType:        log.GetQtype(),
+		ClientIP:     log.GetClientIP(),
+		ClientRegion: log.GetClientRegion(),
+		Carrier:      log.GetCarrier(),
+		SDKVersion:   log.GetSdkVersion(),
+		OS:           log.GetOs(),
+		ResultIPs:    log.GetResultIPs(),
+		Status:       log.GetStatus(),
+		ErrorCode:    log.GetErrorCode(),
+		CostMs:       log.GetCostMs(),
+		CreatedAt:    log.GetCreatedAt(),
+		Day:          log.GetDay(),
+		Summary:      log.GetSummary(),
+	}
+}