引入lumberjack和fluentbit自动分发

2026-02-13 22:36:17 +08:00
parent c6da67db79
commit e9093baffb
47 changed files with 4589 additions and 317 deletions
--- a/EdgeNode/build/build.sh
+++ b/EdgeNode/build/build.sh
@@ -181,11 +181,12 @@ function copy_fluent_bit_assets() {
 		echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
 		return 1
 	fi
+	verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1

 	rm -rf "$FLUENT_DIST"
 	mkdir -p "$FLUENT_DIST"

-	for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
+	for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
 		if [ -f "$FLUENT_ROOT/$file" ]; then
 			cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
 		fi
@@ -203,6 +204,43 @@ function copy_fluent_bit_assets() {
 		fi
 	fi

+	rm -f "$FLUENT_DIST/.gitignore"
+	rm -f "$FLUENT_DIST"/logs.db*
+	rm -rf "$FLUENT_DIST/storage"
+
+	return 0
+}
+
+function verify_fluent_bit_package_matrix() {
+	FLUENT_ROOT=$1
+	ARCH=$2
+	REQUIRED_FILES=()
+	if [ "$ARCH" = "amd64" ]; then
+		REQUIRED_FILES=(
+			"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
+			"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
+		)
+	elif [ "$ARCH" = "arm64" ]; then
+		REQUIRED_FILES=(
+			"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
+			"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
+		)
+	else
+		echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
+		return 1
+	fi
+
+	MISSING=0
+	for FILE in "${REQUIRED_FILES[@]}"; do
+		if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
+			echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
+			MISSING=1
+		fi
+	done
+
+	if [ "$MISSING" -ne 0 ]; then
+		return 1
+	fi
 	return 0
 }

--- a/EdgeNode/go.mod
+++ b/EdgeNode/go.mod
@@ -38,6 +38,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/tdewolff/minify/v2 v2.20.20
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.41
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
 	golang.org/x/image v0.15.0
 	golang.org/x/net v0.47.0
--- a/EdgeNode/go.sum
+++ b/EdgeNode/go.sum
@@ -305,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/EdgeNode/internal/accesslogs/file_writer.go
+++ b/EdgeNode/internal/accesslogs/file_writer.go
@@ -9,7 +9,9 @@ import (
 	"sync"

 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"gopkg.in/natefinch/lumberjack.v2"
 )

 var (
@@ -30,20 +32,23 @@ const (
 	envLogDir     = "EDGE_LOG_DIR"
 )

-// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 logrotate 与 Fluent Bit 采集
+// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 Fluent Bit 采集。
+// 文件轮转由 lumberjack 内建完成。
 type FileWriter struct {
-	dir    string
-	mu     sync.Mutex
-	files  map[string]*os.File // access.log, waf.log, error.log
-	inited bool
+	dir          string
+	mu           sync.Mutex
+	files        map[string]*lumberjack.Logger // access.log, waf.log, error.log
+	rotateConfig *serverconfigs.AccessLogRotateConfig
+	inited       bool
 }

 // NewFileWriter 创建本地日志文件写入器
 func NewFileWriter() *FileWriter {
 	dir := resolveDefaultLogDir()
 	return &FileWriter{
-		dir:   dir,
-		files: make(map[string]*os.File),
+		dir:          dir,
+		files:        make(map[string]*lumberjack.Logger),
+		rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
 	}
 }

@@ -97,9 +102,9 @@ func (w *FileWriter) SetDir(dir string) {
 		return
 	}

-	for name, f := range w.files {
-		if f != nil {
-			_ = f.Close()
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
 		}
 		w.files[name] = nil
 	}
@@ -107,6 +112,27 @@ func (w *FileWriter) SetDir(dir string) {
 	w.dir = dir
 }

+// SetRotateConfig 更新日志轮转配置并重建 writer。
+func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
+	normalized := config.Normalize()
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if equalRotateConfig(w.rotateConfig, normalized) {
+		return
+	}
+
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
+		}
+		w.files[name] = nil
+	}
+	w.inited = false
+	w.rotateConfig = normalized
+}
+
 // IsEnabled 是否启用落盘（目录非空即视为启用）
 func (w *FileWriter) IsEnabled() bool {
 	return w.dir != ""
@@ -138,17 +164,24 @@ func (w *FileWriter) init() error {
 		if w.files[name] != nil {
 			continue
 		}
-		fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
-		if err != nil {
-			remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
-			continue
-		}
-		w.files[name] = fp
+		w.files[name] = w.newLogger(name)
 	}
 	w.inited = true
 	return nil
 }

+func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger {
+	rotateConfig := w.rotateConfig.Normalize()
+	return &lumberjack.Logger{
+		Filename:   filepath.Join(w.dir, fileName),
+		MaxSize:    rotateConfig.MaxSizeMB,
+		MaxBackups: rotateConfig.MaxBackups,
+		MaxAge:     rotateConfig.MaxAgeDays,
+		Compress:   *rotateConfig.Compress,
+		LocalTime:  *rotateConfig.LocalTime,
+	}
+}
+
 // Write 将一条访问日志按 log_type 写入对应文件（access.log / waf.log / error.log）
 func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
 	if w.dir == "" {
@@ -173,13 +206,12 @@ func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
 		fileName = "access.log"
 	}
 	w.mu.Lock()
-	fp := w.files[fileName]
+	file := w.files[fileName]
 	w.mu.Unlock()
-	if fp == nil {
+	if file == nil {
 		return
 	}
-	// 单行写入，末尾换行，便于 Fluent Bit / JSON 解析
-	_, err = fp.Write(append(line, '\n'))
+	_, err = file.Write(append(line, '\n'))
 	if err != nil {
 		remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
 	}
@@ -194,49 +226,49 @@ func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
 		return
 	}
 	w.mu.Lock()
-	accessFp := w.files["access.log"]
-	wafFp := w.files["waf.log"]
-	errorFp := w.files["error.log"]
+	accessFile := w.files["access.log"]
+	wafFile := w.files["waf.log"]
+	errorFile := w.files["error.log"]
 	w.mu.Unlock()
-	if accessFp == nil && wafFp == nil && errorFp == nil {
+	if accessFile == nil && wafFile == nil && errorFile == nil {
 		return
 	}
-	for _, l := range logs {
-		ingest, logType := FromHTTPAccessLog(l, clusterId)
+	for _, logItem := range logs {
+		ingest, logType := FromHTTPAccessLog(logItem, clusterId)
 		line, err := json.Marshal(ingest)
 		if err != nil {
 			continue
 		}
 		line = append(line, '\n')
-		var fp *os.File
+		var file *lumberjack.Logger
 		switch logType {
 		case LogTypeWAF:
-			fp = wafFp
+			file = wafFile
 		case LogTypeError:
-			fp = errorFp
+			file = errorFile
 		default:
-			fp = accessFp
+			file = accessFile
 		}
-		if fp != nil {
-			_, _ = fp.Write(line)
+		if file != nil {
+			_, _ = file.Write(line)
 		}
 	}
 }

-// Reopen 关闭并重新打开所有日志文件（供 logrotate copytruncate 或 SIGHUP 后重开句柄）
+// Reopen 关闭并重建所有日志 writer（供 SIGHUP 兼容调用）。
 func (w *FileWriter) Reopen() error {
 	if w.dir == "" {
 		return nil
 	}
 	w.mu.Lock()
-	defer w.mu.Unlock()
-	for name, f := range w.files {
-		if f != nil {
-			_ = f.Close()
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
 			w.files[name] = nil
 		}
 	}
 	w.inited = false
+	w.mu.Unlock()
 	return w.init()
 }

@@ -245,9 +277,9 @@ func (w *FileWriter) Close() error {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	var lastErr error
-	for name, f := range w.files {
-		if f != nil {
-			if err := f.Close(); err != nil {
+	for name, file := range w.files {
+		if file != nil {
+			if err := file.Close(); err != nil {
 				lastErr = err
 				remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
 			}
@@ -257,3 +289,14 @@ func (w *FileWriter) Close() error {
 	w.inited = false
 	return lastErr
 }
+
+func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
+	if left == nil || right == nil {
+		return left == right
+	}
+	return left.MaxSizeMB == right.MaxSizeMB &&
+		left.MaxBackups == right.MaxBackups &&
+		left.MaxAgeDays == right.MaxAgeDays &&
+		*left.Compress == *right.Compress &&
+		*left.LocalTime == *right.LocalTime
+}
--- a/EdgeNode/internal/nodes/node.go
+++ b/EdgeNode/internal/nodes/node.go
@@ -577,7 +577,7 @@ func (this *Node) listenSignals() {
 	goman.New(func() {
 		for sig := range queue {
 			if sig == syscall.SIGHUP {
-				// 供 logrotate 等旋转日志后重开句柄
+				// 兼容 SIGHUP：重建本地日志 writer
 				if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
 					remotelogs.Error("NODE", "access log file reopen: "+err.Error())
 				}
@@ -890,6 +890,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) {
 	var accessLogFilePath string
 	if config != nil && config.GlobalServerConfig != nil {
 		accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
+		accesslogs.SharedFileWriter().SetRotateConfig(config.GlobalServerConfig.HTTPAccessLog.Rotate)
 	}
 	accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)