引入lumberjack和fluentbit自动分发

2026-02-13 22:36:17 +08:00
parent c6da67db79
commit e9093baffb
47 changed files with 4589 additions and 317 deletions
--- a/EdgeDNS/build/build.sh
+++ b/EdgeDNS/build/build.sh
@@ -107,11 +107,12 @@ function copy_fluent_bit_assets() {
 		echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
 		return 1
 	fi
+	verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1

 	rm -rf "$FLUENT_DIST"
 	mkdir -p "$FLUENT_DIST"

-	for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
+	for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
 		if [ -f "$FLUENT_ROOT/$file" ]; then
 			cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
 		fi
@@ -129,6 +130,43 @@ function copy_fluent_bit_assets() {
 		fi
 	fi

+	rm -f "$FLUENT_DIST/.gitignore"
+	rm -f "$FLUENT_DIST"/logs.db*
+	rm -rf "$FLUENT_DIST/storage"
+
+	return 0
+}
+
+function verify_fluent_bit_package_matrix() {
+	FLUENT_ROOT=$1
+	ARCH=$2
+	REQUIRED_FILES=()
+	if [ "$ARCH" = "amd64" ]; then
+		REQUIRED_FILES=(
+			"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
+			"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
+		)
+	elif [ "$ARCH" = "arm64" ]; then
+		REQUIRED_FILES=(
+			"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
+			"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
+		)
+	else
+		echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
+		return 1
+	fi
+
+	MISSING=0
+	for FILE in "${REQUIRED_FILES[@]}"; do
+		if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
+			echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
+			MISSING=1
+		fi
+	done
+
+	if [ "$MISSING" -ne 0 ]; then
+		return 1
+	fi
 	return 0
 }

--- a/EdgeDNS/go.mod
+++ b/EdgeDNS/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/mdlayher/netlink v1.7.2
 	github.com/miekg/dns v1.1.58
 	github.com/shirou/gopsutil/v3 v3.24.2
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	golang.org/x/sys v0.38.0
 	google.golang.org/grpc v1.78.0
 	gopkg.in/yaml.v3 v3.0.1
--- a/EdgeDNS/go.sum
+++ b/EdgeDNS/go.sum
@@ -111,6 +111,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/EdgeDNS/internal/accesslogs/dns_file_writer.go
+++ b/EdgeDNS/internal/accesslogs/dns_file_writer.go
@@ -9,7 +9,9 @@ import (
 	"sync"

 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
+	"gopkg.in/natefinch/lumberjack.v2"
 )

 const (
@@ -30,18 +32,23 @@ func SharedDNSFileWriter() *DNSFileWriter {
 	return sharedDNSFileWriter
 }

-// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集.
+// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集。
+// 文件轮转由 lumberjack 内建完成。
 type DNSFileWriter struct {
-	dir    string
-	mu     sync.Mutex
-	file   *os.File
-	inited bool
+	dir          string
+	mu           sync.Mutex
+	file         *lumberjack.Logger
+	rotateConfig *serverconfigs.AccessLogRotateConfig
+	inited       bool
 }

 // NewDNSFileWriter 创建 DNS 本地日志写入器.
 func NewDNSFileWriter() *DNSFileWriter {
 	logDir := resolveDefaultDNSLogDir()
-	return &DNSFileWriter{dir: logDir}
+	return &DNSFileWriter{
+		dir:          logDir,
+		rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
+	}
 }

 func resolveDefaultDNSLogDir() string {
@@ -102,6 +109,25 @@ func (w *DNSFileWriter) SetDir(dir string) {
 	w.dir = dir
 }

+// SetRotateConfig 更新日志轮转配置并重建 writer。
+func (w *DNSFileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
+	normalized := config.Normalize()
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if equalDNSRotateConfig(w.rotateConfig, normalized) {
+		return
+	}
+
+	if w.file != nil {
+		_ = w.file.Close()
+		w.file = nil
+	}
+	w.inited = false
+	w.rotateConfig = normalized
+}
+
 // EnsureInit 在启动时预创建目录与 access.log.
 func (w *DNSFileWriter) EnsureInit() error {
 	if w.dir == "" {
@@ -127,13 +153,16 @@ func (w *DNSFileWriter) init() error {
 		return err
 	}

-	fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
-	if err != nil {
-		remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error())
-		return err
+	rotateConfig := w.rotateConfig.Normalize()
+	w.file = &lumberjack.Logger{
+		Filename:   filepath.Join(w.dir, "access.log"),
+		MaxSize:    rotateConfig.MaxSizeMB,
+		MaxBackups: rotateConfig.MaxBackups,
+		MaxAge:     rotateConfig.MaxAgeDays,
+		Compress:   *rotateConfig.Compress,
+		LocalTime:  *rotateConfig.LocalTime,
 	}

-	w.file = fp
 	w.inited = true
 	return nil
 }
@@ -148,14 +177,14 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
 	}

 	w.mu.Lock()
-	fp := w.file
+	file := w.file
 	w.mu.Unlock()
-	if fp == nil {
+	if file == nil {
 		return
 	}

-	for _, log := range logs {
-		ingestLog := FromNSAccessLog(log, clusterId)
+	for _, logItem := range logs {
+		ingestLog := FromNSAccessLog(logItem, clusterId)
 		if ingestLog == nil {
 			continue
 		}
@@ -163,11 +192,11 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
 		if err != nil {
 			continue
 		}
-		_, _ = fp.Write(append(line, '\n'))
+		_, _ = file.Write(append(line, '\n'))
 	}
 }

-// Reopen 关闭并重新打开日志文件（配合 logrotate）.
+// Reopen 关闭并重建日志 writer（供兼容调用）.
 func (w *DNSFileWriter) Reopen() error {
 	w.mu.Lock()
 	if w.file != nil {
@@ -197,3 +226,14 @@ func (w *DNSFileWriter) Close() error {
 	}
 	return nil
 }
+
+func equalDNSRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
+	if left == nil || right == nil {
+		return left == right
+	}
+	return left.MaxSizeMB == right.MaxSizeMB &&
+		left.MaxBackups == right.MaxBackups &&
+		left.MaxAgeDays == right.MaxAgeDays &&
+		*left.Compress == *right.Compress &&
+		*left.LocalTime == *right.LocalTime
+}
--- a/EdgeDNS/internal/nodes/manager_node_config.go
+++ b/EdgeDNS/internal/nodes/manager_node_config.go
@@ -110,6 +110,7 @@ func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) {
 	teaconst.IsPlus = config.IsPlus

 	accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
+	accesslogs.SharedDNSFileWriter().SetRotateConfig(config.AccessLogRotate)

 	needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
 	if needWriteFile {