Initial commit (code only without large binaries)

EdgeNode/internal/accesslogs/file_writer.go

@@ -0,0 +1,302 @@
+package accesslogs
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"gopkg.in/natefinch/lumberjack.v2"
+)
+
+var (
+	sharedFileWriter *FileWriter
+	sharedOnce       sync.Once
+)
+
+// SharedFileWriter 返回全局本地日志文件写入器（单例）
+func SharedFileWriter() *FileWriter {
+	sharedOnce.Do(func() {
+		sharedFileWriter = NewFileWriter()
+	})
+	return sharedFileWriter
+}
+
+const (
+	defaultLogDir = "/var/log/edge/edge-node"
+	envLogDir     = "EDGE_LOG_DIR"
+)
+
+// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 Fluent Bit 采集。
+// 文件轮转由 lumberjack 内建完成。
+type FileWriter struct {
+	dir          string
+	mu           sync.Mutex
+	files        map[string]*lumberjack.Logger // access.log, waf.log, error.log
+	rotateConfig *serverconfigs.AccessLogRotateConfig
+	inited       bool
+}
+
+// NewFileWriter 创建本地日志文件写入器
+func NewFileWriter() *FileWriter {
+	dir := resolveDefaultLogDir()
+	return &FileWriter{
+		dir:          dir,
+		files:        make(map[string]*lumberjack.Logger),
+		rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
+	}
+}
+
+func resolveDefaultLogDir() string {
+	dir := strings.TrimSpace(os.Getenv(envLogDir))
+	if dir == "" {
+		return defaultLogDir
+	}
+	return dir
+}
+
+func resolveDirFromPolicyPath(policyPath string) string {
+	policyPath = strings.TrimSpace(policyPath)
+	if policyPath == "" {
+		return ""
+	}
+
+	if strings.HasSuffix(policyPath, "/") || strings.HasSuffix(policyPath, "\\") {
+		return filepath.Clean(policyPath)
+	}
+
+	baseName := filepath.Base(policyPath)
+	if strings.Contains(baseName, ".") || strings.Contains(baseName, "${") {
+		return filepath.Clean(filepath.Dir(policyPath))
+	}
+
+	return filepath.Clean(policyPath)
+}
+
+// Dir 返回当前配置的日志目录
+func (w *FileWriter) Dir() string {
+	return w.dir
+}
+
+// SetDirByPolicyPath 使用公用日志策略 path 更新目录，空值时回退到 EDGE_LOG_DIR/default。
+func (w *FileWriter) SetDirByPolicyPath(policyPath string) {
+	dir := resolveDirFromPolicyPath(policyPath)
+	w.SetDir(dir)
+}
+
+// SetDir 更新日志目录并重置文件句柄。
+func (w *FileWriter) SetDir(dir string) {
+	if strings.TrimSpace(dir) == "" {
+		dir = resolveDefaultLogDir()
+	}
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if dir == w.dir {
+		return
+	}
+
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
+		}
+		w.files[name] = nil
+	}
+	w.inited = false
+	w.dir = dir
+}
+
+// SetRotateConfig 更新日志轮转配置并重建 writer。
+func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
+	normalized := config.Normalize()
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if equalRotateConfig(w.rotateConfig, normalized) {
+		return
+	}
+
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
+		}
+		w.files[name] = nil
+	}
+	w.inited = false
+	w.rotateConfig = normalized
+}
+
+// IsEnabled 是否启用落盘（目录非空即视为启用）
+func (w *FileWriter) IsEnabled() bool {
+	return w.dir != ""
+}
+
+// EnsureInit 在启动时预创建日志目录和空文件，便于 Fluent Bit 立即 tail，无需等首条访问日志
+func (w *FileWriter) EnsureInit() error {
+	if w.dir == "" {
+		return nil
+	}
+	return w.init()
+}
+
+// init 确保目录存在并打开三个日志文件（仅首次或 Reopen 时）
+func (w *FileWriter) init() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.inited && len(w.files) > 0 {
+		return nil
+	}
+	if w.dir == "" {
+		return nil
+	}
+	if err := os.MkdirAll(w.dir, 0755); err != nil {
+		remotelogs.Error("ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error())
+		return err
+	}
+	for _, name := range []string{"access.log", "waf.log", "error.log"} {
+		if w.files[name] != nil {
+			continue
+		}
+		w.files[name] = w.newLogger(name)
+	}
+	w.inited = true
+	return nil
+}
+
+func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger {
+	rotateConfig := w.rotateConfig.Normalize()
+	return &lumberjack.Logger{
+		Filename:   filepath.Join(w.dir, fileName),
+		MaxSize:    rotateConfig.MaxSizeMB,
+		MaxBackups: rotateConfig.MaxBackups,
+		MaxAge:     rotateConfig.MaxAgeDays,
+		Compress:   *rotateConfig.Compress,
+		LocalTime:  *rotateConfig.LocalTime,
+	}
+}
+
+// Write 将一条访问日志按 log_type 写入对应文件（access.log / waf.log / error.log）
+func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
+	if w.dir == "" {
+		return
+	}
+	if err := w.init(); err != nil || len(w.files) == 0 {
+		return
+	}
+	ingest, logType := FromHTTPAccessLog(l, clusterId)
+	line, err := json.Marshal(ingest)
+	if err != nil {
+		remotelogs.Error("ACCESS_LOG_FILE", "marshal ingest log: "+err.Error())
+		return
+	}
+	var fileName string
+	switch logType {
+	case LogTypeWAF:
+		fileName = "waf.log"
+	case LogTypeError:
+		fileName = "error.log"
+	default:
+		fileName = "access.log"
+	}
+	w.mu.Lock()
+	file := w.files[fileName]
+	w.mu.Unlock()
+	if file == nil {
+		return
+	}
+	_, err = file.Write(append(line, '\n'))
+	if err != nil {
+		remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
+	}
+}
+
+// WriteBatch 批量写入，减少锁竞争
+func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
+	if w.dir == "" || len(logs) == 0 {
+		return
+	}
+	if err := w.init(); err != nil || len(w.files) == 0 {
+		return
+	}
+	w.mu.Lock()
+	accessFile := w.files["access.log"]
+	wafFile := w.files["waf.log"]
+	errorFile := w.files["error.log"]
+	w.mu.Unlock()
+	if accessFile == nil && wafFile == nil && errorFile == nil {
+		return
+	}
+	for _, logItem := range logs {
+		ingest, logType := FromHTTPAccessLog(logItem, clusterId)
+		line, err := json.Marshal(ingest)
+		if err != nil {
+			continue
+		}
+		line = append(line, '\n')
+		var file *lumberjack.Logger
+		switch logType {
+		case LogTypeWAF:
+			file = wafFile
+		case LogTypeError:
+			file = errorFile
+		default:
+			file = accessFile
+		}
+		if file != nil {
+			_, _ = file.Write(line)
+		}
+	}
+}
+
+// Reopen 关闭并重建所有日志 writer（供 SIGHUP 兼容调用）。
+func (w *FileWriter) Reopen() error {
+	if w.dir == "" {
+		return nil
+	}
+	w.mu.Lock()
+	for name, file := range w.files {
+		if file != nil {
+			_ = file.Close()
+			w.files[name] = nil
+		}
+	}
+	w.inited = false
+	w.mu.Unlock()
+	return w.init()
+}
+
+// Close 关闭所有已打开的文件
+func (w *FileWriter) Close() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	var lastErr error
+	for name, file := range w.files {
+		if file != nil {
+			if err := file.Close(); err != nil {
+				lastErr = err
+				remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
+			}
+			w.files[name] = nil
+		}
+	}
+	w.inited = false
+	return lastErr
+}
+
+func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
+	if left == nil || right == nil {
+		return left == right
+	}
+	return left.MaxSizeMB == right.MaxSizeMB &&
+		left.MaxBackups == right.MaxBackups &&
+		left.MaxAgeDays == right.MaxAgeDays &&
+		*left.Compress == *right.Compress &&
+		*left.LocalTime == *right.LocalTime
+}

EdgeNode/internal/accesslogs/ingest_log.go

@@ -0,0 +1,137 @@
+// Package accesslogs 提供边缘节点访问日志落盘（JSON Lines），供 Fluent Bit 采集写入 ClickHouse。
+package accesslogs
+
+import (
+	"encoding/json"
+
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+)
+
+// LogType 与 Fluent Bit / logs_ingest 的 log_type 一致
+const (
+	LogTypeAccess = "access"
+	LogTypeWAF    = "waf"
+	LogTypeError  = "error"
+)
+
+// 请求/响应 body 落盘最大长度（字节），超出截断，避免单条过大
+const maxBodyLen = 512 * 1024
+
+// IngestLog 单行 JSON 结构与方案文档、ClickHouse logs_ingest 表字段对齐
+type IngestLog struct {
+	Timestamp           int64  `json:"timestamp"`
+	NodeId              int64  `json:"node_id"`
+	ClusterId           int64  `json:"cluster_id"`
+	ServerId            int64  `json:"server_id"`
+	Host                string `json:"host"`
+	IP                  string `json:"ip"`
+	Method              string `json:"method"`
+	Path                string `json:"path"`
+	Status              int32  `json:"status"`
+	BytesIn             int64  `json:"bytes_in"`
+	BytesOut            int64  `json:"bytes_out"`
+	CostMs              int64  `json:"cost_ms"`
+	UA                  string `json:"ua"`
+	Referer             string `json:"referer"`
+	LogType             string `json:"log_type"`
+	TraceId             string `json:"trace_id,omitempty"`
+	FirewallPolicyId    int64  `json:"firewall_policy_id,omitempty"`
+	FirewallRuleGroupId int64  `json:"firewall_rule_group_id,omitempty"`
+	FirewallRuleSetId   int64  `json:"firewall_rule_set_id,omitempty"`
+	FirewallRuleId      int64  `json:"firewall_rule_id,omitempty"`
+	RequestHeaders      string `json:"request_headers,omitempty"`
+	RequestBody         string `json:"request_body,omitempty"`
+	ResponseHeaders     string `json:"response_headers,omitempty"`
+	ResponseBody        string `json:"response_body,omitempty"`
+}
+
+// stringsMapToJSON 将 map[string]*Strings 转为 JSON 字符串，便于落盘与 ClickHouse 存储
+func stringsMapToJSON(m map[string]*pb.Strings) string {
+	if len(m) == 0 {
+		return ""
+	}
+	out := make(map[string]string, len(m))
+	for k, v := range m {
+		if v != nil && len(v.Values) > 0 {
+			out[k] = v.Values[0]
+		}
+	}
+	if len(out) == 0 {
+		return ""
+	}
+	b, _ := json.Marshal(out)
+	return string(b)
+}
+
+// truncateBody 截断 body 到最大长度，避免单条过大
+func truncateBody(b []byte) string {
+	if len(b) == 0 {
+		return ""
+	}
+	s := string(b)
+	if len(s) > maxBodyLen {
+		return s[:maxBodyLen]
+	}
+	return s
+}
+
+// buildRequestBody 将查询串与请求体合并写入 request_body 字段（不新增字段）
+func buildRequestBody(l *pb.HTTPAccessLog) string {
+	q := l.GetQueryString()
+	body := l.GetRequestBody()
+	if len(q) == 0 && len(body) == 0 {
+		return ""
+	}
+	if len(body) == 0 {
+		return truncateBody([]byte(q))
+	}
+	combined := make([]byte, 0, len(q)+1+len(body))
+	combined = append(combined, q...)
+	combined = append(combined, '\n')
+	combined = append(combined, body...)
+	return truncateBody(combined)
+}
+
+// FromHTTPAccessLog 从 pb.HTTPAccessLog 转为 IngestLog，并决定 log_type
+func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog, logType string) {
+	ingest = IngestLog{
+		Timestamp:           l.GetTimestamp(),
+		NodeId:              l.GetNodeId(),
+		ClusterId:           clusterId,
+		ServerId:            l.GetServerId(),
+		Host:                l.GetHost(),
+		IP:                  l.GetRawRemoteAddr(),
+		Method:              l.GetRequestMethod(),
+		Path:                l.GetRequestURI(), // 使用 RequestURI 以包含查询参数
+		Status:              l.GetStatus(),
+		BytesIn:             l.GetRequestLength(),
+		BytesOut:            l.GetBytesSent(),
+		CostMs:              int64(l.GetRequestTime() * 1000),
+		UA:                  l.GetUserAgent(),
+		Referer:             l.GetReferer(),
+		TraceId:             l.GetRequestId(),
+		FirewallPolicyId:    l.GetFirewallPolicyId(),
+		FirewallRuleGroupId: l.GetFirewallRuleGroupId(),
+		FirewallRuleSetId:   l.GetFirewallRuleSetId(),
+		FirewallRuleId:      l.GetFirewallRuleId(),
+		RequestHeaders:      stringsMapToJSON(l.GetHeader()),
+		RequestBody:         buildRequestBody(l),
+		ResponseHeaders:     stringsMapToJSON(l.GetSentHeader()),
+	}
+	if ingest.IP == "" {
+		ingest.IP = l.GetRemoteAddr()
+	}
+	// 响应 body 当前 pb 未提供，若后续扩展可在此赋值
+	// ingest.ResponseBody = ...
+
+	// 与方案一致：waf > error > access；攻击日志通过 firewall_rule_id / firewall_policy_id 判断
+	if l.GetFirewallPolicyId() > 0 {
+		logType = LogTypeWAF
+	} else if len(l.GetErrors()) > 0 {
+		logType = LogTypeError
+	} else {
+		logType = LogTypeAccess
+	}
+	ingest.LogType = logType
+	return ingest, logType
+}