主分支代码
This commit is contained in:
robin
206
EdgeNode/internal/accesslogs/file_writer.go
Normal file
206
EdgeNode/internal/accesslogs/file_writer.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package accesslogs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
)
|
||||
|
||||
var (
|
||||
sharedFileWriter *FileWriter
|
||||
sharedOnce sync.Once
|
||||
)
|
||||
|
||||
// SharedFileWriter 返回全局本地日志文件写入器（单例）
|
||||
func SharedFileWriter() *FileWriter {
|
||||
sharedOnce.Do(func() {
|
||||
sharedFileWriter = NewFileWriter()
|
||||
})
|
||||
return sharedFileWriter
|
||||
}
|
||||
|
||||
const (
|
||||
defaultLogDir = "/var/log/edge/edge-node"
|
||||
envLogDir = "EDGE_LOG_DIR"
|
||||
)
|
||||
|
||||
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 logrotate 与 Fluent Bit 采集
|
||||
type FileWriter struct {
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
files map[string]*os.File // access.log, waf.log, error.log
|
||||
inited bool
|
||||
}
|
||||
|
||||
// NewFileWriter 创建本地日志文件写入器
|
||||
func NewFileWriter() *FileWriter {
|
||||
dir := os.Getenv(envLogDir)
|
||||
if dir == "" {
|
||||
dir = defaultLogDir
|
||||
}
|
||||
return &FileWriter{
|
||||
dir: dir,
|
||||
files: make(map[string]*os.File),
|
||||
}
|
||||
}
|
||||
|
||||
// Dir 返回当前配置的日志目录
|
||||
func (w *FileWriter) Dir() string {
|
||||
return w.dir
|
||||
}
|
||||
|
||||
// IsEnabled 是否启用落盘（目录非空即视为启用）
|
||||
func (w *FileWriter) IsEnabled() bool {
|
||||
return w.dir != ""
|
||||
}
|
||||
|
||||
// EnsureInit 在启动时预创建日志目录和空文件，便于 Fluent Bit 立即 tail，无需等首条访问日志
|
||||
func (w *FileWriter) EnsureInit() error {
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
return w.init()
|
||||
}
|
||||
|
||||
// init 确保目录存在并打开三个日志文件（仅首次或 Reopen 时）
|
||||
func (w *FileWriter) init() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.inited && len(w.files) > 0 {
|
||||
return nil
|
||||
}
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(w.dir, 0755); err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error())
|
||||
return err
|
||||
}
|
||||
for _, name := range []string{"access.log", "waf.log", "error.log"} {
|
||||
if w.files[name] != nil {
|
||||
continue
|
||||
}
|
||||
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
w.files[name] = fp
|
||||
}
|
||||
w.inited = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write 将一条访问日志按 log_type 写入对应文件（access.log / waf.log / error.log）
|
||||
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
||||
if w.dir == "" {
|
||||
return
|
||||
}
|
||||
if err := w.init(); err != nil || len(w.files) == 0 {
|
||||
return
|
||||
}
|
||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
||||
line, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "marshal ingest log: "+err.Error())
|
||||
return
|
||||
}
|
||||
var fileName string
|
||||
switch logType {
|
||||
case LogTypeWAF:
|
||||
fileName = "waf.log"
|
||||
case LogTypeError:
|
||||
fileName = "error.log"
|
||||
default:
|
||||
fileName = "access.log"
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp := w.files[fileName]
|
||||
w.mu.Unlock()
|
||||
if fp == nil {
|
||||
return
|
||||
}
|
||||
// 单行写入，末尾换行，便于 Fluent Bit / JSON 解析
|
||||
_, err = fp.Write(append(line, '\n'))
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// WriteBatch 批量写入，减少锁竞争
|
||||
func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
|
||||
if w.dir == "" || len(logs) == 0 {
|
||||
return
|
||||
}
|
||||
if err := w.init(); err != nil || len(w.files) == 0 {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
accessFp := w.files["access.log"]
|
||||
wafFp := w.files["waf.log"]
|
||||
errorFp := w.files["error.log"]
|
||||
w.mu.Unlock()
|
||||
if accessFp == nil && wafFp == nil && errorFp == nil {
|
||||
return
|
||||
}
|
||||
for _, l := range logs {
|
||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
||||
line, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
line = append(line, '\n')
|
||||
var fp *os.File
|
||||
switch logType {
|
||||
case LogTypeWAF:
|
||||
fp = wafFp
|
||||
case LogTypeError:
|
||||
fp = errorFp
|
||||
default:
|
||||
fp = accessFp
|
||||
}
|
||||
if fp != nil {
|
||||
_, _ = fp.Write(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen 关闭并重新打开所有日志文件（供 logrotate copytruncate 或 SIGHUP 后重开句柄）
|
||||
func (w *FileWriter) Reopen() error {
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
w.files[name] = nil
|
||||
}
|
||||
}
|
||||
w.inited = false
|
||||
return w.init()
|
||||
}
|
||||
|
||||
// Close 关闭所有已打开的文件
|
||||
func (w *FileWriter) Close() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var lastErr error
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
if err := f.Close(); err != nil {
|
||||
lastErr = err
|
||||
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
|
||||
}
|
||||
w.files[name] = nil
|
||||
}
|
||||
}
|
||||
w.inited = false
|
||||
return lastErr
|
||||
}
|
||||
137
EdgeNode/internal/accesslogs/ingest_log.go
Normal file
137
EdgeNode/internal/accesslogs/ingest_log.go
Normal file
@@ -0,0 +1,137 @@
|
||||
// Package accesslogs 提供边缘节点访问日志落盘（JSON Lines），供 Fluent Bit 采集写入 ClickHouse。
|
||||
package accesslogs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
|
||||
// LogType 与 Fluent Bit / logs_ingest 的 log_type 一致
|
||||
const (
|
||||
LogTypeAccess = "access"
|
||||
LogTypeWAF = "waf"
|
||||
LogTypeError = "error"
|
||||
)
|
||||
|
||||
// 请求/响应 body 落盘最大长度（字节），超出截断，避免单条过大
|
||||
const maxBodyLen = 512 * 1024
|
||||
|
||||
// IngestLog 单行 JSON 结构与方案文档、ClickHouse logs_ingest 表字段对齐
|
||||
type IngestLog struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
NodeId int64 `json:"node_id"`
|
||||
ClusterId int64 `json:"cluster_id"`
|
||||
ServerId int64 `json:"server_id"`
|
||||
Host string `json:"host"`
|
||||
IP string `json:"ip"`
|
||||
Method string `json:"method"`
|
||||
Path string `json:"path"`
|
||||
Status int32 `json:"status"`
|
||||
BytesIn int64 `json:"bytes_in"`
|
||||
BytesOut int64 `json:"bytes_out"`
|
||||
CostMs int64 `json:"cost_ms"`
|
||||
UA string `json:"ua"`
|
||||
Referer string `json:"referer"`
|
||||
LogType string `json:"log_type"`
|
||||
TraceId string `json:"trace_id,omitempty"`
|
||||
FirewallPolicyId int64 `json:"firewall_policy_id,omitempty"`
|
||||
FirewallRuleGroupId int64 `json:"firewall_rule_group_id,omitempty"`
|
||||
FirewallRuleSetId int64 `json:"firewall_rule_set_id,omitempty"`
|
||||
FirewallRuleId int64 `json:"firewall_rule_id,omitempty"`
|
||||
RequestHeaders string `json:"request_headers,omitempty"`
|
||||
RequestBody string `json:"request_body,omitempty"`
|
||||
ResponseHeaders string `json:"response_headers,omitempty"`
|
||||
ResponseBody string `json:"response_body,omitempty"`
|
||||
}
|
||||
|
||||
// stringsMapToJSON 将 map[string]*Strings 转为 JSON 字符串，便于落盘与 ClickHouse 存储
|
||||
func stringsMapToJSON(m map[string]*pb.Strings) string {
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := make(map[string]string, len(m))
|
||||
for k, v := range m {
|
||||
if v != nil && len(v.Values) > 0 {
|
||||
out[k] = v.Values[0]
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// truncateBody 截断 body 到最大长度，避免单条过大
|
||||
func truncateBody(b []byte) string {
|
||||
if len(b) == 0 {
|
||||
return ""
|
||||
}
|
||||
s := string(b)
|
||||
if len(s) > maxBodyLen {
|
||||
return s[:maxBodyLen]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// buildRequestBody 将查询串与请求体合并写入 request_body 字段（不新增字段）
|
||||
func buildRequestBody(l *pb.HTTPAccessLog) string {
|
||||
q := l.GetQueryString()
|
||||
body := l.GetRequestBody()
|
||||
if len(q) == 0 && len(body) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return truncateBody([]byte(q))
|
||||
}
|
||||
combined := make([]byte, 0, len(q)+1+len(body))
|
||||
combined = append(combined, q...)
|
||||
combined = append(combined, '\n')
|
||||
combined = append(combined, body...)
|
||||
return truncateBody(combined)
|
||||
}
|
||||
|
||||
// FromHTTPAccessLog 从 pb.HTTPAccessLog 转为 IngestLog，并决定 log_type
|
||||
func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog, logType string) {
|
||||
ingest = IngestLog{
|
||||
Timestamp: l.GetTimestamp(),
|
||||
NodeId: l.GetNodeId(),
|
||||
ClusterId: clusterId,
|
||||
ServerId: l.GetServerId(),
|
||||
Host: l.GetHost(),
|
||||
IP: l.GetRawRemoteAddr(),
|
||||
Method: l.GetRequestMethod(),
|
||||
Path: l.GetRequestPath(),
|
||||
Status: l.GetStatus(),
|
||||
BytesIn: l.GetRequestLength(),
|
||||
BytesOut: l.GetBytesSent(),
|
||||
CostMs: int64(l.GetRequestTime() * 1000),
|
||||
UA: l.GetUserAgent(),
|
||||
Referer: l.GetReferer(),
|
||||
TraceId: l.GetRequestId(),
|
||||
FirewallPolicyId: l.GetFirewallPolicyId(),
|
||||
FirewallRuleGroupId: l.GetFirewallRuleGroupId(),
|
||||
FirewallRuleSetId: l.GetFirewallRuleSetId(),
|
||||
FirewallRuleId: l.GetFirewallRuleId(),
|
||||
RequestHeaders: stringsMapToJSON(l.GetHeader()),
|
||||
RequestBody: buildRequestBody(l),
|
||||
ResponseHeaders: stringsMapToJSON(l.GetSentHeader()),
|
||||
}
|
||||
if ingest.IP == "" {
|
||||
ingest.IP = l.GetRemoteAddr()
|
||||
}
|
||||
// 响应 body 当前 pb 未提供，若后续扩展可在此赋值
|
||||
// ingest.ResponseBody = ...
|
||||
|
||||
// 与方案一致：waf > error > access；攻击日志通过 firewall_rule_id / firewall_policy_id 判断
|
||||
if l.GetFirewallPolicyId() > 0 {
|
||||
logType = LogTypeWAF
|
||||
} else if len(l.GetErrors()) > 0 {
|
||||
logType = LogTypeError
|
||||
} else {
|
||||
logType = LogTypeAccess
|
||||
}
|
||||
ingest.LogType = logType
|
||||
return ingest, logType
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5.2" //1.3.8.2
|
||||
Version = "1.4.6" //1.3.8.2
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -3,6 +3,8 @@ package nodes
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
@@ -92,6 +94,22 @@ Loop:
|
||||
return nil
|
||||
}
|
||||
|
||||
var writeTargets *serverconfigs.AccessLogWriteTargets
|
||||
if sharedNodeConfig != nil && sharedNodeConfig.GlobalServerConfig != nil {
|
||||
writeTargets = sharedNodeConfig.GlobalServerConfig.HTTPAccessLog.WriteTargets
|
||||
}
|
||||
needWriteFile := writeTargets == nil || writeTargets.NeedWriteFile()
|
||||
needReportAPI := writeTargets == nil || writeTargets.NeedReportToAPI()
|
||||
|
||||
// 落盘 JSON Lines（Fluent Bit 采集 → ClickHouse）
|
||||
if needWriteFile {
|
||||
var clusterId int64
|
||||
if sharedNodeConfig != nil {
|
||||
clusterId = sharedNodeConfig.GroupId
|
||||
}
|
||||
accesslogs.SharedFileWriter().WriteBatch(accessLogs, clusterId)
|
||||
}
|
||||
|
||||
// 发送到本地
|
||||
if sharedHTTPAccessLogViewer.HasConns() {
|
||||
for _, accessLog := range accessLogs {
|
||||
@@ -99,7 +117,10 @@ Loop:
|
||||
}
|
||||
}
|
||||
|
||||
// 发送到API
|
||||
// 发送到 API（写 MySQL 或 API 直写 ClickHouse 时需要）
|
||||
if !needReportAPI {
|
||||
return nil
|
||||
}
|
||||
if this.rpcClient == nil {
|
||||
client, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
@@ -422,6 +423,9 @@ func (this *Node) syncConfig(taskVersion int64) error {
|
||||
|
||||
this.isLoaded = true
|
||||
|
||||
// 预创建本地日志目录与空文件，便于 Fluent Bit 立即 tail，无需等首条访问日志
|
||||
_ = accesslogs.SharedFileWriter().EnsureInit()
|
||||
|
||||
// 整体更新不需要再更新单个服务
|
||||
this.updatingServerMap = map[int64]*serverconfigs.ServerConfig{}
|
||||
|
||||
@@ -569,9 +573,16 @@ func (this *Node) checkClusterConfig() error {
|
||||
// 监听一些信号
|
||||
func (this *Node) listenSignals() {
|
||||
var queue = make(chan os.Signal, 8)
|
||||
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT)
|
||||
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
goman.New(func() {
|
||||
for range queue {
|
||||
for sig := range queue {
|
||||
if sig == syscall.SIGHUP {
|
||||
// 供 logrotate 等旋转日志后重开句柄
|
||||
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
|
||||
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
utils.Exit()
|
||||
return
|
||||
|
||||
@@ -290,14 +290,6 @@ var AllCheckpoints = []*CheckpointDefinition{
|
||||
Instance: new(RequestISPNameCheckpoint),
|
||||
Priority: 90,
|
||||
},
|
||||
{
|
||||
Name: "CC统计（旧）",
|
||||
Prefix: "cc",
|
||||
Description: "统计某段时间段内的请求信息",
|
||||
HasParams: true,
|
||||
Instance: new(CCCheckpoint),
|
||||
Priority: 10,
|
||||
},
|
||||
{
|
||||
Name: "CC统计（新）",
|
||||
Prefix: "cc2",
|
||||
|
||||
Reference in New Issue
Block a user