robin/waf-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

引入lumberjack和fluentbit自动分发

Browse Source
This commit is contained in:
robin
2026-02-13 22:36:17 +08:00
parent c6da67db79
commit e9093baffb
47 changed files with 4589 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
EdgeAPI/internal/clickhouse/config.go
View File

@@ -18,11 +18,9 @@ const (
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
envScheme = "CLICKHOUSE_SCHEME"
envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY"
envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME"
defaultPort = 8123
defaultPort = 8443
defaultDB = "default"
defaultScheme = "http"
defaultScheme = "https"
)
var (
@@ -62,7 +60,7 @@ func ResetSharedConfig() {
}
func loadConfig() *Config {
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme}
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme, TLSSkipVerify: true}
// 1) 优先从后台页面配置（DB）读取
if models.SharedSysSettingDAO != nil {
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
@@ -72,8 +70,8 @@ func loadConfig() *Config {
cfg.Password = dbCfg.Password
cfg.Database = dbCfg.Database
cfg.Scheme = normalizeScheme(dbCfg.Scheme)
cfg.TLSSkipVerify = dbCfg.TLSSkipVerify
cfg.TLSServerName = dbCfg.TLSServerName
cfg.TLSSkipVerify = true
cfg.TLSServerName = ""
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
@@ -93,8 +91,8 @@ func loadConfig() *Config {
cfg.Password = ch.Password
cfg.Database = ch.Database
cfg.Scheme = normalizeScheme(ch.Scheme)
cfg.TLSSkipVerify = ch.TLSSkipVerify
cfg.TLSServerName = ch.TLSServerName
cfg.TLSSkipVerify = true
cfg.TLSServerName = ""
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
@@ -112,17 +110,13 @@ func loadConfig() *Config {
cfg.Database = defaultDB
}
cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
cfg.TLSServerName = os.Getenv(envTLSServerName)
cfg.TLSServerName = ""
if p := os.Getenv(envPort); p != "" {
if v, err := strconv.Atoi(p); err == nil {
cfg.Port = v
}
}
if v := os.Getenv(envTLSSkipVerify); v != "" {
if b, err := strconv.ParseBool(v); err == nil {
cfg.TLSSkipVerify = b
}
}
cfg.TLSSkipVerify = true
return cfg
}

4
EdgeAPI/internal/clickhouse/logs_ingest_store.go
View File

@@ -176,7 +176,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
}
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
// 列表查询不 SELECT 大字段（request_headers / request_body / response_headers / response_body），
// 避免每次翻页读取 GB 级数据。详情查看时通过 FindByTraceId 单独获取。
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id FROM %s WHERE %s ORDER BY %s LIMIT %d",
table, where, orderBy, limit+1)
var rawRows []map[string]interface{}

4
EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go
View File

@@ -122,8 +122,10 @@ func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*N
limit = 1000
}
// 列表查询不 SELECT content_json 大字段，减少翻页时的数据传输量。
// 详情查看时通过 FindByRequestId 单独获取完整信息。
query := fmt.Sprintf(
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
table,
strings.Join(conditions, " AND "),
orderDir,