引入lumberjack和fluentbit自动分发

This commit is contained in:
robin
2026-02-13 22:36:17 +08:00
parent c6da67db79
commit e9093baffb
47 changed files with 4589 additions and 317 deletions

View File

@@ -18,11 +18,9 @@ const (
envPassword = "CLICKHOUSE_PASSWORD" envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE" envDatabase = "CLICKHOUSE_DATABASE"
envScheme = "CLICKHOUSE_SCHEME" envScheme = "CLICKHOUSE_SCHEME"
envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY" defaultPort = 8443
envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME"
defaultPort = 8123
defaultDB = "default" defaultDB = "default"
defaultScheme = "http" defaultScheme = "https"
) )
var ( var (
@@ -62,7 +60,7 @@ func ResetSharedConfig() {
} }
func loadConfig() *Config { func loadConfig() *Config {
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme} cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme, TLSSkipVerify: true}
// 1) 优先从后台页面配置DB读取 // 1) 优先从后台页面配置DB读取
if models.SharedSysSettingDAO != nil { if models.SharedSysSettingDAO != nil {
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" { if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
@@ -72,8 +70,8 @@ func loadConfig() *Config {
cfg.Password = dbCfg.Password cfg.Password = dbCfg.Password
cfg.Database = dbCfg.Database cfg.Database = dbCfg.Database
cfg.Scheme = normalizeScheme(dbCfg.Scheme) cfg.Scheme = normalizeScheme(dbCfg.Scheme)
cfg.TLSSkipVerify = dbCfg.TLSSkipVerify cfg.TLSSkipVerify = true
cfg.TLSServerName = dbCfg.TLSServerName cfg.TLSServerName = ""
if cfg.Port <= 0 { if cfg.Port <= 0 {
cfg.Port = defaultPort cfg.Port = defaultPort
} }
@@ -93,8 +91,8 @@ func loadConfig() *Config {
cfg.Password = ch.Password cfg.Password = ch.Password
cfg.Database = ch.Database cfg.Database = ch.Database
cfg.Scheme = normalizeScheme(ch.Scheme) cfg.Scheme = normalizeScheme(ch.Scheme)
cfg.TLSSkipVerify = ch.TLSSkipVerify cfg.TLSSkipVerify = true
cfg.TLSServerName = ch.TLSServerName cfg.TLSServerName = ""
if cfg.Port <= 0 { if cfg.Port <= 0 {
cfg.Port = defaultPort cfg.Port = defaultPort
} }
@@ -112,17 +110,13 @@ func loadConfig() *Config {
cfg.Database = defaultDB cfg.Database = defaultDB
} }
cfg.Scheme = normalizeScheme(os.Getenv(envScheme)) cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
cfg.TLSServerName = os.Getenv(envTLSServerName) cfg.TLSServerName = ""
if p := os.Getenv(envPort); p != "" { if p := os.Getenv(envPort); p != "" {
if v, err := strconv.Atoi(p); err == nil { if v, err := strconv.Atoi(p); err == nil {
cfg.Port = v cfg.Port = v
} }
} }
if v := os.Getenv(envTLSSkipVerify); v != "" { cfg.TLSSkipVerify = true
if b, err := strconv.ParseBool(v); err == nil {
cfg.TLSSkipVerify = b
}
}
return cfg return cfg
} }

View File

@@ -176,7 +176,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
} }
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir) orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d", // 列表查询不 SELECT 大字段(request_headers / request_body / response_headers / response_body
// 避免每次翻页读取 GB 级数据。详情查看时通过 FindByTraceId 单独获取。
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id FROM %s WHERE %s ORDER BY %s LIMIT %d",
table, where, orderBy, limit+1) table, where, orderBy, limit+1)
var rawRows []map[string]interface{} var rawRows []map[string]interface{}

View File

@@ -122,8 +122,10 @@ func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*N
limit = 1000 limit = 1000
} }
// 列表查询不 SELECT content_json 大字段,减少翻页时的数据传输量。
// 详情查看时通过 FindByRequestId 单独获取完整信息。
query := fmt.Sprintf( query := fmt.Sprintf(
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d", "SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
table, table,
strings.Join(conditions, " AND "), strings.Join(conditions, " AND "),
orderDir, orderDir,

View File

@@ -20,3 +20,20 @@ func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string {
return strings.TrimSpace(config.Path) return strings.TrimSpace(config.Path)
} }
// ParseHTTPAccessLogPolicyRotateConfig 提取访问日志策略中的文件轮转配置(所有 file* 类型有效)。
func ParseHTTPAccessLogPolicyRotateConfig(policy *HTTPAccessLogPolicy) *serverconfigs.AccessLogRotateConfig {
if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 {
return nil
}
config := &serverconfigs.AccessLogFileStorageConfig{}
if err := json.Unmarshal(policy.Options, config); err != nil {
return nil
}
if config.Rotate == nil {
return nil
}
return config.Rotate.Normalize()
}

View File

@@ -1176,6 +1176,7 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
if publicPolicy != nil { if publicPolicy != nil {
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB) config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
config.GlobalServerConfig.HTTPAccessLog.Rotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
} }
} }
} }

View File

@@ -479,6 +479,7 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.
if publicPolicy != nil { if publicPolicy != nil {
config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB) config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
config.AccessLogRotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
} }
} }
} }

View File

@@ -270,8 +270,9 @@ func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.Clic
} }
if len(valueJSON) == 0 { if len(valueJSON) == 0 {
return &systemconfigs.ClickHouseSetting{ return &systemconfigs.ClickHouseSetting{
Port: 8123, Port: 8443,
Database: "default", Database: "default",
Scheme: "https",
}, nil }, nil
} }
var config = &systemconfigs.ClickHouseSetting{} var config = &systemconfigs.ClickHouseSetting{}

View File

@@ -1,14 +1,23 @@
package installers package installers
import ( import (
"crypto/sha256"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"net/url"
"os" "os"
slashpath "path"
"path/filepath" "path/filepath"
"sort" "sort"
"strconv"
"strings" "strings"
"time"
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
"github.com/iwind/TeaGo/Tea" "github.com/iwind/TeaGo/Tea"
) )
@@ -17,19 +26,45 @@ const (
fluentBitStorageDir = "/var/lib/fluent-bit/storage" fluentBitStorageDir = "/var/lib/fluent-bit/storage"
fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf" fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf"
fluentBitParsersFile = "/etc/fluent-bit/parsers.conf" fluentBitParsersFile = "/etc/fluent-bit/parsers.conf"
fluentBitUpstreamFile = "/etc/fluent-bit/clickhouse-upstream.conf" fluentBitManagedMetaFile = "/etc/fluent-bit/.edge-managed.json"
fluentBitManagedEnvFile = "/etc/fluent-bit/.edge-managed.env"
fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge" fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge"
fluentBitDropInDir = "/etc/systemd/system/fluent-bit.service.d"
fluentBitDropInFile = "/etc/systemd/system/fluent-bit.service.d/edge-managed.conf"
fluentBitServiceName = "fluent-bit" fluentBitServiceName = "fluent-bit"
fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit" fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit"
fluentBitLocalPackagesRoot = "packages" fluentBitLocalPackagesRoot = "packages"
fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log" fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log"
fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log" fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log"
fluentBitManagedMarker = "managed-by-edgeapi"
fluentBitRoleNode = "node"
fluentBitRoleDNS = "dns"
) )
var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found") var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found")
// SetupFluentBit 安装 Fluent Bit仅离线包并同步配置文件。 var fluentBitPackageFileMapping = map[string]string{
// 升级场景下不覆盖已有配置;若已有配置与节点角色不兼容,直接报错终止安装。 "ubuntu22.04-amd64": "fluent-bit_4.2.2_amd64.deb",
"ubuntu22.04-arm64": "fluent-bit_4.2.2_arm64.deb",
"amzn2023-amd64": "fluent-bit-4.2.2-1.x86_64.rpm",
"amzn2023-arm64": "fluent-bit-4.2.2-1.aarch64.rpm",
}
type fluentBitManagedMeta struct {
Roles []string `json:"roles"`
Hash string `json:"hash"`
UpdatedAt int64 `json:"updatedAt"`
SourceVersion string `json:"sourceVersion"`
}
type fluentBitDesiredConfig struct {
Roles []string
ClickHouse *systemconfigs.ClickHouseSetting
HTTPPathPattern string
DNSPathPattern string
}
// SetupFluentBit 安装并托管 Fluent Bit 配置(离线包 + 平台渲染配置)。
func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error { func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
if this.client == nil { if this.client == nil {
return errors.New("ssh client is nil") return errors.New("ssh client is nil")
@@ -41,55 +76,41 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
} }
tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit" tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit"
_, _, _ = this.client.Exec("mkdir -p " + tempDir) _, _, _ = this.client.Exec("mkdir -p " + shQuote(tempDir))
defer func() { defer func() {
_, _, _ = this.client.Exec("rm -rf " + tempDir) _, _, _ = this.client.Exec("rm -rf " + shQuote(tempDir))
}() }()
// 统一使用 fluent-bit.conf已含 HTTP + DNS 两类 input避免同机 Node/DNS 冲突。
files := []struct {
Local string
Remote string
}{
{Local: "fluent-bit.conf", Remote: "fluent-bit.conf"},
{Local: "parsers.conf", Remote: "parsers.conf"},
{Local: "clickhouse-upstream.conf", Remote: "clickhouse-upstream.conf"},
{Local: "logrotate.conf", Remote: "logrotate.conf"},
}
for _, file := range files {
localPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", file.Local)
if _, err := os.Stat(localPath); err != nil {
return fmt.Errorf("fluent-bit file '%s' not found: %w", localPath, err)
}
remotePath := tempDir + "/" + file.Remote
if err := this.client.Copy(localPath, remotePath, 0644); err != nil {
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", file.Local, err)
}
}
if err := this.ensureFluentBitInstalled(tempDir); err != nil { if err := this.ensureFluentBitInstalled(tempDir); err != nil {
return err return err
} }
_, stderr, err := this.client.Exec("mkdir -p " + fluentBitConfigDir + " " + fluentBitStorageDir + " /etc/logrotate.d") _, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitConfigDir) + " " + shQuote(fluentBitStorageDir) + " /etc/logrotate.d")
if err != nil { if err != nil {
return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr) return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr)
} }
exists, err := this.remoteFileExists(fluentBitMainConfigFile) parserContent, err := this.readLocalParsersContent()
if err != nil { if err != nil {
return err return err
} }
// 若已存在配置,先做角色兼容校验,不允许覆盖。 existingMeta, err := this.readManagedMeta()
if exists { if err != nil {
if err := this.validateExistingConfigForRole(role); err != nil {
return err return err
} }
mergedRoles, err := mergeManagedRoles(existingMeta, role)
if err != nil {
return err
} }
configCopied, err := this.copyFluentBitConfigIfMissing(tempDir) desiredConfig, err := this.buildDesiredFluentBitConfig(mergedRoles)
if err != nil {
return err
}
configChanged, err := this.applyManagedConfig(tempDir, desiredConfig, parserContent, existingMeta)
if err != nil { if err != nil {
return err return err
} }
@@ -99,7 +120,7 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
return err return err
} }
if err := this.ensureFluentBitService(binPath, configCopied); err != nil { if err := this.ensureFluentBitService(tempDir, binPath, configChanged); err != nil {
return err return err
} }
@@ -112,102 +133,128 @@ func (this *BaseInstaller) ensureFluentBitInstalled(tempDir string) error {
return nil return nil
} }
if err := this.installFluentBitFromLocalPackage(tempDir); err != nil { platformKey, packageName, arch, err := this.detectRemotePlatformAndPackage()
if err != nil {
return fmt.Errorf("detect fluent-bit platform failed: %w", err)
}
if err := this.installFluentBitFromLocalPackage(tempDir, arch, packageName); err != nil {
if errors.Is(err, errFluentBitLocalPackageNotFound) { if errors.Is(err, errFluentBitLocalPackageNotFound) {
return fmt.Errorf("install fluent-bit failed: local package not found, expected in deploy/fluent-bit/%s/linux-<arch>", fluentBitLocalPackagesRoot) expectedPath := filepath.Join("deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch, packageName)
return fmt.Errorf("install fluent-bit failed: local package missing for platform '%s', expected '%s'", platformKey, expectedPath)
} }
return fmt.Errorf("install fluent-bit from local package failed: %w", err) return fmt.Errorf("install fluent-bit from local package failed: %w", err)
} }
binPath, err := this.lookupFluentBitBinPath() binPath, err = this.lookupFluentBitBinPath()
if err != nil { if err != nil {
return err return err
} }
if binPath == "" { if binPath == "" {
return errors.New("fluent-bit binary not found after local package install") return errors.New("fluent-bit binary not found after local package install")
} }
_, stderr, err := this.client.Exec(binPath + " --version")
if err != nil {
return fmt.Errorf("verify fluent-bit version failed: %w, stderr: %s", err, stderr)
}
return nil return nil
} }
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string) error { func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string, arch string, packageName string) error {
arch, err := this.detectRemoteLinuxArch()
if err != nil {
return err
}
packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch) packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch)
entries, err := os.ReadDir(packageDir) localPackagePath := filepath.Join(packageDir, packageName)
if err != nil { if _, err := os.Stat(localPackagePath); err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return errFluentBitLocalPackageNotFound return errFluentBitLocalPackageNotFound
} }
return fmt.Errorf("read fluent-bit local package dir failed: %w", err) return fmt.Errorf("check local package failed: %w", err)
} }
packageFiles := make([]string, 0)
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := strings.ToLower(entry.Name())
if strings.HasSuffix(name, ".deb") || strings.HasSuffix(name, ".rpm") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") {
packageFiles = append(packageFiles, filepath.Join(packageDir, entry.Name()))
}
}
if len(packageFiles) == 0 {
return errFluentBitLocalPackageNotFound
}
sort.Strings(packageFiles)
var lastErr error
for _, localPackagePath := range packageFiles {
remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath) remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath)
if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil { if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil {
lastErr = fmt.Errorf("upload local package failed: %w", err) return fmt.Errorf("upload local package failed: %w", err)
continue
} }
var installCmd string var installCmd string
lowerName := strings.ToLower(localPackagePath) lowerName := strings.ToLower(localPackagePath)
switch { switch {
case strings.HasSuffix(lowerName, ".deb"): case strings.HasSuffix(lowerName, ".deb"):
installCmd = "dpkg -i " + remotePackagePath installCmd = "dpkg -i " + shQuote(remotePackagePath)
case strings.HasSuffix(lowerName, ".rpm"): case strings.HasSuffix(lowerName, ".rpm"):
installCmd = "rpm -Uvh --force " + remotePackagePath + " || rpm -ivh --force " + remotePackagePath installCmd = "rpm -Uvh --force " + shQuote(remotePackagePath) + " || rpm -ivh --force " + shQuote(remotePackagePath)
case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"): case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"):
extractDir := tempDir + "/extract" extractDir := tempDir + "/extract"
installCmd = "rm -rf " + extractDir + "; mkdir -p " + extractDir + "; tar -xzf " + remotePackagePath + " -C " + extractDir + "; " + installCmd = "rm -rf " + shQuote(extractDir) + "; mkdir -p " + shQuote(extractDir) + "; tar -xzf " + shQuote(remotePackagePath) + " -C " + shQuote(extractDir) + "; " +
"bin=$(find " + extractDir + " -type f -name fluent-bit | head -n 1); " + "bin=$(find " + shQuote(extractDir) + " -type f -name fluent-bit | head -n 1); " +
"if [ -z \"$bin\" ]; then exit 3; fi; " + "if [ -z \"$bin\" ]; then exit 3; fi; " +
"mkdir -p /opt/fluent-bit/bin /usr/local/bin; " + "mkdir -p /opt/fluent-bit/bin /usr/local/bin; " +
"install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " + "install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " +
"ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit" "ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit"
default: default:
continue return fmt.Errorf("unsupported local package format: %s", packageName)
} }
_, stderr, err := this.client.Exec(installCmd) _, stderr, err := this.client.Exec(installCmd)
if err != nil { if err != nil {
lastErr = fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr) return fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
continue
} }
binPath, err := this.lookupFluentBitBinPath()
if err == nil && binPath != "" {
return nil return nil
} }
func (this *BaseInstaller) detectRemotePlatformAndPackage() (platformKey string, packageName string, arch string, err error) {
arch, err = this.detectRemoteLinuxArch()
if err != nil { if err != nil {
lastErr = err return "", "", "", err
} else {
lastErr = errors.New("fluent-bit binary not found after local package install")
}
} }
if lastErr != nil { releaseData, stderr, err := this.client.Exec("cat /etc/os-release")
return lastErr if err != nil {
return "", "", "", fmt.Errorf("read /etc/os-release failed: %w, stderr: %s", err, stderr)
} }
return errFluentBitLocalPackageNotFound if strings.TrimSpace(releaseData) == "" {
return "", "", "", errors.New("/etc/os-release is empty")
}
releaseMap := parseOSRelease(releaseData)
id := strings.ToLower(strings.TrimSpace(releaseMap["ID"]))
versionID := strings.TrimSpace(releaseMap["VERSION_ID"])
var distro string
switch {
case id == "ubuntu" && strings.HasPrefix(versionID, "22.04"):
distro = "ubuntu22.04"
case id == "amzn" && strings.HasPrefix(versionID, "2023"):
distro = "amzn2023"
default:
return "", "", "", fmt.Errorf("unsupported linux platform id='%s' version='%s'", id, versionID)
}
platformKey = distro + "-" + arch
packageName, ok := fluentBitPackageFileMapping[platformKey]
if !ok {
return "", "", "", fmt.Errorf("no local package mapping for platform '%s'", platformKey)
}
return platformKey, packageName, arch, nil
}
func parseOSRelease(content string) map[string]string {
result := map[string]string{}
lines := strings.Split(content, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
continue
}
parts := strings.SplitN(line, "=", 2)
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
value = strings.Trim(value, "\"")
result[key] = value
}
return result
} }
func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) { func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) {
@@ -235,74 +282,456 @@ func (this *BaseInstaller) lookupFluentBitBinPath() (string, error) {
return strings.TrimSpace(stdout), nil return strings.TrimSpace(stdout), nil
} }
func (this *BaseInstaller) copyFluentBitConfigIfMissing(tempDir string) (bool, error) { func (this *BaseInstaller) readLocalParsersContent() (string, error) {
targets := []struct { parsersPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", "parsers.conf")
Src string data, err := os.ReadFile(parsersPath)
Dest string if err != nil {
}{ return "", fmt.Errorf("read local parsers config failed: %w", err)
{Src: tempDir + "/fluent-bit.conf", Dest: fluentBitMainConfigFile}, }
{Src: tempDir + "/parsers.conf", Dest: fluentBitParsersFile}, return string(data), nil
{Src: tempDir + "/clickhouse-upstream.conf", Dest: fluentBitUpstreamFile},
{Src: tempDir + "/logrotate.conf", Dest: fluentBitLogrotateFile},
} }
copied := false func (this *BaseInstaller) readManagedMeta() (*fluentBitManagedMeta, error) {
for _, target := range targets { exists, err := this.remoteFileExists(fluentBitManagedMetaFile)
exists, err := this.remoteFileExists(target.Dest) if err != nil {
return nil, err
}
if !exists {
return nil, nil
}
content, stderr, err := this.client.Exec("cat " + shQuote(fluentBitManagedMetaFile))
if err != nil {
return nil, fmt.Errorf("read fluent-bit managed metadata failed: %w, stderr: %s", err, stderr)
}
if strings.TrimSpace(content) == "" {
return nil, nil
}
meta := &fluentBitManagedMeta{}
if err := json.Unmarshal([]byte(content), meta); err != nil {
return nil, fmt.Errorf("decode fluent-bit managed metadata failed: %w", err)
}
meta.Roles = normalizeRoles(meta.Roles)
return meta, nil
}
func mergeManagedRoles(meta *fluentBitManagedMeta, role nodeconfigs.NodeRole) ([]string, error) {
roleName, err := mapNodeRole(role)
if err != nil {
return nil, err
}
roleSet := map[string]struct{}{}
if meta != nil {
for _, r := range normalizeRoles(meta.Roles) {
roleSet[r] = struct{}{}
}
}
roleSet[roleName] = struct{}{}
roles := make([]string, 0, len(roleSet))
for r := range roleSet {
roles = append(roles, r)
}
sort.Strings(roles)
return roles, nil
}
func mapNodeRole(role nodeconfigs.NodeRole) (string, error) {
switch role {
case nodeconfigs.NodeRoleNode:
return fluentBitRoleNode, nil
case nodeconfigs.NodeRoleDNS:
return fluentBitRoleDNS, nil
default:
return "", fmt.Errorf("unsupported fluent-bit role '%s'", role)
}
}
func normalizeRoles(rawRoles []string) []string {
roleSet := map[string]struct{}{}
for _, role := range rawRoles {
role = strings.ToLower(strings.TrimSpace(role))
if role != fluentBitRoleNode && role != fluentBitRoleDNS {
continue
}
roleSet[role] = struct{}{}
}
roles := make([]string, 0, len(roleSet))
for role := range roleSet {
roles = append(roles, role)
}
sort.Strings(roles)
return roles
}
func hasRole(roles []string, role string) bool {
for _, one := range roles {
if one == role {
return true
}
}
return false
}
func (this *BaseInstaller) buildDesiredFluentBitConfig(roles []string) (*fluentBitDesiredConfig, error) {
if len(roles) == 0 {
return nil, errors.New("fluent-bit roles should not be empty")
}
ch, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil)
if err != nil {
return nil, fmt.Errorf("read clickhouse setting failed: %w", err)
}
if ch == nil {
ch = &systemconfigs.ClickHouseSetting{}
}
if strings.TrimSpace(ch.Host) == "" {
ch.Host = "127.0.0.1"
}
ch.Scheme = strings.ToLower(strings.TrimSpace(ch.Scheme))
if ch.Scheme == "" {
ch.Scheme = "https"
}
if ch.Scheme != "http" && ch.Scheme != "https" {
return nil, fmt.Errorf("unsupported clickhouse scheme '%s'", ch.Scheme)
}
if ch.Port <= 0 {
if ch.Scheme == "https" {
ch.Port = 8443
} else {
ch.Port = 8443
}
}
if strings.TrimSpace(ch.Database) == "" {
ch.Database = "default"
}
if strings.TrimSpace(ch.User) == "" {
ch.User = "default"
}
// 当前平台策略:后台固定跳过 ClickHouse TLS 证书校验,不暴露 ServerName 配置。
ch.TLSSkipVerify = true
ch.TLSServerName = ""
httpPathPattern := fluentBitHTTPPathPattern
dnsPathPattern := fluentBitDNSPathPattern
publicPolicyPath, err := this.readPublicAccessLogPolicyPath()
if err != nil {
return nil, err
}
policyDir := dirFromPolicyPath(publicPolicyPath)
if policyDir != "" {
pattern := strings.TrimRight(policyDir, "/") + "/*.log"
httpPathPattern = pattern
dnsPathPattern = pattern
}
return &fluentBitDesiredConfig{
Roles: normalizeRoles(roles),
ClickHouse: ch,
HTTPPathPattern: httpPathPattern,
DNSPathPattern: dnsPathPattern,
}, nil
}
func (this *BaseInstaller) readPublicAccessLogPolicyPath() (string, error) {
policyId, err := models.SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(nil)
if err != nil {
return "", fmt.Errorf("find current public access log policy failed: %w", err)
}
if policyId <= 0 {
return "", nil
}
policy, err := models.SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(nil, policyId)
if err != nil {
return "", fmt.Errorf("read public access log policy failed: %w", err)
}
if policy == nil {
return "", nil
}
return strings.TrimSpace(models.ParseHTTPAccessLogPolicyFilePath(policy)), nil
}
func dirFromPolicyPath(policyPath string) string {
pathValue := strings.TrimSpace(policyPath)
if pathValue == "" {
return ""
}
pathValue = strings.ReplaceAll(pathValue, "\\", "/")
dir := slashpath.Dir(pathValue)
if dir == "." {
return ""
}
return strings.TrimRight(dir, "/")
}
func (this *BaseInstaller) applyManagedConfig(tempDir string, desired *fluentBitDesiredConfig, parserContent string, existingMeta *fluentBitManagedMeta) (bool, error) {
mainExists, err := this.remoteFileExists(fluentBitMainConfigFile)
if err != nil { if err != nil {
return false, err return false, err
} }
if exists {
continue if mainExists && existingMeta == nil {
} containsMarker, err := this.remoteFileContains(fluentBitMainConfigFile, fluentBitManagedMarker)
_, stderr, err := this.client.Exec("cp -f " + target.Src + " " + target.Dest)
if err != nil { if err != nil {
return false, fmt.Errorf("copy fluent-bit file to '%s' failed: %w, stderr: %s", target.Dest, err, stderr) return false, err
}
if !containsMarker {
// Adopt unmanaged config by backing it up and replacing with managed config below.
} }
copied = true
} }
return copied, nil configContent, err := renderManagedConfig(desired)
}
func (this *BaseInstaller) validateExistingConfigForRole(role nodeconfigs.NodeRole) error {
requiredPatterns := []string{}
switch role {
case nodeconfigs.NodeRoleNode:
requiredPatterns = append(requiredPatterns, fluentBitHTTPPathPattern)
case nodeconfigs.NodeRoleDNS:
requiredPatterns = append(requiredPatterns, fluentBitDNSPathPattern)
}
for _, pattern := range requiredPatterns {
ok, err := this.remoteFileContains(fluentBitMainConfigFile, pattern)
if err != nil { if err != nil {
return err return false, err
} }
if !ok { envContent := renderManagedEnv(desired.ClickHouse)
return fmt.Errorf("existing fluent-bit config '%s' does not contain required path '%s'; skip overwrite by design, please update config manually", fluentBitMainConfigFile, pattern) metaContent, newMeta, err := renderManagedMeta(desired, configContent, parserContent, envContent)
if err != nil {
return false, err
} }
requiredFiles := []string{fluentBitMainConfigFile, fluentBitParsersFile, fluentBitManagedEnvFile, fluentBitManagedMetaFile}
if existingMeta != nil && existingMeta.Hash == newMeta.Hash {
allExists := true
for _, file := range requiredFiles {
exists, err := this.remoteFileExists(file)
if err != nil {
return false, err
}
if !exists {
allExists = false
break
}
}
if allExists {
return false, nil
}
}
if mainExists {
backup := fluentBitMainConfigFile + ".bak." + strconv.FormatInt(time.Now().Unix(), 10)
_, stderr, err := this.client.Exec("cp -f " + shQuote(fluentBitMainConfigFile) + " " + shQuote(backup))
if err != nil {
return false, fmt.Errorf("backup existing fluent-bit config failed: %w, stderr: %s", err, stderr)
}
}
if err := this.writeRemoteFileByTemp(tempDir, fluentBitMainConfigFile, configContent, 0644); err != nil {
return false, err
}
if err := this.writeRemoteFileByTemp(tempDir, fluentBitParsersFile, parserContent, 0644); err != nil {
return false, err
}
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedEnvFile, envContent, 0600); err != nil {
return false, err
}
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedMetaFile, metaContent, 0644); err != nil {
return false, err
}
localLogrotate := filepath.Join(Tea.Root, "deploy", "fluent-bit", "logrotate.conf")
if _, err := os.Stat(localLogrotate); err == nil {
if err := this.copyLocalFileToRemote(tempDir, localLogrotate, fluentBitLogrotateFile, 0644); err != nil {
return false, err
}
}
return true, nil
}
func renderManagedConfig(desired *fluentBitDesiredConfig) (string, error) {
if desired == nil || desired.ClickHouse == nil {
return "", errors.New("invalid fluent-bit desired config")
}
scheme := strings.ToLower(strings.TrimSpace(desired.ClickHouse.Scheme))
if scheme == "" {
scheme = "http"
}
if scheme != "http" && scheme != "https" {
return "", fmt.Errorf("unsupported clickhouse scheme '%s'", desired.ClickHouse.Scheme)
}
useTLS := scheme == "https"
insertHTTP := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
insertDNS := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.dns_logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
lines := []string{
"# " + fluentBitManagedMarker,
"[SERVICE]",
" Flush 2",
" Log_Level info",
" Parsers_File " + fluentBitParsersFile,
" storage.path " + fluentBitStorageDir,
" storage.sync normal",
" storage.checksum off",
" storage.backlog.mem_limit 256MB",
"",
}
if hasRole(desired.Roles, fluentBitRoleNode) {
lines = append(lines,
"[INPUT]",
" Name tail",
" Path "+desired.HTTPPathPattern,
" Tag app.http.logs",
" Parser json",
" Refresh_Interval 2",
" Read_from_Head false",
" DB /var/lib/fluent-bit/http-logs.db",
" storage.type filesystem",
" Mem_Buf_Limit 128MB",
" Skip_Long_Lines On",
"",
)
}
if hasRole(desired.Roles, fluentBitRoleDNS) {
lines = append(lines,
"[INPUT]",
" Name tail",
" Path "+desired.DNSPathPattern,
" Tag app.dns.logs",
" Parser json",
" Refresh_Interval 2",
" Read_from_Head false",
" DB /var/lib/fluent-bit/dns-logs.db",
" storage.type filesystem",
" Mem_Buf_Limit 128MB",
" Skip_Long_Lines On",
"",
)
}
if hasRole(desired.Roles, fluentBitRoleNode) {
lines = append(lines,
"[OUTPUT]",
" Name http",
" Match app.http.logs",
" Host "+desired.ClickHouse.Host,
" Port "+strconv.Itoa(desired.ClickHouse.Port),
" URI /?query="+insertHTTP,
" Format json_lines",
" http_user ${CH_USER}",
" http_passwd ${CH_PASSWORD}",
" json_date_key timestamp",
" json_date_format epoch",
" workers 1",
" net.keepalive On",
" Retry_Limit False",
)
if useTLS {
lines = append(lines, " tls On")
if desired.ClickHouse.TLSSkipVerify {
lines = append(lines, " tls.verify Off")
} else {
lines = append(lines, " tls.verify On")
}
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
}
}
lines = append(lines, "")
}
if hasRole(desired.Roles, fluentBitRoleDNS) {
lines = append(lines,
"[OUTPUT]",
" Name http",
" Match app.dns.logs",
" Host "+desired.ClickHouse.Host,
" Port "+strconv.Itoa(desired.ClickHouse.Port),
" URI /?query="+insertDNS,
" Format json_lines",
" http_user ${CH_USER}",
" http_passwd ${CH_PASSWORD}",
" json_date_key timestamp",
" json_date_format epoch",
" workers 1",
" net.keepalive On",
" Retry_Limit False",
)
if useTLS {
lines = append(lines, " tls On")
if desired.ClickHouse.TLSSkipVerify {
lines = append(lines, " tls.verify Off")
} else {
lines = append(lines, " tls.verify On")
}
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
}
}
lines = append(lines, "")
}
return strings.Join(lines, "\n"), nil
}
func renderManagedEnv(ch *systemconfigs.ClickHouseSetting) string {
user := "default"
password := ""
if ch != nil {
if strings.TrimSpace(ch.User) != "" {
user = strings.TrimSpace(ch.User)
}
password = ch.Password
}
return "CH_USER=" + strconv.Quote(user) + "\n" +
"CH_PASSWORD=" + strconv.Quote(password) + "\n"
}
func renderManagedMeta(desired *fluentBitDesiredConfig, configContent string, parserContent string, envContent string) (string, *fluentBitManagedMeta, error) {
hashInput := configContent + "\n---\n" + parserContent + "\n---\n" + envContent + "\n---\n" + strings.Join(desired.Roles, ",")
hashBytes := sha256.Sum256([]byte(hashInput))
hash := fmt.Sprintf("%x", hashBytes[:])
meta := &fluentBitManagedMeta{
Roles: desired.Roles,
Hash: hash,
UpdatedAt: time.Now().Unix(),
SourceVersion: teaconst.Version,
}
data, err := json.MarshalIndent(meta, "", " ")
if err != nil {
return "", nil, fmt.Errorf("encode fluent-bit managed metadata failed: %w", err)
}
return string(data) + "\n", meta, nil
}
func (this *BaseInstaller) copyLocalFileToRemote(tempDir string, localPath string, remotePath string, mode os.FileMode) error {
tempFile := tempDir + "/" + filepath.Base(remotePath)
if err := this.client.Copy(localPath, tempFile, mode); err != nil {
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", localPath, err)
}
_, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
if err != nil {
return fmt.Errorf("install remote fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
} }
return nil return nil
} }
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) { func (this *BaseInstaller) writeRemoteFileByTemp(tempDir string, remotePath string, content string, mode os.FileMode) error {
stdout, stderr, err := this.client.Exec("if [ -f \"" + path + "\" ]; then echo 1; else echo 0; fi") tempFile := tempDir + "/" + filepath.Base(remotePath) + ".tmp"
if err != nil { if _, err := this.client.WriteFile(tempFile, []byte(content)); err != nil {
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr) return fmt.Errorf("write temp fluent-bit file '%s' failed: %w", tempFile, err)
}
return strings.TrimSpace(stdout) == "1", nil
} }
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) { _, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
stdout, stderr, err := this.client.Exec("if grep -F \"" + pattern + "\" \"" + path + "\" >/dev/null 2>&1; then echo 1; else echo 0; fi")
if err != nil { if err != nil {
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr) return fmt.Errorf("write managed fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
} }
return strings.TrimSpace(stdout) == "1", nil return nil
} }
func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied bool) error { func (this *BaseInstaller) ensureFluentBitService(tempDir string, binPath string, configChanged bool) error {
_, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " + _, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " +
"cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" + "cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" +
"[Unit]\n" + "[Unit]\n" +
@@ -319,26 +748,90 @@ func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied b
"EOF\n" + "EOF\n" +
"fi") "fi")
stdout, stderr, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " + stdout, _, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then echo 1; else echo 0; fi")
"if [ \"" + boolToString(configCopied) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " + if err != nil {
"else systemctl start " + fluentBitServiceName + "; fi; else echo no-systemctl; fi") return fmt.Errorf("check systemctl failed: %w", err)
}
if strings.TrimSpace(stdout) == "1" {
dropInChanged, err := this.ensureServiceDropIn(tempDir, binPath)
if err != nil {
return err
}
restartRequired := configChanged || dropInChanged
_, stderr, err := this.client.Exec("systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; " +
"if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
"if [ \"" + boolToString(restartRequired) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
"else systemctl start " + fluentBitServiceName + "; fi")
if err != nil { if err != nil {
return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr) return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr)
} }
return nil
}
if configChanged {
_, _, _ = this.client.Exec("pkill -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1 || true")
}
if strings.TrimSpace(stdout) == "no-systemctl" {
_, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1") _, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1")
if runningErr != nil { if runningErr != nil {
_, stderr, err = this.client.Exec(binPath + " -c " + fluentBitMainConfigFile + " >/dev/null 2>&1 &") startCmd := "set -a; [ -f " + shQuote(fluentBitManagedEnvFile) + " ] && . " + shQuote(fluentBitManagedEnvFile) + "; set +a; " +
shQuote(binPath) + " -c " + shQuote(fluentBitMainConfigFile) + " >/dev/null 2>&1 &"
_, stderr, err := this.client.Exec(startCmd)
if err != nil { if err != nil {
return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr) return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr)
} }
} }
}
return nil return nil
} }
func (this *BaseInstaller) ensureServiceDropIn(tempDir string, binPath string) (bool, error) {
_, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitDropInDir))
if err != nil {
return false, fmt.Errorf("prepare fluent-bit drop-in dir failed: %w, stderr: %s", err, stderr)
}
content := "[Service]\n" +
"EnvironmentFile=-" + fluentBitManagedEnvFile + "\n" +
"ExecStart=\n" +
"ExecStart=" + binPath + " -c " + fluentBitMainConfigFile + "\n"
existing, _, _ := this.client.Exec("if [ -f " + shQuote(fluentBitDropInFile) + " ]; then cat " + shQuote(fluentBitDropInFile) + "; fi")
if existing == content {
return false, nil
}
if err := this.writeRemoteFileByTemp(tempDir, fluentBitDropInFile, content, 0644); err != nil {
return false, err
}
return true, nil
}
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
stdout, stderr, err := this.client.Exec("if [ -f " + shQuote(path) + " ]; then echo 1; else echo 0; fi")
if err != nil {
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
}
return strings.TrimSpace(stdout) == "1", nil
}
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
stdout, stderr, err := this.client.Exec("if grep -F " + shQuote(pattern) + " " + shQuote(path) + " >/dev/null 2>&1; then echo 1; else echo 0; fi")
if err != nil {
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
}
return strings.TrimSpace(stdout) == "1", nil
}
func shQuote(value string) string {
if value == "" {
return "''"
}
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
}
func boolToString(v bool) string { func boolToString(v bool) string {
if v { if v {
return "1" return "1"

View File

@@ -34,7 +34,7 @@ function build() {
done done
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go) VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
# 生成 zip 文件名时不包含 plus 标记 # 鐢熸垚 zip 鏂囦欢鍚嶆椂涓嶅寘鍚?plus 鏍囪
if [ "${TAG}" = "plus" ]; then if [ "${TAG}" = "plus" ]; then
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip" ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
else else
@@ -97,7 +97,7 @@ function build() {
done done
fi fi
# 查找 EdgeAPI zip 文件时不包含 plus 标记 # 鏌ユ壘 EdgeAPI zip 鏂囦欢鏃朵笉鍖呭惈 plus 鏍囪
if [ "${TAG}" = "plus" ]; then if [ "${TAG}" = "plus" ]; then
EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip" EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip"
else else
@@ -107,7 +107,44 @@ function build() {
cd "$DIST"/ || exit cd "$DIST"/ || exit
unzip -q "$(basename "$EDGE_API_ZIP_FILE")" unzip -q "$(basename "$EDGE_API_ZIP_FILE")"
rm -f "$(basename "$EDGE_API_ZIP_FILE")" rm -f "$(basename "$EDGE_API_ZIP_FILE")"
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件)
# ensure edge-api package always contains fluent-bit templates/packages
FLUENT_ROOT="$ROOT/../../deploy/fluent-bit"
FLUENT_DIST="$DIST/edge-api/deploy/fluent-bit"
if [ -d "$FLUENT_ROOT" ]; then
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || exit 1
rm -rf "$FLUENT_DIST"
mkdir -p "$FLUENT_DIST"
FLUENT_FILES=(
"fluent-bit.conf"
"fluent-bit-dns.conf"
"fluent-bit-https.conf"
"fluent-bit-dns-https.conf"
"fluent-bit-windows.conf"
"fluent-bit-windows-https.conf"
"parsers.conf"
"clickhouse-upstream.conf"
"clickhouse-upstream-windows.conf"
"logrotate.conf"
"README.md"
)
for file in "${FLUENT_FILES[@]}"; do
if [ -f "$FLUENT_ROOT/$file" ]; then
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
fi
done
if [ -d "$FLUENT_ROOT/packages" ]; then
cp -R "$FLUENT_ROOT/packages" "$FLUENT_DIST/"
fi
rm -f "$FLUENT_DIST/.gitignore"
rm -f "$FLUENT_DIST"/logs.db*
rm -rf "$FLUENT_DIST/storage"
fi
# 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
find . -name "*.mmdb" -type f -delete find . -name "*.mmdb" -type f -delete
find . -type d -name "iplibrary" -empty -delete find . -type d -name "iplibrary" -empty -delete
cd - || exit cd - || exit
@@ -150,7 +187,7 @@ function build() {
#find "$DIST" -name "*.css.map" -delete #find "$DIST" -name "*.css.map" -delete
#find "$DIST" -name "*.js.map" -delete #find "$DIST" -name "*.js.map" -delete
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件) # 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
find "$DIST" -name "*.mmdb" -type f -delete find "$DIST" -name "*.mmdb" -type f -delete
find "$DIST" -type d -name "iplibrary" -empty -delete find "$DIST" -type d -name "iplibrary" -empty -delete
@@ -167,6 +204,39 @@ function build() {
echo "[done]" echo "[done]"
} }
function verify_fluent_bit_package_matrix() {
FLUENT_ROOT=$1
ARCH=$2
REQUIRED_FILES=()
if [ "$ARCH" = "amd64" ]; then
REQUIRED_FILES=(
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
)
elif [ "$ARCH" = "arm64" ]; then
REQUIRED_FILES=(
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
)
else
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
return 1
fi
MISSING=0
for FILE in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
MISSING=1
fi
done
if [ "$MISSING" -ne 0 ]; then
return 1
fi
return 0
}
function lookup-version() { function lookup-version() {
FILE=$1 FILE=$1
VERSION_DATA=$(cat "$FILE") VERSION_DATA=$(cat "$FILE")

View File

@@ -3,13 +3,18 @@
package db package db
import ( import (
"crypto/tls"
"encoding/json" "encoding/json"
"fmt"
"net/http"
"strings"
"time"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes" "github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
"github.com/iwind/TeaGo/actions" "github.com/iwind/TeaGo/actions"
"strings"
) )
const clickhouseConfigCode = "clickhouseConfig" const clickhouseConfigCode = "clickhouseConfig"
@@ -29,18 +34,18 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
this.ErrorPage(err) this.ErrorPage(err)
return return
} }
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default", Scheme: "http"} cfg := &systemconfigs.ClickHouseSetting{Port: 8443, Database: "default", Scheme: "https"}
if len(resp.ValueJSON) > 0 { if len(resp.ValueJSON) > 0 {
_ = json.Unmarshal(resp.ValueJSON, cfg) _ = json.Unmarshal(resp.ValueJSON, cfg)
} }
if cfg.Port <= 0 { if cfg.Port <= 0 {
cfg.Port = 8123 cfg.Port = 8443
} }
if cfg.Database == "" { if cfg.Database == "" {
cfg.Database = "default" cfg.Database = "default"
} }
if strings.TrimSpace(cfg.Scheme) == "" { if strings.TrimSpace(cfg.Scheme) == "" {
cfg.Scheme = "http" cfg.Scheme = "https"
} }
this.Data["config"] = map[string]interface{}{ this.Data["config"] = map[string]interface{}{
"host": cfg.Host, "host": cfg.Host,
@@ -49,9 +54,17 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
"password": cfg.Password, "password": cfg.Password,
"database": cfg.Database, "database": cfg.Database,
"scheme": cfg.Scheme, "scheme": cfg.Scheme,
"tlsSkipVerify": cfg.TLSSkipVerify,
"tlsServerName": cfg.TLSServerName,
} }
// 自动检测连接状态
connStatus := "unconfigured" // unconfigured / connected / disconnected
connError := ""
if strings.TrimSpace(cfg.Host) != "" {
connStatus, connError = this.probeClickHouse(cfg)
}
this.Data["connStatus"] = connStatus
this.Data["connError"] = connError
this.Show() this.Show()
} }
@@ -62,20 +75,18 @@ func (this *ClickHouseAction) RunPost(params struct {
Password string Password string
Database string Database string
Scheme string Scheme string
TLSSkipVerify bool
TLSServerName string
Must *actions.Must Must *actions.Must
}) { }) {
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0) defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
if params.Port <= 0 {
params.Port = 8123
}
if params.Database == "" { if params.Database == "" {
params.Database = "default" params.Database = "default"
} }
if params.Scheme != "https" { if params.Scheme != "http" {
params.Scheme = "http" params.Scheme = "https"
}
if params.Port <= 0 {
params.Port = 8443
} }
password := params.Password password := params.Password
if password == "" { if password == "" {
@@ -94,8 +105,8 @@ func (this *ClickHouseAction) RunPost(params struct {
Password: password, Password: password,
Database: params.Database, Database: params.Database,
Scheme: params.Scheme, Scheme: params.Scheme,
TLSSkipVerify: params.TLSSkipVerify, TLSSkipVerify: true,
TLSServerName: strings.TrimSpace(params.TLSServerName), TLSServerName: "",
} }
valueJSON, err := json.Marshal(cfg) valueJSON, err := json.Marshal(cfg)
if err != nil { if err != nil {
@@ -112,3 +123,51 @@ func (this *ClickHouseAction) RunPost(params struct {
} }
this.Success() this.Success()
} }
// probeClickHouse 快速检测 ClickHouse 连接状态SELECT 1
func (this *ClickHouseAction) probeClickHouse(cfg *systemconfigs.ClickHouseSetting) (status string, errMsg string) {
scheme := strings.ToLower(strings.TrimSpace(cfg.Scheme))
if scheme == "" {
scheme = "https"
}
port := cfg.Port
if port <= 0 {
port = 8443
}
db := cfg.Database
if db == "" {
db = "default"
}
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s", scheme, cfg.Host, port, db)
transport := &http.Transport{}
if scheme == "https" {
transport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
}
}
client := &http.Client{
Timeout: 3 * time.Second,
Transport: transport,
}
req, err := http.NewRequest(http.MethodGet, testURL, nil)
if err != nil {
return "disconnected", err.Error()
}
if cfg.User != "" || cfg.Password != "" {
req.SetBasicAuth(cfg.User, cfg.Password)
}
resp, err := client.Do(req)
if err != nil {
return "disconnected", err.Error()
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "disconnected", fmt.Sprintf("HTTP %d", resp.StatusCode)
}
return "connected", ""
}

View File

@@ -17,7 +17,7 @@ func (this *ClickHouseAction) Init() {
func (this *ClickHouseAction) RunGet(params struct{}) { func (this *ClickHouseAction) RunGet(params struct{}) {
this.Data["mainTab"] = "clickhouse" this.Data["mainTab"] = "clickhouse"
this.Data["config"] = map[string]interface{}{ this.Data["config"] = map[string]interface{}{
"host": "", "port": 8123, "user": "", "password": "", "database": "default", "host": "", "port": 8443, "user": "", "password": "", "database": "default", "scheme": "https",
} }
this.Show() this.Show()
} }

View File

@@ -25,6 +25,7 @@ func init() {
Get("/logs", new(LogsAction)). Get("/logs", new(LogsAction)).
Post("/status", new(StatusAction)). Post("/status", new(StatusAction)).
GetPost("/clickhouse", new(ClickHouseAction)). GetPost("/clickhouse", new(ClickHouseAction)).
Post("/testClickhouse", new(TestClickHouseAction)).
EndAll() EndAll()
}) })
} }

View File

@@ -0,0 +1,92 @@
//go:build plus
package db
import (
"crypto/tls"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/iwind/TeaGo/actions"
)
type TestClickHouseAction struct {
actionutils.ParentAction
}
func (this *TestClickHouseAction) Init() {
this.Nav("db", "db", "clickhouse")
}
func (this *TestClickHouseAction) RunPost(params struct {
Host string
Port int
User string
Password string
Database string
Scheme string
Must *actions.Must
}) {
params.Must.
Field("host", params.Host).
Require("请输入 ClickHouse 连接地址")
if params.Database == "" {
params.Database = "default"
}
scheme := "https"
if strings.EqualFold(params.Scheme, "http") {
scheme = "http"
}
if params.Port <= 0 {
params.Port = 8443
}
// 构造测试请求
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s",
scheme, params.Host, params.Port, params.Database)
transport := &http.Transport{}
if scheme == "https" {
transport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: true,
}
}
client := &http.Client{
Timeout: 5 * time.Second,
Transport: transport,
}
req, err := http.NewRequest(http.MethodGet, testURL, nil)
if err != nil {
this.Fail("请求构造失败: " + err.Error())
return
}
if params.User != "" || params.Password != "" {
req.SetBasicAuth(params.User, params.Password)
}
resp, err := client.Do(req)
if err != nil {
this.Fail("连接失败: " + err.Error())
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
msg := strings.TrimSpace(string(body))
if len(msg) > 200 {
msg = msg[:200] + "..."
}
this.Fail(fmt.Sprintf("ClickHouse 返回 HTTP %d: %s", resp.StatusCode, msg))
return
}
this.Success()
}

View File

@@ -0,0 +1,23 @@
//go:build !plus
package db
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/iwind/TeaGo/actions"
)
type TestClickHouseAction struct {
actionutils.ParentAction
}
func (this *TestClickHouseAction) Init() {
this.Nav("db", "db", "clickhouse")
}
func (this *TestClickHouseAction) RunPost(params struct {
Host string
Must *actions.Must
}) {
this.Fail("请使用商业版以测试 ClickHouse 连接")
}

View File

@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
<div> <div>
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a> <a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
<!-- 网站 -->
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span> <span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>&quot;<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}&quot; </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword> <ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>&quot;<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}&quot; </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
</span> </span>
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small">&nbsp; ({{accessLog.humanTime}})</span> <span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small">&nbsp; ({{accessLog.humanTime}})</span>
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
&nbsp; <a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a> &nbsp; <a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
</div> </div>
</div>` </div>`

View File

@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
<div> <div>
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a> <a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
<!-- 网站 -->
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span> <span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>&quot;<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}&quot; </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword> <ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>&quot;<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}&quot; </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
</span> </span>
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small">&nbsp; ({{accessLog.humanTime}})</span> <span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small">&nbsp; ({{accessLog.humanTime}})</span>
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
&nbsp; <a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a> &nbsp; <a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
</div> </div>
</div>` </div>`

View File

@@ -1,70 +1,76 @@
{$layout} {$layout}
{$template "menu"} {$template "menu"}
<h3>ClickHouse 配置</h3> <div style="display:flex;align-items:baseline;gap:0.8em;margin-bottom:0.5em">
<p class="comment">用于访问日志列表查询logs_ingest 表)。配置后,访问日志列表将优先从 ClickHouse 读取;不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。</p> <h3 style="margin:0">ClickHouse 配置</h3>
<span v-if="connStatus === 'connected'" class="ui label green tiny" style="vertical-align:baseline">
<i class="icon circle" style="margin-right:0.25em"></i>已连接
</span>
<span v-else-if="connStatus === 'disconnected'" class="ui label red tiny" style="vertical-align:baseline"
:title="connError">
<i class="icon circle" style="margin-right:0.25em"></i>已断开
</span>
<span v-else class="ui label grey tiny" style="vertical-align:baseline">
<i class="icon circle outline" style="margin-right:0.25em"></i>未配置
</span>
</div>
<p class="comment">配置后,需要在网站列表-日志策略中创建"文件+ClickHouse"的策略才能将访问日志绕过api组件直接由边缘节点发送给ClickHouse。</p>
<form method="post" class="ui form" data-tea-action="$" data-tea-success="success" @submit.prevent="onSubmit"> <form method="post" class="ui form" @submit.prevent="onSubmit">
<csrf-token></csrf-token> <csrf-token></csrf-token>
<table class="ui table definition selectable"> <table class="ui table definition selectable">
<tr> <tr>
<td class="title">连接地址Host</td> <td class="title">连接地址Host</td>
<td> <td>
<input type="text" name="host" maxlength="200" ref="focus" placeholder="如 127.0.0.1 或 clickhouse.example.com" :value="config.host"/> <input type="text" name="host" maxlength="200" ref="focus"
placeholder="如 127.0.0.1 或 clickhouse.example.com" v-model="form.host" />
<p class="comment">ClickHouse 服务器地址。</p> <p class="comment">ClickHouse 服务器地址。</p>
</td> </td>
</tr> </tr>
<tr> <tr>
<td>协议Scheme</td> <td>协议Scheme</td>
<td> <td>
<select name="scheme" class="ui dropdown auto-width"> <select name="scheme" class="ui dropdown auto-width" v-model="form.scheme">
<option value="http" :selected="config.scheme != 'https'">HTTP</option> <option value="http">HTTP</option>
<option value="https" :selected="config.scheme == 'https'">HTTPS</option> <option value="https">HTTPS</option>
</select> </select>
<p class="comment">默认 HTTP;选择 HTTPS 时将启用 TLS 连接</p> <p class="comment">默认 HTTPS当前后台固定跳过证书校验</p>
</td> </td>
</tr> </tr>
<tr> <tr>
<td>端口Port</td> <td>端口Port</td>
<td> <td>
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/> <input type="number" name="port" min="1" max="65535" style="width:6em" v-model.number="form.port" />
<p class="comment">接口端口HTTP 默认 8123HTTPS 常用 8443以你的 ClickHouse 实际配置为准)</p> <p class="comment">接口端口默认 8443HTTPS请与 ClickHouse 实际开放端口保持一致</p>
</td> </td>
</tr> </tr>
<tr> <tr>
<td>用户名User</td> <td>用户名User</td>
<td> <td>
<input type="text" name="user" maxlength="100" :value="config.user"/> <input type="text" name="user" maxlength="100" v-model="form.user" />
</td> </td>
</tr> </tr>
<tr> <tr>
<td>密码Password</td> <td>密码Password</td>
<td> <td>
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" value=""/> <input type="password" name="password" maxlength="200" placeholder="不修改请留空" v-model="form.password" />
<p class="comment">留空则不修改已保存的密码。</p> <p class="comment">留空则不修改已保存的密码。</p>
</td> </td>
</tr> </tr>
<tr>
<td>TLS 跳过证书校验</td>
<td>
<checkbox name="tlsSkipVerify" value="1" :checked="config.tlsSkipVerify"></checkbox>
<p class="comment">仅测试环境建议开启;生产建议关闭并使用受信任证书。</p>
</td>
</tr>
<tr>
<td>TLS Server Name</td>
<td>
<input type="text" name="tlsServerName" maxlength="200" placeholder="可选证书校验域名SNI" :value="config.tlsServerName"/>
<p class="comment">可选;当 ClickHouse 证书域名与连接 Host 不一致时使用。</p>
</td>
</tr>
<tr> <tr>
<td>数据库名Database</td> <td>数据库名Database</td>
<td> <td>
<input type="text" name="database" maxlength="100" placeholder="default" :value="config.database"/> <input type="text" name="database" maxlength="100" placeholder="default" v-model="form.database" />
<p class="comment">logs_ingest 表所在库,默认 default。</p> <p class="comment">logs_ingest 表所在库,默认 default。</p>
</td> </td>
</tr> </tr>
</table> </table>
<div style="display:flex; align-items:center; gap:0.5em; margin-top:1em">
<submit-btn></submit-btn> <submit-btn></submit-btn>
<button type="button" class="ui button basic blue" @click="testConnection" :disabled="isTesting">
<i :class="isTesting ? 'icon spinner loading' : 'icon plug'"></i> 测试连接
</button>
<span v-if="testResult"
:style="{color: testOk ? '#21ba45' : '#db2828', fontWeight:'bold'}">{{testResult}}</span>
</div>
</form> </form>

View File

@@ -1,4 +1,22 @@
Tea.context(function () { Tea.context(function () {
var config = this.config || {}
this.form = {
host: config.host || "",
scheme: config.scheme || "https",
port: config.port > 0 ? config.port : 8443,
user: config.user || "",
password: "",
database: config.database || "default",
}
this.isTesting = false
this.testResult = ""
this.testOk = false
// 页面加载时的连接状态(后端自动检测)
this.connStatus = this.connStatus || "unconfigured"
this.connError = this.connError || ""
this.success = function () { this.success = function () {
teaweb.success("保存成功") teaweb.success("保存成功")
} }
@@ -9,4 +27,50 @@ Tea.context(function () {
Tea.Vue.success() Tea.Vue.success()
}) })
} }
this.testConnection = function () {
var that = Tea.Vue
that.isTesting = true
that.testResult = ""
var form = document.querySelector("form")
var fd = new FormData(form)
fd.set("host", that.form.host || "")
fd.set("scheme", that.form.scheme || "https")
fd.set("port", String(that.form.port > 0 ? that.form.port : 8443))
fd.set("user", that.form.user || "")
fd.set("password", that.form.password || "")
fd.set("database", that.form.database || "default")
var xhr = new XMLHttpRequest()
xhr.open("POST", Tea.url("/db/testClickhouse"), true)
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest")
xhr.timeout = 10000
xhr.onload = function () {
that.isTesting = false
try {
var resp = JSON.parse(xhr.responseText)
if (resp.code === 200) {
that.testOk = true
that.testResult = "✅ 连接成功"
} else {
that.testOk = false
that.testResult = "❌ " + (resp.message || "连接失败")
}
} catch (e) {
that.testOk = false
that.testResult = "❌ 响应解析失败"
}
}
xhr.onerror = function () {
that.isTesting = false
that.testOk = false
that.testResult = "❌ 网络请求失败"
}
xhr.ontimeout = function () {
that.isTesting = false
that.testOk = false
that.testResult = "❌ 请求超时"
}
xhr.send(fd)
}
}) })

View File

@@ -19,6 +19,7 @@ type NSNodeConfig struct {
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"` AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"` AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"`
AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"` AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"`
AccessLogRotate *serverconfigs.AccessLogRotateConfig `yaml:"accessLogRotate" json:"accessLogRotate"`
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"` RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"` DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"` AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`

View File

@@ -2,8 +2,65 @@
package serverconfigs package serverconfigs
const (
DefaultAccessLogRotateMaxSizeMB = 256
DefaultAccessLogRotateMaxBackups = 14
DefaultAccessLogRotateMaxAgeDays = 7
)
// AccessLogRotateConfig 文件轮转配置。
type AccessLogRotateConfig struct {
MaxSizeMB int `yaml:"maxSizeMB" json:"maxSizeMB"` // 单文件最大大小MB
MaxBackups int `yaml:"maxBackups" json:"maxBackups"` // 保留历史文件数
MaxAgeDays int `yaml:"maxAgeDays" json:"maxAgeDays"` // 保留天数
Compress *bool `yaml:"compress" json:"compress"` // 是否压缩历史文件
LocalTime *bool `yaml:"localTime" json:"localTime"` // 轮转时间使用本地时区
}
// AccessLogFileStorageConfig 文件存储配置 // AccessLogFileStorageConfig 文件存储配置
type AccessLogFileStorageConfig struct { type AccessLogFileStorageConfig struct {
Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second} Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second}
AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录 AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 文件轮转配置
}
// NewDefaultAccessLogRotateConfig 默认轮转配置。
func NewDefaultAccessLogRotateConfig() *AccessLogRotateConfig {
compress := false
localTime := true
return &AccessLogRotateConfig{
MaxSizeMB: DefaultAccessLogRotateMaxSizeMB,
MaxBackups: DefaultAccessLogRotateMaxBackups,
MaxAgeDays: DefaultAccessLogRotateMaxAgeDays,
Compress: &compress,
LocalTime: &localTime,
}
}
// Normalize 归一化轮转配置,空值/非法值回退默认。
func (c *AccessLogRotateConfig) Normalize() *AccessLogRotateConfig {
defaultConfig := NewDefaultAccessLogRotateConfig()
if c == nil {
return defaultConfig
}
if c.MaxSizeMB > 0 {
defaultConfig.MaxSizeMB = c.MaxSizeMB
}
if c.MaxBackups > 0 {
defaultConfig.MaxBackups = c.MaxBackups
}
if c.MaxAgeDays > 0 {
defaultConfig.MaxAgeDays = c.MaxAgeDays
}
if c.Compress != nil {
v := *c.Compress
defaultConfig.Compress = &v
}
if c.LocalTime != nil {
v := *c.LocalTime
defaultConfig.LocalTime = &v
}
return defaultConfig
} }

View File

@@ -26,6 +26,7 @@ func NewGlobalServerConfig() *GlobalServerConfig {
config.HTTPAccessLog.EnableResponseHeaders = true config.HTTPAccessLog.EnableResponseHeaders = true
config.HTTPAccessLog.EnableCookies = true config.HTTPAccessLog.EnableCookies = true
config.HTTPAccessLog.EnableServerNotFound = true config.HTTPAccessLog.EnableServerNotFound = true
config.HTTPAccessLog.Rotate = NewDefaultAccessLogRotateConfig()
config.Log.RecordServerError = false config.Log.RecordServerError = false
@@ -79,6 +80,7 @@ type GlobalServerConfig struct {
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志 EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse双写/单写) WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse双写/单写)
FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用) FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用)
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 本地日志轮转配置lumberjack
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置 } `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
Stat struct { Stat struct {

View File

@@ -107,11 +107,12 @@ function copy_fluent_bit_assets() {
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT" echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
return 1 return 1
fi fi
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
rm -rf "$FLUENT_DIST" rm -rf "$FLUENT_DIST"
mkdir -p "$FLUENT_DIST" mkdir -p "$FLUENT_DIST"
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
if [ -f "$FLUENT_ROOT/$file" ]; then if [ -f "$FLUENT_ROOT/$file" ]; then
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/" cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
fi fi
@@ -129,6 +130,43 @@ function copy_fluent_bit_assets() {
fi fi
fi fi
rm -f "$FLUENT_DIST/.gitignore"
rm -f "$FLUENT_DIST"/logs.db*
rm -rf "$FLUENT_DIST/storage"
return 0
}
function verify_fluent_bit_package_matrix() {
FLUENT_ROOT=$1
ARCH=$2
REQUIRED_FILES=()
if [ "$ARCH" = "amd64" ]; then
REQUIRED_FILES=(
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
)
elif [ "$ARCH" = "arm64" ]; then
REQUIRED_FILES=(
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
)
else
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
return 1
fi
MISSING=0
for FILE in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
MISSING=1
fi
done
if [ "$MISSING" -ne 0 ]; then
return 1
fi
return 0 return 0
} }

View File

@@ -13,6 +13,7 @@ require (
github.com/mdlayher/netlink v1.7.2 github.com/mdlayher/netlink v1.7.2
github.com/miekg/dns v1.1.58 github.com/miekg/dns v1.1.58
github.com/shirou/gopsutil/v3 v3.24.2 github.com/shirou/gopsutil/v3 v3.24.2
gopkg.in/natefinch/lumberjack.v2 v2.2.1
golang.org/x/sys v0.38.0 golang.org/x/sys v0.38.0
google.golang.org/grpc v1.78.0 google.golang.org/grpc v1.78.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1

View File

@@ -111,6 +111,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -9,7 +9,9 @@ import (
"sync" "sync"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs" "github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
"gopkg.in/natefinch/lumberjack.v2"
) )
const ( const (
@@ -30,18 +32,23 @@ func SharedDNSFileWriter() *DNSFileWriter {
return sharedDNSFileWriter return sharedDNSFileWriter
} }
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集. // DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集
// 文件轮转由 lumberjack 内建完成。
type DNSFileWriter struct { type DNSFileWriter struct {
dir string dir string
mu sync.Mutex mu sync.Mutex
file *os.File file *lumberjack.Logger
rotateConfig *serverconfigs.AccessLogRotateConfig
inited bool inited bool
} }
// NewDNSFileWriter 创建 DNS 本地日志写入器. // NewDNSFileWriter 创建 DNS 本地日志写入器.
func NewDNSFileWriter() *DNSFileWriter { func NewDNSFileWriter() *DNSFileWriter {
logDir := resolveDefaultDNSLogDir() logDir := resolveDefaultDNSLogDir()
return &DNSFileWriter{dir: logDir} return &DNSFileWriter{
dir: logDir,
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
}
} }
func resolveDefaultDNSLogDir() string { func resolveDefaultDNSLogDir() string {
@@ -102,6 +109,25 @@ func (w *DNSFileWriter) SetDir(dir string) {
w.dir = dir w.dir = dir
} }
// SetRotateConfig 更新日志轮转配置并重建 writer。
func (w *DNSFileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
normalized := config.Normalize()
w.mu.Lock()
defer w.mu.Unlock()
if equalDNSRotateConfig(w.rotateConfig, normalized) {
return
}
if w.file != nil {
_ = w.file.Close()
w.file = nil
}
w.inited = false
w.rotateConfig = normalized
}
// EnsureInit 在启动时预创建目录与 access.log. // EnsureInit 在启动时预创建目录与 access.log.
func (w *DNSFileWriter) EnsureInit() error { func (w *DNSFileWriter) EnsureInit() error {
if w.dir == "" { if w.dir == "" {
@@ -127,13 +153,16 @@ func (w *DNSFileWriter) init() error {
return err return err
} }
fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) rotateConfig := w.rotateConfig.Normalize()
if err != nil { w.file = &lumberjack.Logger{
remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error()) Filename: filepath.Join(w.dir, "access.log"),
return err MaxSize: rotateConfig.MaxSizeMB,
MaxBackups: rotateConfig.MaxBackups,
MaxAge: rotateConfig.MaxAgeDays,
Compress: *rotateConfig.Compress,
LocalTime: *rotateConfig.LocalTime,
} }
w.file = fp
w.inited = true w.inited = true
return nil return nil
} }
@@ -148,14 +177,14 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
} }
w.mu.Lock() w.mu.Lock()
fp := w.file file := w.file
w.mu.Unlock() w.mu.Unlock()
if fp == nil { if file == nil {
return return
} }
for _, log := range logs { for _, logItem := range logs {
ingestLog := FromNSAccessLog(log, clusterId) ingestLog := FromNSAccessLog(logItem, clusterId)
if ingestLog == nil { if ingestLog == nil {
continue continue
} }
@@ -163,11 +192,11 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
if err != nil { if err != nil {
continue continue
} }
_, _ = fp.Write(append(line, '\n')) _, _ = file.Write(append(line, '\n'))
} }
} }
// Reopen 关闭并重新打开日志文件(配合 logrotate. // Reopen 关闭并重建日志 writer供兼容调用.
func (w *DNSFileWriter) Reopen() error { func (w *DNSFileWriter) Reopen() error {
w.mu.Lock() w.mu.Lock()
if w.file != nil { if w.file != nil {
@@ -197,3 +226,14 @@ func (w *DNSFileWriter) Close() error {
} }
return nil return nil
} }
func equalDNSRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
if left == nil || right == nil {
return left == right
}
return left.MaxSizeMB == right.MaxSizeMB &&
left.MaxBackups == right.MaxBackups &&
left.MaxAgeDays == right.MaxAgeDays &&
*left.Compress == *right.Compress &&
*left.LocalTime == *right.LocalTime
}

View File

@@ -110,6 +110,7 @@ func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) {
teaconst.IsPlus = config.IsPlus teaconst.IsPlus = config.IsPlus
accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath) accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
accesslogs.SharedDNSFileWriter().SetRotateConfig(config.AccessLogRotate)
needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
if needWriteFile { if needWriteFile {

View File

@@ -181,11 +181,12 @@ function copy_fluent_bit_assets() {
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT" echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
return 1 return 1
fi fi
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
rm -rf "$FLUENT_DIST" rm -rf "$FLUENT_DIST"
mkdir -p "$FLUENT_DIST" mkdir -p "$FLUENT_DIST"
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
if [ -f "$FLUENT_ROOT/$file" ]; then if [ -f "$FLUENT_ROOT/$file" ]; then
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/" cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
fi fi
@@ -203,6 +204,43 @@ function copy_fluent_bit_assets() {
fi fi
fi fi
rm -f "$FLUENT_DIST/.gitignore"
rm -f "$FLUENT_DIST"/logs.db*
rm -rf "$FLUENT_DIST/storage"
return 0
}
function verify_fluent_bit_package_matrix() {
FLUENT_ROOT=$1
ARCH=$2
REQUIRED_FILES=()
if [ "$ARCH" = "amd64" ]; then
REQUIRED_FILES=(
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
)
elif [ "$ARCH" = "arm64" ]; then
REQUIRED_FILES=(
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
)
else
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
return 1
fi
MISSING=0
for FILE in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
MISSING=1
fi
done
if [ "$MISSING" -ne 0 ]; then
return 1
fi
return 0 return 0
} }

View File

@@ -38,6 +38,7 @@ require (
github.com/shirou/gopsutil/v3 v3.22.2 github.com/shirou/gopsutil/v3 v3.22.2
github.com/tdewolff/minify/v2 v2.20.20 github.com/tdewolff/minify/v2 v2.20.20
github.com/tencentyun/cos-go-sdk-v5 v0.7.41 github.com/tencentyun/cos-go-sdk-v5 v0.7.41
gopkg.in/natefinch/lumberjack.v2 v2.2.1
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
golang.org/x/image v0.15.0 golang.org/x/image v0.15.0
golang.org/x/net v0.47.0 golang.org/x/net v0.47.0

View File

@@ -305,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@@ -9,7 +9,9 @@ import (
"sync" "sync"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs" "github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"gopkg.in/natefinch/lumberjack.v2"
) )
var ( var (
@@ -30,11 +32,13 @@ const (
envLogDir = "EDGE_LOG_DIR" envLogDir = "EDGE_LOG_DIR"
) )
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 logrotate 与 Fluent Bit 采集 // FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 Fluent Bit 采集
// 文件轮转由 lumberjack 内建完成。
type FileWriter struct { type FileWriter struct {
dir string dir string
mu sync.Mutex mu sync.Mutex
files map[string]*os.File // access.log, waf.log, error.log files map[string]*lumberjack.Logger // access.log, waf.log, error.log
rotateConfig *serverconfigs.AccessLogRotateConfig
inited bool inited bool
} }
@@ -43,7 +47,8 @@ func NewFileWriter() *FileWriter {
dir := resolveDefaultLogDir() dir := resolveDefaultLogDir()
return &FileWriter{ return &FileWriter{
dir: dir, dir: dir,
files: make(map[string]*os.File), files: make(map[string]*lumberjack.Logger),
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
} }
} }
@@ -97,9 +102,9 @@ func (w *FileWriter) SetDir(dir string) {
return return
} }
for name, f := range w.files { for name, file := range w.files {
if f != nil { if file != nil {
_ = f.Close() _ = file.Close()
} }
w.files[name] = nil w.files[name] = nil
} }
@@ -107,6 +112,27 @@ func (w *FileWriter) SetDir(dir string) {
w.dir = dir w.dir = dir
} }
// SetRotateConfig 更新日志轮转配置并重建 writer。
func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
normalized := config.Normalize()
w.mu.Lock()
defer w.mu.Unlock()
if equalRotateConfig(w.rotateConfig, normalized) {
return
}
for name, file := range w.files {
if file != nil {
_ = file.Close()
}
w.files[name] = nil
}
w.inited = false
w.rotateConfig = normalized
}
// IsEnabled 是否启用落盘(目录非空即视为启用) // IsEnabled 是否启用落盘(目录非空即视为启用)
func (w *FileWriter) IsEnabled() bool { func (w *FileWriter) IsEnabled() bool {
return w.dir != "" return w.dir != ""
@@ -138,17 +164,24 @@ func (w *FileWriter) init() error {
if w.files[name] != nil { if w.files[name] != nil {
continue continue
} }
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) w.files[name] = w.newLogger(name)
if err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
continue
}
w.files[name] = fp
} }
w.inited = true w.inited = true
return nil return nil
} }
func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger {
rotateConfig := w.rotateConfig.Normalize()
return &lumberjack.Logger{
Filename: filepath.Join(w.dir, fileName),
MaxSize: rotateConfig.MaxSizeMB,
MaxBackups: rotateConfig.MaxBackups,
MaxAge: rotateConfig.MaxAgeDays,
Compress: *rotateConfig.Compress,
LocalTime: *rotateConfig.LocalTime,
}
}
// Write 将一条访问日志按 log_type 写入对应文件access.log / waf.log / error.log // Write 将一条访问日志按 log_type 写入对应文件access.log / waf.log / error.log
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) { func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
if w.dir == "" { if w.dir == "" {
@@ -173,13 +206,12 @@ func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
fileName = "access.log" fileName = "access.log"
} }
w.mu.Lock() w.mu.Lock()
fp := w.files[fileName] file := w.files[fileName]
w.mu.Unlock() w.mu.Unlock()
if fp == nil { if file == nil {
return return
} }
// 单行写入,末尾换行,便于 Fluent Bit / JSON 解析 _, err = file.Write(append(line, '\n'))
_, err = fp.Write(append(line, '\n'))
if err != nil { if err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error()) remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
} }
@@ -194,49 +226,49 @@ func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
return return
} }
w.mu.Lock() w.mu.Lock()
accessFp := w.files["access.log"] accessFile := w.files["access.log"]
wafFp := w.files["waf.log"] wafFile := w.files["waf.log"]
errorFp := w.files["error.log"] errorFile := w.files["error.log"]
w.mu.Unlock() w.mu.Unlock()
if accessFp == nil && wafFp == nil && errorFp == nil { if accessFile == nil && wafFile == nil && errorFile == nil {
return return
} }
for _, l := range logs { for _, logItem := range logs {
ingest, logType := FromHTTPAccessLog(l, clusterId) ingest, logType := FromHTTPAccessLog(logItem, clusterId)
line, err := json.Marshal(ingest) line, err := json.Marshal(ingest)
if err != nil { if err != nil {
continue continue
} }
line = append(line, '\n') line = append(line, '\n')
var fp *os.File var file *lumberjack.Logger
switch logType { switch logType {
case LogTypeWAF: case LogTypeWAF:
fp = wafFp file = wafFile
case LogTypeError: case LogTypeError:
fp = errorFp file = errorFile
default: default:
fp = accessFp file = accessFile
} }
if fp != nil { if file != nil {
_, _ = fp.Write(line) _, _ = file.Write(line)
} }
} }
} }
// Reopen 关闭并重新打开所有日志文件(供 logrotate copytruncate 或 SIGHUP 后重开句柄) // Reopen 关闭并重所有日志 writer SIGHUP 兼容调用)。
func (w *FileWriter) Reopen() error { func (w *FileWriter) Reopen() error {
if w.dir == "" { if w.dir == "" {
return nil return nil
} }
w.mu.Lock() w.mu.Lock()
defer w.mu.Unlock() for name, file := range w.files {
for name, f := range w.files { if file != nil {
if f != nil { _ = file.Close()
_ = f.Close()
w.files[name] = nil w.files[name] = nil
} }
} }
w.inited = false w.inited = false
w.mu.Unlock()
return w.init() return w.init()
} }
@@ -245,9 +277,9 @@ func (w *FileWriter) Close() error {
w.mu.Lock() w.mu.Lock()
defer w.mu.Unlock() defer w.mu.Unlock()
var lastErr error var lastErr error
for name, f := range w.files { for name, file := range w.files {
if f != nil { if file != nil {
if err := f.Close(); err != nil { if err := file.Close(); err != nil {
lastErr = err lastErr = err
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err)) remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
} }
@@ -257,3 +289,14 @@ func (w *FileWriter) Close() error {
w.inited = false w.inited = false
return lastErr return lastErr
} }
func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
if left == nil || right == nil {
return left == right
}
return left.MaxSizeMB == right.MaxSizeMB &&
left.MaxBackups == right.MaxBackups &&
left.MaxAgeDays == right.MaxAgeDays &&
*left.Compress == *right.Compress &&
*left.LocalTime == *right.LocalTime
}

View File

@@ -577,7 +577,7 @@ func (this *Node) listenSignals() {
goman.New(func() { goman.New(func() {
for sig := range queue { for sig := range queue {
if sig == syscall.SIGHUP { if sig == syscall.SIGHUP {
// 供 logrotate 等旋转日志后重开句柄 // 兼容 SIGHUP重建本地日志 writer
if err := accesslogs.SharedFileWriter().Reopen(); err != nil { if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
remotelogs.Error("NODE", "access log file reopen: "+err.Error()) remotelogs.Error("NODE", "access log file reopen: "+err.Error())
} }
@@ -890,6 +890,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) {
var accessLogFilePath string var accessLogFilePath string
if config != nil && config.GlobalServerConfig != nil { if config != nil && config.GlobalServerConfig != nil {
accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
accesslogs.SharedFileWriter().SetRotateConfig(config.GlobalServerConfig.HTTPAccessLog.Rotate)
} }
accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath) accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)

1984
config.xml Normal file

File diff suppressed because it is too large Load Diff

348
deploy.sh Normal file
View File

@@ -0,0 +1,348 @@
#!/bin/bash
# CloudWAF 部署脚本 - Gname DNS 提供商更新
# 使用方法: ./deploy.sh [zip文件路径]
set -e # 遇到错误立即退出
# ========== 配置区域 ==========
# 支持环境变量和配置文件
# 从环境变量读取
INSTALL_BASE_DIR="${BRAND_INSTALL_PATH:-/usr/local/goedge}"
# 从配置文件读取(如果存在)
if [ -f "/etc/goedge/brand.conf" ]; then
source /etc/goedge/brand.conf
if [ -n "$BRAND_INSTALL_PATH" ]; then
INSTALL_BASE_DIR="$BRAND_INSTALL_PATH"
fi
fi
# 如果通过参数传入 zip 文件路径,则使用参数;否则使用默认路径
ZIP_FILE_RAW="${1:-/tmp/edge-admin-linux-amd64-plus-v1.3.8.zip}"
# 获取绝对路径,防止 cd 到临时目录后找不到文件
if [[ "$ZIP_FILE_RAW" = /* ]]; then
ZIP_FILE="$ZIP_FILE_RAW"
else
ZIP_FILE="$(pwd)/$ZIP_FILE_RAW"
fi
TARGET_DIR="${INSTALL_BASE_DIR}/edge-admin"
BACKUP_DIR="${INSTALL_BASE_DIR}/backup_$(date +%Y%m%d_%H%M%S)"
TEMP_DIR="/tmp/edge-admin-update-$(date +%s)"
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# 日志函数
log_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# ========== 检查前置条件 ==========
check_prerequisites() {
log_info "检查前置条件..."
# 检查 zip 文件是否存在
if [ ! -f "$ZIP_FILE" ]; then
log_error "未找到更新包: $ZIP_FILE"
log_info "使用方法: $0 [zip文件路径]"
exit 1
fi
# 检查目标目录是否存在
if [ ! -d "$TARGET_DIR" ]; then
log_error "目标目录不存在: $TARGET_DIR"
exit 1
fi
# 检查是否有足够的磁盘空间(至少 500MB
available_space=$(df "$TARGET_DIR" | tail -1 | awk '{print $4}')
if [ "$available_space" -lt 512000 ]; then
log_warn "可用磁盘空间不足 500MB可能影响部署"
fi
log_info "前置条件检查通过"
}
# ========== 1. 备份关键目录 ==========
backup_files() {
log_info "=========================================="
log_info "步骤 1: 备份关键目录"
log_info "=========================================="
mkdir -p "$BACKUP_DIR"
# 备份 EdgeAdmin 配置
if [ -d "$TARGET_DIR/configs" ]; then
log_info "备份 EdgeAdmin 配置文件..."
cp -r "$TARGET_DIR/configs" "$BACKUP_DIR/edge-admin-configs"
log_info "✅ EdgeAdmin 配置已备份"
else
log_warn "EdgeAdmin configs 目录不存在,跳过备份"
fi
# 备份 EdgeAdmin 日志(可选)
if [ -d "$TARGET_DIR/logs" ]; then
log_info "备份 EdgeAdmin 日志文件..."
cp -r "$TARGET_DIR/logs" "$BACKUP_DIR/edge-admin-logs" 2>/dev/null || true
fi
# 备份 EdgeAPI 配置和数据
if [ -d "$TARGET_DIR/edge-api" ]; then
if [ -d "$TARGET_DIR/edge-api/configs" ]; then
log_info "备份 EdgeAPI 配置文件..."
cp -r "$TARGET_DIR/edge-api/configs" "$BACKUP_DIR/edge-api-configs"
log_info "✅ EdgeAPI 配置已备份"
fi
if [ -d "$TARGET_DIR/edge-api/logs" ]; then
log_info "备份 EdgeAPI 日志文件..."
cp -r "$TARGET_DIR/edge-api/logs" "$BACKUP_DIR/edge-api-logs" 2>/dev/null || true
fi
if [ -d "$TARGET_DIR/edge-api/data" ]; then
log_info "备份 EdgeAPI 数据文件..."
cp -r "$TARGET_DIR/edge-api/data" "$BACKUP_DIR/edge-api-data" 2>/dev/null || true
fi
fi
log_info "✅ 备份完成,备份位置: $BACKUP_DIR"
echo ""
}
# ========== 2. 停止服务 ==========
stop_services() {
log_info "=========================================="
log_info "步骤 2: 停止服务"
log_info "=========================================="
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
log_info "停止 EdgeAdmin 服务..."
"$TARGET_DIR/bin/edge-admin" stop || true
sleep 2
# 检查是否还有进程在运行 (精确匹配二进制路径,避免杀死解析 zip 路径的脚本自身)
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
log_warn "检测到 edge-admin 进程仍在运行,尝试强制停止..."
pkill -9 -f "$TARGET_DIR/bin/edge-admin" || true
sleep 1
fi
log_info "✅ 服务已停止"
else
log_warn "未找到 edge-admin 可执行文件,跳过停止步骤"
fi
echo ""
}
# ========== 3. 解压新版本 ==========
extract_package() {
log_info "=========================================="
log_info "步骤 3: 解压新版本"
log_info "=========================================="
log_info "解压更新包到临时目录: $TEMP_DIR"
mkdir -p "$TEMP_DIR"
cd "$TEMP_DIR"
if ! unzip -q "$ZIP_FILE"; then
log_error "解压失败,请检查 zip 文件是否损坏"
exit 1
fi
log_info "✅ 解压完成"
echo ""
}
# ========== 4. 替换文件 ==========
replace_files() {
log_info "=========================================="
log_info "步骤 4: 替换文件"
log_info "=========================================="
# 替换 EdgeAdmin bin
if [ -d "$TEMP_DIR/edge-admin/bin" ]; then
log_info "替换 EdgeAdmin 可执行文件..."
cp -r "$TEMP_DIR/edge-admin/bin"/* "$TARGET_DIR/bin/"
log_info "✅ EdgeAdmin bin 已更新"
fi
# 替换 EdgeAdmin web排除 tmp
if [ -d "$TEMP_DIR/edge-admin/web" ]; then
log_info "替换 EdgeAdmin 前端文件..."
if command -v rsync > /dev/null; then
rsync -av --exclude='tmp' \
"$TEMP_DIR/edge-admin/web/" "$TARGET_DIR/web/"
else
# 如果没有 rsync使用 cp
cp -r "$TEMP_DIR/edge-admin/web"/* "$TARGET_DIR/web/" 2>/dev/null || true
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
fi
# 清空 tmp 目录
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
log_info "✅ EdgeAdmin web 已更新"
fi
# 替换 EdgeAPI 文件
if [ -d "$TEMP_DIR/edge-admin/edge-api" ]; then
log_info "替换 EdgeAPI 文件..."
# 确保 edge-api 目录存在
mkdir -p "$TARGET_DIR/edge-api"
# 替换 bin
if [ -d "$TEMP_DIR/edge-admin/edge-api/bin" ]; then
mkdir -p "$TARGET_DIR/edge-api/bin"
cp -r "$TEMP_DIR/edge-admin/edge-api/bin"/* \
"$TARGET_DIR/edge-api/bin/" 2>/dev/null || true
log_info "✅ EdgeAPI bin 已更新"
fi
# 替换 deploy节点安装包
if [ -d "$TEMP_DIR/edge-admin/edge-api/deploy" ]; then
mkdir -p "$TARGET_DIR/edge-api/deploy"
cp -r "$TEMP_DIR/edge-admin/edge-api/deploy"/* \
"$TARGET_DIR/edge-api/deploy/" 2>/dev/null || true
log_info "✅ EdgeAPI deploy 已更新"
fi
# 替换 installers安装工具
if [ -d "$TEMP_DIR/edge-admin/edge-api/installers" ]; then
mkdir -p "$TARGET_DIR/edge-api/installers"
cp -r "$TEMP_DIR/edge-admin/edge-api/installers"/* \
"$TARGET_DIR/edge-api/installers/" 2>/dev/null || true
log_info "✅ EdgeAPI installers 已更新"
fi
fi
log_info "✅ 文件替换完成"
echo ""
}
# ========== 5. 恢复配置文件 ==========
restore_configs() {
log_info "=========================================="
log_info "步骤 5: 恢复配置文件"
log_info "=========================================="
# 恢复 EdgeAdmin 配置
if [ -d "$BACKUP_DIR/edge-admin-configs" ]; then
log_info "恢复 EdgeAdmin 配置文件..."
cp -r "$BACKUP_DIR/edge-admin-configs"/* "$TARGET_DIR/configs/"
log_info "✅ EdgeAdmin 配置已恢复"
else
log_warn "未找到 EdgeAdmin 配置备份,请手动检查配置文件"
fi
# 恢复 EdgeAPI 配置
if [ -d "$BACKUP_DIR/edge-api-configs" ]; then
log_info "恢复 EdgeAPI 配置文件..."
mkdir -p "$TARGET_DIR/edge-api/configs"
cp -r "$BACKUP_DIR/edge-api-configs"/* "$TARGET_DIR/edge-api/configs/" 2>/dev/null || true
log_info "✅ EdgeAPI 配置已恢复"
else
log_warn "未找到 EdgeAPI 配置备份,如果存在 edge-api 目录,请手动检查配置文件"
fi
log_info "✅ 配置文件恢复完成"
echo ""
}
# ========== 6. 清理临时文件 ==========
cleanup() {
log_info "=========================================="
log_info "步骤 6: 清理临时文件"
log_info "=========================================="
rm -rf "$TEMP_DIR"
log_info "✅ 临时文件已清理"
echo ""
}
# ========== 7. 启动服务 ==========
start_services() {
log_info "=========================================="
log_info "步骤 7: 启动服务"
log_info "=========================================="
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
log_info "启动 EdgeAdmin 服务..."
"$TARGET_DIR/bin/edge-admin" start
sleep 3
# 检查服务状态
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
log_info "✅ 服务启动成功"
else
log_warn "服务可能未正常启动,请检查日志"
log_info "查看日志: tail -f $TARGET_DIR/logs/run.log"
fi
else
log_error "未找到 edge-admin 可执行文件"
exit 1
fi
echo ""
}
# ========== 主函数 ==========
main() {
echo ""
log_info "=========================================="
log_info "CloudWAF 部署脚本 - Gname DNS 提供商更新"
log_info "=========================================="
echo ""
log_info "部署配置:"
log_info " ZIP 文件: $ZIP_FILE"
log_info " 目标目录: $TARGET_DIR"
log_info " 备份目录: $BACKUP_DIR"
echo ""
# 确认操作
read -p "确认开始部署? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
log_info "部署已取消"
exit 0
fi
echo ""
# 执行部署步骤
check_prerequisites
backup_files
stop_services
extract_package
replace_files
restore_configs
cleanup
start_services
# 完成
log_info "=========================================="
log_info "✅ 部署完成!"
log_info "=========================================="
echo ""
log_info "备份位置: $BACKUP_DIR"
log_info "请检查服务运行状态和日志"
echo ""
log_info "验证步骤:"
log_info "1. 检查服务进程: ps aux | grep \"$TARGET_DIR/bin/edge-admin\""
log_info "2. 检查服务日志: tail -f $TARGET_DIR/logs/run.log"
log_info "3. 访问管理后台,测试 Gname DNS 提供商功能"
echo ""
}
# 执行主函数
main

111
deploy/clickhouse/README.md Normal file
View File

@@ -0,0 +1,111 @@
# ClickHouse + Fluent Bit 使用手册Ubuntu 22.04 / Amazon Linux 2023
## 1. 支持范围
- Ubuntu 22.04
- Amazon Linux 2023AWS
安装脚本:`install_clickhouse_linux.sh`(自动识别上述系统)。
## 2. 安装 ClickHouse
```bash
cd /path/to/waf-platform/deploy/clickhouse
chmod +x install_clickhouse_linux.sh
sudo ./install_clickhouse_linux.sh
```
可选:安装时初始化 `default` 用户密码:
```bash
sudo CLICKHOUSE_DEFAULT_PASSWORD='YourStrongPassword' ./install_clickhouse_linux.sh
```
## 3. 开启 HTTPS默认仅 crt+key
脚本默认生成 `server.crt + server.key`(带 SAN并启用 8443
```bash
cd /path/to/waf-platform/deploy/clickhouse
chmod +x configure_clickhouse_https.sh
sudo CH_HTTPS_PORT=8443 \
CH_CERT_CN=clickhouse.example.com \
CH_CERT_DNS=clickhouse.example.com \
CH_CERT_IP=<CLICKHOUSE_IP> \
./configure_clickhouse_https.sh
```
使用已有证书:
```bash
sudo SRC_CERT=/path/to/server.crt \
SRC_KEY=/path/to/server.key \
CH_HTTPS_PORT=8443 \
./configure_clickhouse_https.sh
```
## 4. 初始化日志表(含优化)
```bash
cd /path/to/waf-platform/deploy/clickhouse
chmod +x init_waf_logs_tables.sh
sudo CH_HOST=127.0.0.1 \
CH_PORT=9000 \
CH_USER=default \
CH_PASSWORD='YourStrongPassword' \
CH_DATABASE=default \
./init_waf_logs_tables.sh
```
说明:
- `init_waf_logs_tables.sql` 已内置主要优化(`CODEC``LowCardinality`、跳数索引)。
- `optimize_schema.sql` 主要用于历史表补齐优化,不是首次建表必需步骤。
## 5. 平台侧配置EdgeAdmin
在 ClickHouse 设置页配置:
- HostClickHouse 地址
- Port`8443`
- Database`default`
- Scheme`https`
当前实现说明:
- 前端不再提供 `TLS跳过校验``TLS Server Name` 配置项。
- 后端固定 `TLSSkipVerify=true`(默认不校验证书)。
保存后点击“测试连接”。
## 6. Fluent Bit 配置方式
推荐平台托管模式(在线安装/升级 Node、DNS 时自动下发):
- `/etc/fluent-bit/fluent-bit.conf`
- `/etc/fluent-bit/.edge-managed.env`
- `/etc/fluent-bit/.edge-managed.json`
检查状态:
```bash
sudo systemctl status fluent-bit --no-pager
sudo cat /etc/fluent-bit/.edge-managed.json
```
## 7. 验证与排障
查看 Fluent Bit 日志:
```bash
sudo journalctl -u fluent-bit -f
```
查看写入:
```sql
SELECT count() FROM default.logs_ingest;
SELECT count() FROM default.dns_logs_ingest;
```
常见错误:
- `connection refused`8443 未监听或网络未放行。
- `legacy Common Name`:证书缺 SAN需重签。

View File

@@ -0,0 +1,227 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${EUID}" -ne 0 ]]; then
echo "[ERROR] please run as root"
exit 1
fi
if [[ ! -f /etc/os-release ]]; then
echo "[ERROR] /etc/os-release not found"
exit 1
fi
# shellcheck disable=SC1091
source /etc/os-release
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
os_ver="${VERSION_ID:-}"
is_ubuntu22=false
is_amzn2023=false
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
is_ubuntu22=true
fi
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
is_amzn2023=true
fi
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
exit 1
fi
if ! command -v openssl >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then
if [[ "${is_ubuntu22}" == "true" ]]; then
apt-get update -y
DEBIAN_FRONTEND=noninteractive apt-get install -y openssl curl ca-certificates
else
dnf makecache -y
dnf install -y openssl curl ca-certificates
fi
fi
CH_HTTPS_PORT="${CH_HTTPS_PORT:-8443}"
CH_LISTEN_HOST="${CH_LISTEN_HOST:-::}"
CH_CERT_CN="${CH_CERT_CN:-$(hostname -f 2>/dev/null || hostname)}"
CH_CERT_DNS="${CH_CERT_DNS:-}"
CH_CERT_IP="${CH_CERT_IP:-}"
CH_CERT_DAYS="${CH_CERT_DAYS:-825}"
CH_GENERATE_CA="${CH_GENERATE_CA:-false}"
SRC_CERT="${SRC_CERT:-}"
SRC_KEY="${SRC_KEY:-}"
SRC_CA="${SRC_CA:-}"
CH_DIR="/etc/clickhouse-server"
CH_CONFIG_D_DIR="${CH_DIR}/config.d"
PKI_DIR="${CH_DIR}/pki"
SERVER_CERT="${CH_DIR}/server.crt"
SERVER_KEY="${CH_DIR}/server.key"
CA_CERT="${CH_DIR}/ca.crt"
OVERRIDE_FILE="${CH_CONFIG_D_DIR}/waf-https.xml"
mkdir -p "${CH_CONFIG_D_DIR}" "${PKI_DIR}"
split_csv() {
local raw="$1"
if [[ -z "${raw}" ]]; then
return 0
fi
IFS=',' read -r -a arr <<<"${raw}"
for item in "${arr[@]}"; do
item="$(echo "${item}" | xargs)"
if [[ -n "${item}" ]]; then
echo "${item}"
fi
done
}
build_san_line() {
local san_entries=()
while IFS= read -r dns_item; do
san_entries+=("DNS:${dns_item}")
done < <(split_csv "${CH_CERT_DNS}")
while IFS= read -r ip_item; do
san_entries+=("IP:${ip_item}")
done < <(split_csv "${CH_CERT_IP}")
if [[ ${#san_entries[@]} -eq 0 ]]; then
san_entries+=("DNS:${CH_CERT_CN}")
fi
local san_line
san_line="$(IFS=,; echo "${san_entries[*]}")"
echo "${san_line}"
}
generate_self_signed_cert() {
echo "[INFO] generating self-signed server certificate (crt+key only) ..."
local server_key="${PKI_DIR}/server.key"
local server_csr="${PKI_DIR}/server.csr"
local server_crt="${PKI_DIR}/server.crt"
local ext_file="${PKI_DIR}/server.ext"
local san_line
san_line="$(build_san_line)"
openssl genrsa -out "${server_key}" 2048
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
cat >"${ext_file}" <<EOF
subjectAltName=${san_line}
keyUsage=digitalSignature,keyEncipherment
extendedKeyUsage=serverAuth
EOF
openssl x509 -req -in "${server_csr}" -signkey "${server_key}" \
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
cp -f "${server_crt}" "${SERVER_CERT}"
cp -f "${server_key}" "${SERVER_KEY}"
rm -f "${CA_CERT}"
}
generate_cert_with_ca() {
echo "[INFO] generating local CA and server certificate ..."
local ca_key="${PKI_DIR}/ca.key"
local ca_crt="${PKI_DIR}/ca.crt"
local server_key="${PKI_DIR}/server.key"
local server_csr="${PKI_DIR}/server.csr"
local server_crt="${PKI_DIR}/server.crt"
local ext_file="${PKI_DIR}/server.ext"
local san_line
san_line="$(build_san_line)"
openssl genrsa -out "${ca_key}" 4096
openssl req -x509 -new -nodes -key "${ca_key}" -sha256 -days 3650 \
-out "${ca_crt}" -subj "/CN=ClickHouse Local CA"
openssl genrsa -out "${server_key}" 2048
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
cat >"${ext_file}" <<EOF
subjectAltName=${san_line}
keyUsage=digitalSignature,keyEncipherment
extendedKeyUsage=serverAuth
EOF
openssl x509 -req -in "${server_csr}" -CA "${ca_crt}" -CAkey "${ca_key}" -CAcreateserial \
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
cp -f "${server_crt}" "${SERVER_CERT}"
cp -f "${server_key}" "${SERVER_KEY}"
cp -f "${ca_crt}" "${CA_CERT}"
}
if [[ -n "${SRC_CERT}" || -n "${SRC_KEY}" ]]; then
if [[ -z "${SRC_CERT}" || -z "${SRC_KEY}" ]]; then
echo "[ERROR] SRC_CERT and SRC_KEY must be provided together"
exit 1
fi
echo "[INFO] using provided certificate files ..."
cp -f "${SRC_CERT}" "${SERVER_CERT}"
cp -f "${SRC_KEY}" "${SERVER_KEY}"
if [[ -n "${SRC_CA}" ]]; then
cp -f "${SRC_CA}" "${CA_CERT}"
else
rm -f "${CA_CERT}"
fi
else
case "$(echo "${CH_GENERATE_CA}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on)
generate_cert_with_ca
;;
*)
generate_self_signed_cert
;;
esac
fi
chown clickhouse:clickhouse "${SERVER_CERT}" "${SERVER_KEY}" || true
chmod 0644 "${SERVER_CERT}"
chmod 0640 "${SERVER_KEY}"
if [[ -f "${CA_CERT}" ]]; then
chown clickhouse:clickhouse "${CA_CERT}" || true
chmod 0644 "${CA_CERT}"
fi
echo "[INFO] writing ClickHouse HTTPS override config ..."
cat >"${OVERRIDE_FILE}" <<EOF
<clickhouse>
<https_port>${CH_HTTPS_PORT}</https_port>
<listen_host>${CH_LISTEN_HOST}</listen_host>
<openSSL>
<server>
<certificateFile>${SERVER_CERT}</certificateFile>
<privateKeyFile>${SERVER_KEY}</privateKeyFile>
<verificationMode>none</verificationMode>
<loadDefaultCAFile>true</loadDefaultCAFile>
<cacheSessions>true</cacheSessions>
<disableProtocols>sslv2,sslv3</disableProtocols>
<preferServerCiphers>true</preferServerCiphers>
<invalidCertificateHandler>
<name>RejectCertificateHandler</name>
</invalidCertificateHandler>
</server>
</openSSL>
</clickhouse>
EOF
echo "[INFO] restarting clickhouse-server ..."
systemctl restart clickhouse-server
sleep 2
echo "[INFO] service status ..."
systemctl --no-pager -l status clickhouse-server | sed -n '1,15p'
echo "[INFO] verifying HTTPS endpoint ..."
curl -sk "https://127.0.0.1:${CH_HTTPS_PORT}/?query=SELECT%201" || true
echo
echo "[OK] ClickHouse HTTPS setup finished"
echo " HTTPS port : ${CH_HTTPS_PORT}"
echo " cert file : ${SERVER_CERT}"
echo " key file : ${SERVER_KEY}"
if [[ -f "${CA_CERT}" ]]; then
echo " CA file : ${CA_CERT}"
echo " import this CA file into API/Fluent Bit hosts if tls.verify=On"
fi

View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SQL_FILE="${SCRIPT_DIR}/init_waf_logs_tables.sql"
if [[ ! -f "${SQL_FILE}" ]]; then
echo "[ERROR] SQL file not found: ${SQL_FILE}"
exit 1
fi
if ! command -v clickhouse-client >/dev/null 2>&1; then
echo "[ERROR] clickhouse-client not found. Please install ClickHouse client first."
exit 1
fi
CH_HOST="${CH_HOST:-127.0.0.1}"
CH_PORT="${CH_PORT:-9000}"
CH_USER="${CH_USER:-default}"
CH_PASSWORD="${CH_PASSWORD:-}"
CH_DATABASE="${CH_DATABASE:-default}"
args=(--host "${CH_HOST}" --port "${CH_PORT}" --user "${CH_USER}")
if [[ -n "${CH_PASSWORD}" ]]; then
args+=(--password "${CH_PASSWORD}")
fi
echo "[INFO] creating database if not exists: ${CH_DATABASE}"
clickhouse-client "${args[@]}" --query "CREATE DATABASE IF NOT EXISTS ${CH_DATABASE}"
echo "[INFO] initializing tables in database: ${CH_DATABASE}"
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" < "${SQL_FILE}"
echo "[INFO] checking table status ..."
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" --query \
"SELECT name, engine FROM system.tables WHERE database='${CH_DATABASE}' AND name IN ('logs_ingest','dns_logs_ingest') ORDER BY name"
echo "[OK] ClickHouse ingest tables are ready in database '${CH_DATABASE}'"

View File

@@ -0,0 +1,69 @@
-- Initialize HTTP and DNS ingest tables for GoEdge access logs.
-- Run with:
-- clickhouse-client --database <db_name> < init_waf_logs_tables.sql
CREATE TABLE IF NOT EXISTS logs_ingest
(
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
node_id UInt64,
cluster_id UInt64,
server_id UInt64,
host LowCardinality(String),
ip String,
method LowCardinality(String),
path String CODEC(ZSTD(1)),
status UInt16,
bytes_in UInt64 CODEC(Delta, ZSTD(1)),
bytes_out UInt64 CODEC(Delta, ZSTD(1)),
cost_ms UInt32 CODEC(Delta, ZSTD(1)),
ua String CODEC(ZSTD(1)),
referer String CODEC(ZSTD(1)),
log_type LowCardinality(String),
trace_id String,
firewall_policy_id UInt64 DEFAULT 0,
firewall_rule_group_id UInt64 DEFAULT 0,
firewall_rule_set_id UInt64 DEFAULT 0,
firewall_rule_id UInt64 DEFAULT 0,
request_headers String CODEC(ZSTD(3)) DEFAULT '',
request_body String CODEC(ZSTD(3)) DEFAULT '',
response_headers String CODEC(ZSTD(3)) DEFAULT '',
response_body String CODEC(ZSTD(3)) DEFAULT '',
INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
INDEX idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4,
INDEX idx_status status TYPE minmax GRANULARITY 4
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(timestamp)
ORDER BY (timestamp, node_id, server_id, trace_id)
SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS dns_logs_ingest
(
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
request_id String,
node_id UInt64,
cluster_id UInt64,
domain_id UInt64,
record_id UInt64,
remote_addr String,
question_name String,
question_type LowCardinality(String),
record_name String,
record_type LowCardinality(String),
record_value String,
networking LowCardinality(String),
is_recursive UInt8,
error String CODEC(ZSTD(1)),
ns_route_codes Array(String),
content_json String CODEC(ZSTD(3)) DEFAULT '',
INDEX idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4,
INDEX idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
INDEX idx_domain_id domain_id TYPE minmax GRANULARITY 4
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(timestamp)
ORDER BY (timestamp, request_id, node_id)
SETTINGS index_granularity = 8192;

View File

@@ -0,0 +1,95 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "${EUID}" -ne 0 ]]; then
echo "[ERROR] please run as root"
exit 1
fi
if [[ ! -f /etc/os-release ]]; then
echo "[ERROR] /etc/os-release not found"
exit 1
fi
# shellcheck disable=SC1091
source /etc/os-release
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
os_ver="${VERSION_ID:-}"
is_ubuntu22=false
is_amzn2023=false
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
is_ubuntu22=true
fi
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
is_amzn2023=true
fi
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
exit 1
fi
if [[ "${is_ubuntu22}" == "true" ]]; then
echo "[INFO] detected Ubuntu 22.04"
echo "[INFO] installing prerequisites ..."
apt-get update -y
DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates gnupg apt-transport-https lsb-release
echo "[INFO] configuring ClickHouse apt repository ..."
install -d -m 0755 /etc/apt/keyrings
if [[ ! -f /etc/apt/keyrings/clickhouse.gpg ]]; then
curl -fsSL https://packages.clickhouse.com/CLICKHOUSE-KEY.GPG | gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg
fi
cat >/etc/apt/sources.list.d/clickhouse.list <<'EOF'
deb [signed-by=/etc/apt/keyrings/clickhouse.gpg arch=amd64,arm64] https://packages.clickhouse.com/deb stable main
EOF
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
apt-get update -y
DEBIAN_FRONTEND=noninteractive apt-get install -y clickhouse-server clickhouse-client clickhouse-common-static
fi
if [[ "${is_amzn2023}" == "true" ]]; then
echo "[INFO] detected Amazon Linux 2023"
echo "[INFO] installing prerequisites ..."
dnf makecache -y
dnf install -y curl ca-certificates gnupg2 dnf-plugins-core
echo "[INFO] configuring ClickHouse yum repository ..."
cat >/etc/yum.repos.d/clickhouse.repo <<'EOF'
[clickhouse-stable]
name=ClickHouse Stable Repository
baseurl=https://packages.clickhouse.com/rpm/stable/$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.clickhouse.com/rpm/stable/repodata/repomd.xml.key
https://packages.clickhouse.com/rpm/clickhouse-static.key
EOF
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
dnf clean all
dnf makecache -y
if ! dnf install -y clickhouse-server clickhouse-client clickhouse-common-static; then
dnf install -y clickhouse-server clickhouse-client
fi
fi
echo "[INFO] enabling clickhouse-server ..."
systemctl enable clickhouse-server >/dev/null 2>&1 || true
systemctl restart clickhouse-server
sleep 2
if [[ -n "${CLICKHOUSE_DEFAULT_PASSWORD:-}" ]]; then
echo "[INFO] setting default user password ..."
if [[ "${CLICKHOUSE_DEFAULT_PASSWORD}" == *"'"* ]]; then
echo "[ERROR] CLICKHOUSE_DEFAULT_PASSWORD contains single quote, please set password manually with clickhouse-client"
exit 1
fi
clickhouse-client --query "ALTER USER default IDENTIFIED WITH plaintext_password BY '${CLICKHOUSE_DEFAULT_PASSWORD}'"
fi
echo "[INFO] health check ..."
clickhouse-client --query "SELECT version()"
echo "[OK] ClickHouse install completed: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"

View File

@@ -0,0 +1,123 @@
-- =============================================================================
-- ClickHouse logs_ingest 表优化脚本
--
-- 说明:
-- - 所有 ALTER 操作均为在线操作,无需停服
-- - 建议按阶段顺序执行,每阶段执行后观察 system.parts 确认生效
-- - 压缩编解码器变更仅影响新写入的 part存量数据需等 merge 或手动 OPTIMIZE
--
-- 执行方式:
-- clickhouse-client --host 127.0.0.1 --port 9000 --user default --password 'xxx' < optimize_schema.sql
-- =============================================================================
-- =============================================
-- 阶段 1大字段压缩优化效果最显著
-- =============================================
-- 大文本字段改用 ZSTD(3),对 JSON / HTTP 文本压缩率远优于默认 LZ4
-- 预期效果:磁盘占用减少 40%-60%
ALTER TABLE logs_ingest MODIFY COLUMN request_headers String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN request_body String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN response_headers String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN response_body String CODEC(ZSTD(3));
-- 中等长度文本字段用 ZSTD(1),平衡压缩率与 CPU 开销
ALTER TABLE logs_ingest MODIFY COLUMN ua String CODEC(ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN path String CODEC(ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN referer String CODEC(ZSTD(1));
-- 低基数字段改用 LowCardinality内存+磁盘双降)
-- method 的基数极低GET/POST/PUT/DELETE 等host 基数取决于站点数量
ALTER TABLE logs_ingest MODIFY COLUMN method LowCardinality(String);
ALTER TABLE logs_ingest MODIFY COLUMN log_type LowCardinality(String);
ALTER TABLE logs_ingest MODIFY COLUMN host LowCardinality(String);
-- 数值字段使用 Delta + ZSTD 编码(利用相邻行的时间/大小相关性)
ALTER TABLE logs_ingest MODIFY COLUMN bytes_in UInt64 CODEC(Delta, ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN bytes_out UInt64 CODEC(Delta, ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN cost_ms UInt32 CODEC(Delta, ZSTD(1));
-- =============================================
-- 阶段 2添加 Skipping Index加速高频过滤查询
-- =============================================
-- trace_id 精确查找(查看日志详情 FindByTraceId
-- bloom_filter(0.01) = 1% 误判率GRANULARITY 4 = 每 4 个 granule 一个 bloom block
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4;
-- IP 精确查找
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4;
-- host 模糊查询支持tokenbf_v1 对 LIKE '%xxx%' 有效)
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
-- firewall_policy_id 过滤HasFirewallPolicy: > 0
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4;
-- status 范围过滤HasError: status >= 400
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_status status TYPE minmax GRANULARITY 4;
-- =============================================
-- 阶段 3物化索引到现有数据对存量数据生效
-- =============================================
-- 注意MATERIALIZE INDEX 会触发后台 mutation大表可能需要一定时间
-- 可通过 SELECT * FROM system.mutations WHERE is_done = 0 监控进度
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_trace_id;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_ip;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_host;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_fw_policy;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_status;
-- =============================================================================
-- dns_logs_ingest 表优化DNS 日志表)
-- =============================================================================
-- 大文本字段压缩
ALTER TABLE dns_logs_ingest MODIFY COLUMN content_json String CODEC(ZSTD(3));
ALTER TABLE dns_logs_ingest MODIFY COLUMN error String CODEC(ZSTD(1));
-- 低基数字段
ALTER TABLE dns_logs_ingest MODIFY COLUMN question_type LowCardinality(String);
ALTER TABLE dns_logs_ingest MODIFY COLUMN record_type LowCardinality(String);
ALTER TABLE dns_logs_ingest MODIFY COLUMN networking LowCardinality(String);
-- request_id 精确查找
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4;
-- remote_addr 精确查找
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4;
-- question_name 模糊查询
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
-- domain_id 过滤
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_domain_id domain_id TYPE minmax GRANULARITY 4;
-- 物化索引到现有数据
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_request_id;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_remote_addr;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_question_name;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_domain_id;
-- =============================================================================
-- 验证命令(执行完上述 ALTER 后运行)
-- =============================================================================
-- 查看列的压缩编解码器
-- SELECT name, type, compression_codec FROM system.columns WHERE table = 'logs_ingest' AND database = currentDatabase();
-- 查看表的压缩率
-- SELECT table, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed, round(sum(data_uncompressed_bytes) / sum(data_compressed_bytes), 2) AS ratio FROM system.columns WHERE table IN ('logs_ingest', 'dns_logs_ingest') GROUP BY table;
-- 查看各列占用的磁盘空间(找出最大的列)
-- SELECT name, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed FROM system.columns WHERE table = 'logs_ingest' GROUP BY name ORDER BY sum(data_compressed_bytes) DESC;
-- 查看 mutation 进度
-- SELECT database, table, mutation_id, command, is_done, parts_to_do FROM system.mutations WHERE is_done = 0;
-- 强制触发 merge可选让压缩编解码器变更对存量数据生效
-- OPTIMIZE TABLE logs_ingest FINAL;
-- OPTIMIZE TABLE dns_logs_ingest FINAL;

View File

@@ -21,7 +21,14 @@
- **边缘节点EdgeNode** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log``waf.log``error.log`JSON Lines - **边缘节点EdgeNode** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log``waf.log``error.log`JSON Lines
- **DNS 节点EdgeDNS** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines - **DNS 节点EdgeDNS** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest`见下文「五、ClickHouse 建表」)。 - **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest`见下文「五、ClickHouse 建表」)。
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123 - 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTPS 端口 8443
- 日志轮转默认由 Node/DNS 内建 `lumberjack` 执行:
- `maxSizeMB=256`
- `maxBackups=14`
- `maxAgeDays=7`
- `compress=false`
- `localTime=true`
可通过公用日志策略 `file.rotate` 调整。
--- ---
@@ -71,7 +78,7 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port 编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port
- **单机**:保留一个 `[NODE]`,改 `Host``Port`(默认 8123 - **单机**:保留一个 `[NODE]`,改 `Host``Port`(默认 8443
- **集群**:复制多段 `[NODE]`,每段一个节点,例如: - **集群**:复制多段 `[NODE]`,每段一个节点,例如:
```ini ```ini
@@ -81,12 +88,12 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
[NODE] [NODE]
Name node-01 Name node-01
Host 192.168.1.10 Host 192.168.1.10
Port 8123 Port 8443
[NODE] [NODE]
Name node-02 Name node-02
Host 192.168.1.11 Host 192.168.1.11
Port 8123 Port 8443
``` ```
### 3.3 ClickHouse 账号密码(有密码时必做) ### 3.3 ClickHouse 账号密码(有密码时必做)
@@ -319,9 +326,9 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
| 组件 | 说明 | | 组件 | 说明 |
|------|------| |------|------|
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 | | **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`内建 lumberjack 轮转(默认 256MB/14份/7天可按策略调整仍支持 SIGHUP 重建 writer。 |
| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines由 Fluent Bit 采集写入 `dns_logs_ingest`。 | | **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`JSON Lines由 Fluent Bit 采集写入 `dns_logs_ingest`。 |
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 | | **logrotate** | 可选的历史兼容方案(已非必需);默认建议使用节点内建 lumberjack 轮转。 |
| **平台EdgeAPI** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 | | **平台EdgeAPI** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
--- ---
@@ -411,3 +418,54 @@ sudo systemctl restart fluent-bit
``` ```
回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。 回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。
---
## 十、平台托管模式(推荐)
从 `v1.4.7` 开始Node/DNS 在线安装流程会由平台托管 Fluent Bit默认不再要求逐台手改 `/etc/fluent-bit/fluent-bit.conf`。
### 10.1 托管行为
- 安装器优先使用发布包内置离线包(不走 `curl | sh`)。
- 首次安装后写入:
- `/etc/fluent-bit/fluent-bit.conf`
- `/etc/fluent-bit/parsers.conf`
- `/etc/fluent-bit/.edge-managed.env`
- `/etc/fluent-bit/.edge-managed.json`
- 配置发生变化时按 `hash` 幂等更新,仅在内容变化时重启服务。
- Node 与 DNS 同机安装时会自动合并角色,输出单份配置。
### 10.2 托管元数据
平台会维护 `/etc/fluent-bit/.edge-managed.json`,核心字段:
- `roles`: 当前机器启用角色(`node`/`dns`
- `hash`: 当前托管配置摘要
- `sourceVersion`: 平台版本号
- `updatedAt`: 最近更新时间戳
### 10.3 支持矩阵(离线包)
当前固定支持以下平台键:
- `ubuntu22.04-amd64`
- `ubuntu22.04-arm64`
- `amzn2023-amd64`
- `amzn2023-arm64`
构建阶段会校验矩阵包是否齐全,缺失会直接失败并打印期望文件路径。
### 10.4 手工配置兼容
- 若现有 `fluent-bit.conf` 不是平台托管文件(不含 `managed-by-edgeapi` 标记),安装器不会强制覆盖,会返回明确错误提示。
- 需要切到托管模式时,先备份旧配置,再由平台触发一次安装/更新任务。
### 10.5 Resource Profile Notes (New)
- Managed default is now tuned for `2C4G` nodes (conservative and stable).
- Additional sample profiles are provided for larger nodes:
- `deploy/fluent-bit/fluent-bit-sample-4c8g.conf`
- `deploy/fluent-bit/fluent-bit-sample-8c16g.conf`
- These sample files are for benchmark/reference only and are not auto-applied by installer.
- To use higher profiles in managed mode, sync those parameters into `EdgeAPI/internal/installers/fluent_bit.go` and then trigger node reinstall/upgrade.

View File

@@ -8,4 +8,4 @@
[NODE] [NODE]
Name node-01 Name node-01
Host 127.0.0.1 Host 127.0.0.1
Port 8123 Port 8443

View File

@@ -1,4 +1,4 @@
# DNS 节点专用:使用 HTTP 输出写入 ClickHouse无需 out_clickhouse 插件) # DNS 节点专用:使用 HTTPS 输出写入 ClickHouse无需 out_clickhouse 插件)
# 启动前设置CH_USER、CH_PASSWORD若 ClickHouse 不在本机,请修改 Host、Port # 启动前设置CH_USER、CH_PASSWORD若 ClickHouse 不在本机,请修改 Host、Port
# Read_from_Head=true首次启动会发送已有日志若只采新日志建议改为 false # Read_from_Head=true首次启动会发送已有日志若只采新日志建议改为 false
@@ -26,11 +26,15 @@
Name http Name http
Match app.dns.logs Match app.dns.logs
Host 127.0.0.1 Host 127.0.0.1
Port 8123 Port 8443
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines Format json_lines
http_user ${CH_USER} http_user ${CH_USER}
http_passwd ${CH_PASSWORD} http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
# tls.vhost clickhouse.example.com
json_date_key timestamp json_date_key timestamp
json_date_format epoch json_date_format epoch
Retry_Limit 10 Retry_Limit 10

View File

@@ -0,0 +1,69 @@
# Sample profile for 4C8G nodes (Node + DNS on same host).
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
[SERVICE]
Flush 1
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 512MB
[INPUT]
Name tail
Path /var/log/edge/edge-node/*.log
Tag app.http.logs
Parser json
Refresh_Interval 2
Read_from_Head false
DB /var/lib/fluent-bit/http-logs.db
storage.type filesystem
Mem_Buf_Limit 256MB
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Parser json
Refresh_Interval 2
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
storage.type filesystem
Mem_Buf_Limit 256MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.http.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
workers 2
net.keepalive On
Retry_Limit False
tls On
tls.verify On
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
workers 2
net.keepalive On
Retry_Limit False
tls On
tls.verify On

View File

@@ -0,0 +1,69 @@
# Sample profile for 8C16G nodes (Node + DNS on same host).
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
[SERVICE]
Flush 1
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 1024MB
[INPUT]
Name tail
Path /var/log/edge/edge-node/*.log
Tag app.http.logs
Parser json
Refresh_Interval 1
Read_from_Head false
DB /var/lib/fluent-bit/http-logs.db
storage.type filesystem
Mem_Buf_Limit 512MB
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Parser json
Refresh_Interval 1
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
storage.type filesystem
Mem_Buf_Limit 512MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.http.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
workers 4
net.keepalive On
Retry_Limit False
tls On
tls.verify On
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
workers 4
net.keepalive On
Retry_Limit False
tls On
tls.verify On

View File

@@ -47,6 +47,32 @@ flowchart TD
- `/var/log/edge/edge-node/*.log` - `/var/log/edge/edge-node/*.log`
3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。 3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。
### 3.3 本地日志轮转(默认开启)
从当前版本开始EdgeNode / EdgeDNS 使用内建 `lumberjack` 轮转,不再依赖系统 `logrotate`。
默认值:
- `maxSizeMB=256`
- `maxBackups=14`
- `maxAgeDays=7`
- `compress=false`
- `localTime=true`
可在策略 `file.rotate` 中配置,例如:
```json
{
"path": "/var/log/web-access-${date}.log",
"autoCreate": true,
"rotate": {
"maxSizeMB": 256,
"maxBackups": 14,
"maxAgeDays": 7,
"compress": false,
"localTime": true
}
}
```
--- ---
## 4. 三种目标模式怎么配 ## 4. 三种目标模式怎么配