diff --git a/EdgeAPI/internal/clickhouse/config.go b/EdgeAPI/internal/clickhouse/config.go index 6e882aa..c2fe014 100644 --- a/EdgeAPI/internal/clickhouse/config.go +++ b/EdgeAPI/internal/clickhouse/config.go @@ -18,11 +18,9 @@ const ( envPassword = "CLICKHOUSE_PASSWORD" envDatabase = "CLICKHOUSE_DATABASE" envScheme = "CLICKHOUSE_SCHEME" - envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY" - envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME" - defaultPort = 8123 + defaultPort = 8443 defaultDB = "default" - defaultScheme = "http" + defaultScheme = "https" ) var ( @@ -62,7 +60,7 @@ func ResetSharedConfig() { } func loadConfig() *Config { - cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme} + cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme, TLSSkipVerify: true} // 1) 优先从后台页面配置（DB）读取 if models.SharedSysSettingDAO != nil { if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" { @@ -72,8 +70,8 @@ func loadConfig() *Config { cfg.Password = dbCfg.Password cfg.Database = dbCfg.Database cfg.Scheme = normalizeScheme(dbCfg.Scheme) - cfg.TLSSkipVerify = dbCfg.TLSSkipVerify - cfg.TLSServerName = dbCfg.TLSServerName + cfg.TLSSkipVerify = true + cfg.TLSServerName = "" if cfg.Port <= 0 { cfg.Port = defaultPort } @@ -93,8 +91,8 @@ func loadConfig() *Config { cfg.Password = ch.Password cfg.Database = ch.Database cfg.Scheme = normalizeScheme(ch.Scheme) - cfg.TLSSkipVerify = ch.TLSSkipVerify - cfg.TLSServerName = ch.TLSServerName + cfg.TLSSkipVerify = true + cfg.TLSServerName = "" if cfg.Port <= 0 { cfg.Port = defaultPort } @@ -112,17 +110,13 @@ func loadConfig() *Config { cfg.Database = defaultDB } cfg.Scheme = normalizeScheme(os.Getenv(envScheme)) - cfg.TLSServerName = os.Getenv(envTLSServerName) + cfg.TLSServerName = "" if p := os.Getenv(envPort); p != "" { if v, err := strconv.Atoi(p); err == nil { cfg.Port = v } } - if v := os.Getenv(envTLSSkipVerify); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - cfg.TLSSkipVerify = b - } - } + cfg.TLSSkipVerify = true return cfg } diff --git a/EdgeAPI/internal/clickhouse/logs_ingest_store.go b/EdgeAPI/internal/clickhouse/logs_ingest_store.go index dde2623..4cbc965 100644 --- a/EdgeAPI/internal/clickhouse/logs_ingest_store.go +++ b/EdgeAPI/internal/clickhouse/logs_ingest_store.go @@ -176,7 +176,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI } orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir) - query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d", + // 列表查询不 SELECT 大字段（request_headers / request_body / response_headers / response_body）， + // 避免每次翻页读取 GB 级数据。详情查看时通过 FindByTraceId 单独获取。 + query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id FROM %s WHERE %s ORDER BY %s LIMIT %d", table, where, orderBy, limit+1) var rawRows []map[string]interface{} diff --git a/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go b/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go index a8a38c6..597a886 100644 --- a/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go +++ b/EdgeAPI/internal/clickhouse/ns_logs_ingest_store.go @@ -122,8 +122,10 @@ func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*N limit = 1000 } + // 列表查询不 SELECT content_json 大字段，减少翻页时的数据传输量。 + // 详情查看时通过 FindByRequestId 单独获取完整信息。 query := fmt.Sprintf( - "SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d", + "SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d", table, strings.Join(conditions, " AND "), orderDir, diff --git a/EdgeAPI/internal/db/models/http_access_log_policy_utils.go b/EdgeAPI/internal/db/models/http_access_log_policy_utils.go index cfa151e..3be2c46 100644 --- a/EdgeAPI/internal/db/models/http_access_log_policy_utils.go +++ b/EdgeAPI/internal/db/models/http_access_log_policy_utils.go @@ -20,3 +20,20 @@ func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string { return strings.TrimSpace(config.Path) } + +// ParseHTTPAccessLogPolicyRotateConfig 提取访问日志策略中的文件轮转配置（所有 file* 类型有效）。 +func ParseHTTPAccessLogPolicyRotateConfig(policy *HTTPAccessLogPolicy) *serverconfigs.AccessLogRotateConfig { + if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 { + return nil + } + + config := &serverconfigs.AccessLogFileStorageConfig{} + if err := json.Unmarshal(policy.Options, config); err != nil { + return nil + } + if config.Rotate == nil { + return nil + } + + return config.Rotate.Normalize() +} diff --git a/EdgeAPI/internal/db/models/node_dao.go b/EdgeAPI/internal/db/models/node_dao.go index cb0452e..e032c6b 100644 --- a/EdgeAPI/internal/db/models/node_dao.go +++ b/EdgeAPI/internal/db/models/node_dao.go @@ -1176,6 +1176,7 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared if publicPolicy != nil { config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB) config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) + config.GlobalServerConfig.HTTPAccessLog.Rotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy) } } } diff --git a/EdgeAPI/internal/db/models/ns_node_dao_plus.go b/EdgeAPI/internal/db/models/ns_node_dao_plus.go index b074929..d74fd32 100644 --- a/EdgeAPI/internal/db/models/ns_node_dao_plus.go +++ b/EdgeAPI/internal/db/models/ns_node_dao_plus.go @@ -479,6 +479,7 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs. if publicPolicy != nil { config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB) config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy) + config.AccessLogRotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy) } } } diff --git a/EdgeAPI/internal/db/models/sys_setting_dao.go b/EdgeAPI/internal/db/models/sys_setting_dao.go index 8795cca..086267a 100644 --- a/EdgeAPI/internal/db/models/sys_setting_dao.go +++ b/EdgeAPI/internal/db/models/sys_setting_dao.go @@ -270,8 +270,9 @@ func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.Clic } if len(valueJSON) == 0 { return &systemconfigs.ClickHouseSetting{ - Port: 8123, + Port: 8443, Database: "default", + Scheme: "https", }, nil } var config = &systemconfigs.ClickHouseSetting{} diff --git a/EdgeAPI/internal/installers/fluent_bit.go b/EdgeAPI/internal/installers/fluent_bit.go index 3164d89..be009dd 100644 --- a/EdgeAPI/internal/installers/fluent_bit.go +++ b/EdgeAPI/internal/installers/fluent_bit.go @@ -1,14 +1,23 @@ package installers import ( + "crypto/sha256" + "encoding/json" "errors" "fmt" + "net/url" "os" + slashpath "path" "path/filepath" "sort" + "strconv" "strings" + "time" + teaconst "github.com/TeaOSLab/EdgeAPI/internal/const" + "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" + "github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs" "github.com/iwind/TeaGo/Tea" ) @@ -17,19 +26,45 @@ const ( fluentBitStorageDir = "/var/lib/fluent-bit/storage" fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf" fluentBitParsersFile = "/etc/fluent-bit/parsers.conf" - fluentBitUpstreamFile = "/etc/fluent-bit/clickhouse-upstream.conf" + fluentBitManagedMetaFile = "/etc/fluent-bit/.edge-managed.json" + fluentBitManagedEnvFile = "/etc/fluent-bit/.edge-managed.env" fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge" + fluentBitDropInDir = "/etc/systemd/system/fluent-bit.service.d" + fluentBitDropInFile = "/etc/systemd/system/fluent-bit.service.d/edge-managed.conf" fluentBitServiceName = "fluent-bit" fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit" fluentBitLocalPackagesRoot = "packages" fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log" fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log" + fluentBitManagedMarker = "managed-by-edgeapi" + fluentBitRoleNode = "node" + fluentBitRoleDNS = "dns" ) var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found") -// SetupFluentBit 安装 Fluent Bit（仅离线包），并同步配置文件。 -// 升级场景下不覆盖已有配置；若已有配置与节点角色不兼容，直接报错终止安装。 +var fluentBitPackageFileMapping = map[string]string{ + "ubuntu22.04-amd64": "fluent-bit_4.2.2_amd64.deb", + "ubuntu22.04-arm64": "fluent-bit_4.2.2_arm64.deb", + "amzn2023-amd64": "fluent-bit-4.2.2-1.x86_64.rpm", + "amzn2023-arm64": "fluent-bit-4.2.2-1.aarch64.rpm", +} + +type fluentBitManagedMeta struct { + Roles []string `json:"roles"` + Hash string `json:"hash"` + UpdatedAt int64 `json:"updatedAt"` + SourceVersion string `json:"sourceVersion"` +} + +type fluentBitDesiredConfig struct { + Roles []string + ClickHouse *systemconfigs.ClickHouseSetting + HTTPPathPattern string + DNSPathPattern string +} + +// SetupFluentBit 安装并托管 Fluent Bit 配置（离线包 + 平台渲染配置）。 func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error { if this.client == nil { return errors.New("ssh client is nil") @@ -41,55 +76,41 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error { } tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit" - _, _, _ = this.client.Exec("mkdir -p " + tempDir) + _, _, _ = this.client.Exec("mkdir -p " + shQuote(tempDir)) defer func() { - _, _, _ = this.client.Exec("rm -rf " + tempDir) + _, _, _ = this.client.Exec("rm -rf " + shQuote(tempDir)) }() - // 统一使用 fluent-bit.conf（已含 HTTP + DNS 两类 input），避免同机 Node/DNS 冲突。 - files := []struct { - Local string - Remote string - }{ - {Local: "fluent-bit.conf", Remote: "fluent-bit.conf"}, - {Local: "parsers.conf", Remote: "parsers.conf"}, - {Local: "clickhouse-upstream.conf", Remote: "clickhouse-upstream.conf"}, - {Local: "logrotate.conf", Remote: "logrotate.conf"}, - } - - for _, file := range files { - localPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", file.Local) - if _, err := os.Stat(localPath); err != nil { - return fmt.Errorf("fluent-bit file '%s' not found: %w", localPath, err) - } - remotePath := tempDir + "/" + file.Remote - if err := this.client.Copy(localPath, remotePath, 0644); err != nil { - return fmt.Errorf("upload fluent-bit file '%s' failed: %w", file.Local, err) - } - } - if err := this.ensureFluentBitInstalled(tempDir); err != nil { return err } - _, stderr, err := this.client.Exec("mkdir -p " + fluentBitConfigDir + " " + fluentBitStorageDir + " /etc/logrotate.d") + _, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitConfigDir) + " " + shQuote(fluentBitStorageDir) + " /etc/logrotate.d") if err != nil { return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr) } - exists, err := this.remoteFileExists(fluentBitMainConfigFile) + parserContent, err := this.readLocalParsersContent() if err != nil { return err } - // 若已存在配置，先做角色兼容校验，不允许覆盖。 - if exists { - if err := this.validateExistingConfigForRole(role); err != nil { - return err - } + existingMeta, err := this.readManagedMeta() + if err != nil { + return err } - configCopied, err := this.copyFluentBitConfigIfMissing(tempDir) + mergedRoles, err := mergeManagedRoles(existingMeta, role) + if err != nil { + return err + } + + desiredConfig, err := this.buildDesiredFluentBitConfig(mergedRoles) + if err != nil { + return err + } + + configChanged, err := this.applyManagedConfig(tempDir, desiredConfig, parserContent, existingMeta) if err != nil { return err } @@ -99,7 +120,7 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error { return err } - if err := this.ensureFluentBitService(binPath, configCopied); err != nil { + if err := this.ensureFluentBitService(tempDir, binPath, configChanged); err != nil { return err } @@ -112,102 +133,128 @@ func (this *BaseInstaller) ensureFluentBitInstalled(tempDir string) error { return nil } - if err := this.installFluentBitFromLocalPackage(tempDir); err != nil { + platformKey, packageName, arch, err := this.detectRemotePlatformAndPackage() + if err != nil { + return fmt.Errorf("detect fluent-bit platform failed: %w", err) + } + + if err := this.installFluentBitFromLocalPackage(tempDir, arch, packageName); err != nil { if errors.Is(err, errFluentBitLocalPackageNotFound) { - return fmt.Errorf("install fluent-bit failed: local package not found, expected in deploy/fluent-bit/%s/linux-", fluentBitLocalPackagesRoot) + expectedPath := filepath.Join("deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch, packageName) + return fmt.Errorf("install fluent-bit failed: local package missing for platform '%s', expected '%s'", platformKey, expectedPath) } return fmt.Errorf("install fluent-bit from local package failed: %w", err) } - binPath, err := this.lookupFluentBitBinPath() + binPath, err = this.lookupFluentBitBinPath() if err != nil { return err } if binPath == "" { return errors.New("fluent-bit binary not found after local package install") } + + _, stderr, err := this.client.Exec(binPath + " --version") + if err != nil { + return fmt.Errorf("verify fluent-bit version failed: %w, stderr: %s", err, stderr) + } + return nil } -func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string) error { - arch, err := this.detectRemoteLinuxArch() - if err != nil { - return err - } - +func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string, arch string, packageName string) error { packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch) - entries, err := os.ReadDir(packageDir) - if err != nil { + localPackagePath := filepath.Join(packageDir, packageName) + if _, err := os.Stat(localPackagePath); err != nil { if os.IsNotExist(err) { return errFluentBitLocalPackageNotFound } - return fmt.Errorf("read fluent-bit local package dir failed: %w", err) + return fmt.Errorf("check local package failed: %w", err) } - packageFiles := make([]string, 0) - for _, entry := range entries { - if entry.IsDir() { + remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath) + if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil { + return fmt.Errorf("upload local package failed: %w", err) + } + + var installCmd string + lowerName := strings.ToLower(localPackagePath) + switch { + case strings.HasSuffix(lowerName, ".deb"): + installCmd = "dpkg -i " + shQuote(remotePackagePath) + case strings.HasSuffix(lowerName, ".rpm"): + installCmd = "rpm -Uvh --force " + shQuote(remotePackagePath) + " || rpm -ivh --force " + shQuote(remotePackagePath) + case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"): + extractDir := tempDir + "/extract" + installCmd = "rm -rf " + shQuote(extractDir) + "; mkdir -p " + shQuote(extractDir) + "; tar -xzf " + shQuote(remotePackagePath) + " -C " + shQuote(extractDir) + "; " + + "bin=$(find " + shQuote(extractDir) + " -type f -name fluent-bit | head -n 1); " + + "if [ -z \"$bin\" ]; then exit 3; fi; " + + "mkdir -p /opt/fluent-bit/bin /usr/local/bin; " + + "install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " + + "ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit" + default: + return fmt.Errorf("unsupported local package format: %s", packageName) + } + + _, stderr, err := this.client.Exec(installCmd) + if err != nil { + return fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr) + } + + return nil +} + +func (this *BaseInstaller) detectRemotePlatformAndPackage() (platformKey string, packageName string, arch string, err error) { + arch, err = this.detectRemoteLinuxArch() + if err != nil { + return "", "", "", err + } + + releaseData, stderr, err := this.client.Exec("cat /etc/os-release") + if err != nil { + return "", "", "", fmt.Errorf("read /etc/os-release failed: %w, stderr: %s", err, stderr) + } + if strings.TrimSpace(releaseData) == "" { + return "", "", "", errors.New("/etc/os-release is empty") + } + + releaseMap := parseOSRelease(releaseData) + id := strings.ToLower(strings.TrimSpace(releaseMap["ID"])) + versionID := strings.TrimSpace(releaseMap["VERSION_ID"]) + + var distro string + switch { + case id == "ubuntu" && strings.HasPrefix(versionID, "22.04"): + distro = "ubuntu22.04" + case id == "amzn" && strings.HasPrefix(versionID, "2023"): + distro = "amzn2023" + default: + return "", "", "", fmt.Errorf("unsupported linux platform id='%s' version='%s'", id, versionID) + } + + platformKey = distro + "-" + arch + packageName, ok := fluentBitPackageFileMapping[platformKey] + if !ok { + return "", "", "", fmt.Errorf("no local package mapping for platform '%s'", platformKey) + } + return platformKey, packageName, arch, nil +} + +func parseOSRelease(content string) map[string]string { + result := map[string]string{} + lines := strings.Split(content, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") { continue } - name := strings.ToLower(entry.Name()) - if strings.HasSuffix(name, ".deb") || strings.HasSuffix(name, ".rpm") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") { - packageFiles = append(packageFiles, filepath.Join(packageDir, entry.Name())) - } + parts := strings.SplitN(line, "=", 2) + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + value = strings.Trim(value, "\"") + result[key] = value } - if len(packageFiles) == 0 { - return errFluentBitLocalPackageNotFound - } - - sort.Strings(packageFiles) - - var lastErr error - for _, localPackagePath := range packageFiles { - remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath) - if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil { - lastErr = fmt.Errorf("upload local package failed: %w", err) - continue - } - - var installCmd string - lowerName := strings.ToLower(localPackagePath) - switch { - case strings.HasSuffix(lowerName, ".deb"): - installCmd = "dpkg -i " + remotePackagePath - case strings.HasSuffix(lowerName, ".rpm"): - installCmd = "rpm -Uvh --force " + remotePackagePath + " || rpm -ivh --force " + remotePackagePath - case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"): - extractDir := tempDir + "/extract" - installCmd = "rm -rf " + extractDir + "; mkdir -p " + extractDir + "; tar -xzf " + remotePackagePath + " -C " + extractDir + "; " + - "bin=$(find " + extractDir + " -type f -name fluent-bit | head -n 1); " + - "if [ -z \"$bin\" ]; then exit 3; fi; " + - "mkdir -p /opt/fluent-bit/bin /usr/local/bin; " + - "install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " + - "ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit" - default: - continue - } - - _, stderr, err := this.client.Exec(installCmd) - if err != nil { - lastErr = fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr) - continue - } - - binPath, err := this.lookupFluentBitBinPath() - if err == nil && binPath != "" { - return nil - } - if err != nil { - lastErr = err - } else { - lastErr = errors.New("fluent-bit binary not found after local package install") - } - } - - if lastErr != nil { - return lastErr - } - return errFluentBitLocalPackageNotFound + return result } func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) { @@ -235,74 +282,456 @@ func (this *BaseInstaller) lookupFluentBitBinPath() (string, error) { return strings.TrimSpace(stdout), nil } -func (this *BaseInstaller) copyFluentBitConfigIfMissing(tempDir string) (bool, error) { - targets := []struct { - Src string - Dest string - }{ - {Src: tempDir + "/fluent-bit.conf", Dest: fluentBitMainConfigFile}, - {Src: tempDir + "/parsers.conf", Dest: fluentBitParsersFile}, - {Src: tempDir + "/clickhouse-upstream.conf", Dest: fluentBitUpstreamFile}, - {Src: tempDir + "/logrotate.conf", Dest: fluentBitLogrotateFile}, +func (this *BaseInstaller) readLocalParsersContent() (string, error) { + parsersPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", "parsers.conf") + data, err := os.ReadFile(parsersPath) + if err != nil { + return "", fmt.Errorf("read local parsers config failed: %w", err) + } + return string(data), nil +} + +func (this *BaseInstaller) readManagedMeta() (*fluentBitManagedMeta, error) { + exists, err := this.remoteFileExists(fluentBitManagedMetaFile) + if err != nil { + return nil, err + } + if !exists { + return nil, nil } - copied := false - for _, target := range targets { - exists, err := this.remoteFileExists(target.Dest) + content, stderr, err := this.client.Exec("cat " + shQuote(fluentBitManagedMetaFile)) + if err != nil { + return nil, fmt.Errorf("read fluent-bit managed metadata failed: %w, stderr: %s", err, stderr) + } + if strings.TrimSpace(content) == "" { + return nil, nil + } + + meta := &fluentBitManagedMeta{} + if err := json.Unmarshal([]byte(content), meta); err != nil { + return nil, fmt.Errorf("decode fluent-bit managed metadata failed: %w", err) + } + meta.Roles = normalizeRoles(meta.Roles) + return meta, nil +} + +func mergeManagedRoles(meta *fluentBitManagedMeta, role nodeconfigs.NodeRole) ([]string, error) { + roleName, err := mapNodeRole(role) + if err != nil { + return nil, err + } + + roleSet := map[string]struct{}{} + if meta != nil { + for _, r := range normalizeRoles(meta.Roles) { + roleSet[r] = struct{}{} + } + } + roleSet[roleName] = struct{}{} + + roles := make([]string, 0, len(roleSet)) + for r := range roleSet { + roles = append(roles, r) + } + sort.Strings(roles) + return roles, nil +} + +func mapNodeRole(role nodeconfigs.NodeRole) (string, error) { + switch role { + case nodeconfigs.NodeRoleNode: + return fluentBitRoleNode, nil + case nodeconfigs.NodeRoleDNS: + return fluentBitRoleDNS, nil + default: + return "", fmt.Errorf("unsupported fluent-bit role '%s'", role) + } +} + +func normalizeRoles(rawRoles []string) []string { + roleSet := map[string]struct{}{} + for _, role := range rawRoles { + role = strings.ToLower(strings.TrimSpace(role)) + if role != fluentBitRoleNode && role != fluentBitRoleDNS { + continue + } + roleSet[role] = struct{}{} + } + + roles := make([]string, 0, len(roleSet)) + for role := range roleSet { + roles = append(roles, role) + } + sort.Strings(roles) + return roles +} + +func hasRole(roles []string, role string) bool { + for _, one := range roles { + if one == role { + return true + } + } + return false +} + +func (this *BaseInstaller) buildDesiredFluentBitConfig(roles []string) (*fluentBitDesiredConfig, error) { + if len(roles) == 0 { + return nil, errors.New("fluent-bit roles should not be empty") + } + + ch, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil) + if err != nil { + return nil, fmt.Errorf("read clickhouse setting failed: %w", err) + } + if ch == nil { + ch = &systemconfigs.ClickHouseSetting{} + } + if strings.TrimSpace(ch.Host) == "" { + ch.Host = "127.0.0.1" + } + + ch.Scheme = strings.ToLower(strings.TrimSpace(ch.Scheme)) + if ch.Scheme == "" { + ch.Scheme = "https" + } + if ch.Scheme != "http" && ch.Scheme != "https" { + return nil, fmt.Errorf("unsupported clickhouse scheme '%s'", ch.Scheme) + } + + if ch.Port <= 0 { + if ch.Scheme == "https" { + ch.Port = 8443 + } else { + ch.Port = 8443 + } + } + if strings.TrimSpace(ch.Database) == "" { + ch.Database = "default" + } + if strings.TrimSpace(ch.User) == "" { + ch.User = "default" + } + // 当前平台策略：后台固定跳过 ClickHouse TLS 证书校验，不暴露 ServerName 配置。 + ch.TLSSkipVerify = true + ch.TLSServerName = "" + + httpPathPattern := fluentBitHTTPPathPattern + dnsPathPattern := fluentBitDNSPathPattern + publicPolicyPath, err := this.readPublicAccessLogPolicyPath() + if err != nil { + return nil, err + } + policyDir := dirFromPolicyPath(publicPolicyPath) + if policyDir != "" { + pattern := strings.TrimRight(policyDir, "/") + "/*.log" + httpPathPattern = pattern + dnsPathPattern = pattern + } + + return &fluentBitDesiredConfig{ + Roles: normalizeRoles(roles), + ClickHouse: ch, + HTTPPathPattern: httpPathPattern, + DNSPathPattern: dnsPathPattern, + }, nil +} + +func (this *BaseInstaller) readPublicAccessLogPolicyPath() (string, error) { + policyId, err := models.SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(nil) + if err != nil { + return "", fmt.Errorf("find current public access log policy failed: %w", err) + } + if policyId <= 0 { + return "", nil + } + + policy, err := models.SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(nil, policyId) + if err != nil { + return "", fmt.Errorf("read public access log policy failed: %w", err) + } + if policy == nil { + return "", nil + } + + return strings.TrimSpace(models.ParseHTTPAccessLogPolicyFilePath(policy)), nil +} + +func dirFromPolicyPath(policyPath string) string { + pathValue := strings.TrimSpace(policyPath) + if pathValue == "" { + return "" + } + pathValue = strings.ReplaceAll(pathValue, "\\", "/") + dir := slashpath.Dir(pathValue) + if dir == "." { + return "" + } + return strings.TrimRight(dir, "/") +} + +func (this *BaseInstaller) applyManagedConfig(tempDir string, desired *fluentBitDesiredConfig, parserContent string, existingMeta *fluentBitManagedMeta) (bool, error) { + mainExists, err := this.remoteFileExists(fluentBitMainConfigFile) + if err != nil { + return false, err + } + + if mainExists && existingMeta == nil { + containsMarker, err := this.remoteFileContains(fluentBitMainConfigFile, fluentBitManagedMarker) if err != nil { return false, err } - if exists { - continue + if !containsMarker { + // Adopt unmanaged config by backing it up and replacing with managed config below. } - _, stderr, err := this.client.Exec("cp -f " + target.Src + " " + target.Dest) - if err != nil { - return false, fmt.Errorf("copy fluent-bit file to '%s' failed: %w, stderr: %s", target.Dest, err, stderr) - } - copied = true } - return copied, nil + configContent, err := renderManagedConfig(desired) + if err != nil { + return false, err + } + envContent := renderManagedEnv(desired.ClickHouse) + metaContent, newMeta, err := renderManagedMeta(desired, configContent, parserContent, envContent) + if err != nil { + return false, err + } + + requiredFiles := []string{fluentBitMainConfigFile, fluentBitParsersFile, fluentBitManagedEnvFile, fluentBitManagedMetaFile} + if existingMeta != nil && existingMeta.Hash == newMeta.Hash { + allExists := true + for _, file := range requiredFiles { + exists, err := this.remoteFileExists(file) + if err != nil { + return false, err + } + if !exists { + allExists = false + break + } + } + if allExists { + return false, nil + } + } + + if mainExists { + backup := fluentBitMainConfigFile + ".bak." + strconv.FormatInt(time.Now().Unix(), 10) + _, stderr, err := this.client.Exec("cp -f " + shQuote(fluentBitMainConfigFile) + " " + shQuote(backup)) + if err != nil { + return false, fmt.Errorf("backup existing fluent-bit config failed: %w, stderr: %s", err, stderr) + } + } + + if err := this.writeRemoteFileByTemp(tempDir, fluentBitMainConfigFile, configContent, 0644); err != nil { + return false, err + } + if err := this.writeRemoteFileByTemp(tempDir, fluentBitParsersFile, parserContent, 0644); err != nil { + return false, err + } + if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedEnvFile, envContent, 0600); err != nil { + return false, err + } + if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedMetaFile, metaContent, 0644); err != nil { + return false, err + } + + localLogrotate := filepath.Join(Tea.Root, "deploy", "fluent-bit", "logrotate.conf") + if _, err := os.Stat(localLogrotate); err == nil { + if err := this.copyLocalFileToRemote(tempDir, localLogrotate, fluentBitLogrotateFile, 0644); err != nil { + return false, err + } + } + + return true, nil } -func (this *BaseInstaller) validateExistingConfigForRole(role nodeconfigs.NodeRole) error { - requiredPatterns := []string{} - switch role { - case nodeconfigs.NodeRoleNode: - requiredPatterns = append(requiredPatterns, fluentBitHTTPPathPattern) - case nodeconfigs.NodeRoleDNS: - requiredPatterns = append(requiredPatterns, fluentBitDNSPathPattern) +func renderManagedConfig(desired *fluentBitDesiredConfig) (string, error) { + if desired == nil || desired.ClickHouse == nil { + return "", errors.New("invalid fluent-bit desired config") } - for _, pattern := range requiredPatterns { - ok, err := this.remoteFileContains(fluentBitMainConfigFile, pattern) - if err != nil { - return err + scheme := strings.ToLower(strings.TrimSpace(desired.ClickHouse.Scheme)) + if scheme == "" { + scheme = "http" + } + if scheme != "http" && scheme != "https" { + return "", fmt.Errorf("unsupported clickhouse scheme '%s'", desired.ClickHouse.Scheme) + } + useTLS := scheme == "https" + + insertHTTP := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database)) + insertDNS := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.dns_logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database)) + + lines := []string{ + "# " + fluentBitManagedMarker, + "[SERVICE]", + " Flush 2", + " Log_Level info", + " Parsers_File " + fluentBitParsersFile, + " storage.path " + fluentBitStorageDir, + " storage.sync normal", + " storage.checksum off", + " storage.backlog.mem_limit 256MB", + "", + } + + if hasRole(desired.Roles, fluentBitRoleNode) { + lines = append(lines, + "[INPUT]", + " Name tail", + " Path "+desired.HTTPPathPattern, + " Tag app.http.logs", + " Parser json", + " Refresh_Interval 2", + " Read_from_Head false", + " DB /var/lib/fluent-bit/http-logs.db", + " storage.type filesystem", + " Mem_Buf_Limit 128MB", + " Skip_Long_Lines On", + "", + ) + } + + if hasRole(desired.Roles, fluentBitRoleDNS) { + lines = append(lines, + "[INPUT]", + " Name tail", + " Path "+desired.DNSPathPattern, + " Tag app.dns.logs", + " Parser json", + " Refresh_Interval 2", + " Read_from_Head false", + " DB /var/lib/fluent-bit/dns-logs.db", + " storage.type filesystem", + " Mem_Buf_Limit 128MB", + " Skip_Long_Lines On", + "", + ) + } + + if hasRole(desired.Roles, fluentBitRoleNode) { + lines = append(lines, + "[OUTPUT]", + " Name http", + " Match app.http.logs", + " Host "+desired.ClickHouse.Host, + " Port "+strconv.Itoa(desired.ClickHouse.Port), + " URI /?query="+insertHTTP, + " Format json_lines", + " http_user ${CH_USER}", + " http_passwd ${CH_PASSWORD}", + " json_date_key timestamp", + " json_date_format epoch", + " workers 1", + " net.keepalive On", + " Retry_Limit False", + ) + if useTLS { + lines = append(lines, " tls On") + if desired.ClickHouse.TLSSkipVerify { + lines = append(lines, " tls.verify Off") + } else { + lines = append(lines, " tls.verify On") + } + if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" { + lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName)) + } } - if !ok { - return fmt.Errorf("existing fluent-bit config '%s' does not contain required path '%s'; skip overwrite by design, please update config manually", fluentBitMainConfigFile, pattern) + lines = append(lines, "") + } + + if hasRole(desired.Roles, fluentBitRoleDNS) { + lines = append(lines, + "[OUTPUT]", + " Name http", + " Match app.dns.logs", + " Host "+desired.ClickHouse.Host, + " Port "+strconv.Itoa(desired.ClickHouse.Port), + " URI /?query="+insertDNS, + " Format json_lines", + " http_user ${CH_USER}", + " http_passwd ${CH_PASSWORD}", + " json_date_key timestamp", + " json_date_format epoch", + " workers 1", + " net.keepalive On", + " Retry_Limit False", + ) + if useTLS { + lines = append(lines, " tls On") + if desired.ClickHouse.TLSSkipVerify { + lines = append(lines, " tls.verify Off") + } else { + lines = append(lines, " tls.verify On") + } + if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" { + lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName)) + } } + lines = append(lines, "") + } + + return strings.Join(lines, "\n"), nil +} + +func renderManagedEnv(ch *systemconfigs.ClickHouseSetting) string { + user := "default" + password := "" + if ch != nil { + if strings.TrimSpace(ch.User) != "" { + user = strings.TrimSpace(ch.User) + } + password = ch.Password + } + return "CH_USER=" + strconv.Quote(user) + "\n" + + "CH_PASSWORD=" + strconv.Quote(password) + "\n" +} + +func renderManagedMeta(desired *fluentBitDesiredConfig, configContent string, parserContent string, envContent string) (string, *fluentBitManagedMeta, error) { + hashInput := configContent + "\n---\n" + parserContent + "\n---\n" + envContent + "\n---\n" + strings.Join(desired.Roles, ",") + hashBytes := sha256.Sum256([]byte(hashInput)) + hash := fmt.Sprintf("%x", hashBytes[:]) + + meta := &fluentBitManagedMeta{ + Roles: desired.Roles, + Hash: hash, + UpdatedAt: time.Now().Unix(), + SourceVersion: teaconst.Version, + } + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return "", nil, fmt.Errorf("encode fluent-bit managed metadata failed: %w", err) + } + return string(data) + "\n", meta, nil +} + +func (this *BaseInstaller) copyLocalFileToRemote(tempDir string, localPath string, remotePath string, mode os.FileMode) error { + tempFile := tempDir + "/" + filepath.Base(remotePath) + if err := this.client.Copy(localPath, tempFile, mode); err != nil { + return fmt.Errorf("upload fluent-bit file '%s' failed: %w", localPath, err) + } + _, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath)) + if err != nil { + return fmt.Errorf("install remote fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr) } return nil } -func (this *BaseInstaller) remoteFileExists(path string) (bool, error) { - stdout, stderr, err := this.client.Exec("if [ -f \"" + path + "\" ]; then echo 1; else echo 0; fi") - if err != nil { - return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr) +func (this *BaseInstaller) writeRemoteFileByTemp(tempDir string, remotePath string, content string, mode os.FileMode) error { + tempFile := tempDir + "/" + filepath.Base(remotePath) + ".tmp" + if _, err := this.client.WriteFile(tempFile, []byte(content)); err != nil { + return fmt.Errorf("write temp fluent-bit file '%s' failed: %w", tempFile, err) } - return strings.TrimSpace(stdout) == "1", nil + + _, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath)) + if err != nil { + return fmt.Errorf("write managed fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr) + } + return nil } -func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) { - stdout, stderr, err := this.client.Exec("if grep -F \"" + pattern + "\" \"" + path + "\" >/dev/null 2>&1; then echo 1; else echo 0; fi") - if err != nil { - return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr) - } - return strings.TrimSpace(stdout) == "1", nil -} - -func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied bool) error { +func (this *BaseInstaller) ensureFluentBitService(tempDir string, binPath string, configChanged bool) error { _, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " + "cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" + "[Unit]\n" + @@ -319,26 +748,90 @@ func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied b "EOF\n" + "fi") - stdout, stderr, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " + - "if [ \"" + boolToString(configCopied) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " + - "else systemctl start " + fluentBitServiceName + "; fi; else echo no-systemctl; fi") + stdout, _, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then echo 1; else echo 0; fi") if err != nil { - return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr) + return fmt.Errorf("check systemctl failed: %w", err) } - if strings.TrimSpace(stdout) == "no-systemctl" { - _, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1") - if runningErr != nil { - _, stderr, err = this.client.Exec(binPath + " -c " + fluentBitMainConfigFile + " >/dev/null 2>&1 &") - if err != nil { - return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr) - } + if strings.TrimSpace(stdout) == "1" { + dropInChanged, err := this.ensureServiceDropIn(tempDir, binPath) + if err != nil { + return err + } + + restartRequired := configChanged || dropInChanged + _, stderr, err := this.client.Exec("systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; " + + "if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " + + "if [ \"" + boolToString(restartRequired) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " + + "else systemctl start " + fluentBitServiceName + "; fi") + if err != nil { + return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr) + } + return nil + } + + if configChanged { + _, _, _ = this.client.Exec("pkill -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1 || true") + } + + _, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1") + if runningErr != nil { + startCmd := "set -a; [ -f " + shQuote(fluentBitManagedEnvFile) + " ] && . " + shQuote(fluentBitManagedEnvFile) + "; set +a; " + + shQuote(binPath) + " -c " + shQuote(fluentBitMainConfigFile) + " >/dev/null 2>&1 &" + _, stderr, err := this.client.Exec(startCmd) + if err != nil { + return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr) } } return nil } +func (this *BaseInstaller) ensureServiceDropIn(tempDir string, binPath string) (bool, error) { + _, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitDropInDir)) + if err != nil { + return false, fmt.Errorf("prepare fluent-bit drop-in dir failed: %w, stderr: %s", err, stderr) + } + + content := "[Service]\n" + + "EnvironmentFile=-" + fluentBitManagedEnvFile + "\n" + + "ExecStart=\n" + + "ExecStart=" + binPath + " -c " + fluentBitMainConfigFile + "\n" + + existing, _, _ := this.client.Exec("if [ -f " + shQuote(fluentBitDropInFile) + " ]; then cat " + shQuote(fluentBitDropInFile) + "; fi") + if existing == content { + return false, nil + } + + if err := this.writeRemoteFileByTemp(tempDir, fluentBitDropInFile, content, 0644); err != nil { + return false, err + } + return true, nil +} + +func (this *BaseInstaller) remoteFileExists(path string) (bool, error) { + stdout, stderr, err := this.client.Exec("if [ -f " + shQuote(path) + " ]; then echo 1; else echo 0; fi") + if err != nil { + return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr) + } + return strings.TrimSpace(stdout) == "1", nil +} + +func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) { + stdout, stderr, err := this.client.Exec("if grep -F " + shQuote(pattern) + " " + shQuote(path) + " >/dev/null 2>&1; then echo 1; else echo 0; fi") + if err != nil { + return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr) + } + return strings.TrimSpace(stdout) == "1", nil +} + +func shQuote(value string) string { + if value == "" { + return "''" + } + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + func boolToString(v bool) string { if v { return "1" diff --git a/EdgeAdmin/build/build.sh b/EdgeAdmin/build/build.sh index d044751..a09eca2 100644 --- a/EdgeAdmin/build/build.sh +++ b/EdgeAdmin/build/build.sh @@ -34,7 +34,7 @@ function build() { done VERSION=$(lookup-version "$ROOT"/../internal/const/const.go) - # 生成 zip 文件名时不包含 plus 标记 + # 鐢熸垚 zip 鏂囦欢鍚嶆椂涓嶅寘鍚?plus 鏍囪 if [ "${TAG}" = "plus" ]; then ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip" else @@ -97,7 +97,7 @@ function build() { done fi - # 查找 EdgeAPI zip 文件时不包含 plus 标记 + # 鏌ユ壘 EdgeAPI zip 鏂囦欢鏃朵笉鍖呭惈 plus 鏍囪 if [ "${TAG}" = "plus" ]; then EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip" else @@ -107,7 +107,44 @@ function build() { cd "$DIST"/ || exit unzip -q "$(basename "$EDGE_API_ZIP_FILE")" rm -f "$(basename "$EDGE_API_ZIP_FILE")" - # 删除 MaxMind 数据库文件（使用嵌入的数据库，不需要外部文件） + + # ensure edge-api package always contains fluent-bit templates/packages + FLUENT_ROOT="$ROOT/../../deploy/fluent-bit" + FLUENT_DIST="$DIST/edge-api/deploy/fluent-bit" + if [ -d "$FLUENT_ROOT" ]; then + verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || exit 1 + rm -rf "$FLUENT_DIST" + mkdir -p "$FLUENT_DIST" + + FLUENT_FILES=( + "fluent-bit.conf" + "fluent-bit-dns.conf" + "fluent-bit-https.conf" + "fluent-bit-dns-https.conf" + "fluent-bit-windows.conf" + "fluent-bit-windows-https.conf" + "parsers.conf" + "clickhouse-upstream.conf" + "clickhouse-upstream-windows.conf" + "logrotate.conf" + "README.md" + ) + for file in "${FLUENT_FILES[@]}"; do + if [ -f "$FLUENT_ROOT/$file" ]; then + cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/" + fi + done + + if [ -d "$FLUENT_ROOT/packages" ]; then + cp -R "$FLUENT_ROOT/packages" "$FLUENT_DIST/" + fi + + rm -f "$FLUENT_DIST/.gitignore" + rm -f "$FLUENT_DIST"/logs.db* + rm -rf "$FLUENT_DIST/storage" + fi + + # 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級 find . -name "*.mmdb" -type f -delete find . -type d -name "iplibrary" -empty -delete cd - || exit @@ -150,7 +187,7 @@ function build() { #find "$DIST" -name "*.css.map" -delete #find "$DIST" -name "*.js.map" -delete - # 删除 MaxMind 数据库文件（使用嵌入的数据库，不需要外部文件） + # 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級 find "$DIST" -name "*.mmdb" -type f -delete find "$DIST" -type d -name "iplibrary" -empty -delete @@ -167,6 +204,39 @@ function build() { echo "[done]" } +function verify_fluent_bit_package_matrix() { + FLUENT_ROOT=$1 + ARCH=$2 + REQUIRED_FILES=() + if [ "$ARCH" = "amd64" ]; then + REQUIRED_FILES=( + "packages/linux-amd64/fluent-bit_4.2.2_amd64.deb" + "packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm" + ) + elif [ "$ARCH" = "arm64" ]; then + REQUIRED_FILES=( + "packages/linux-arm64/fluent-bit_4.2.2_arm64.deb" + "packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm" + ) + else + echo "[error] unsupported arch for fluent-bit package validation: $ARCH" + return 1 + fi + + MISSING=0 + for FILE in "${REQUIRED_FILES[@]}"; do + if [ ! -f "$FLUENT_ROOT/$FILE" ]; then + echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE" + MISSING=1 + fi + done + + if [ "$MISSING" -ne 0 ]; then + return 1 + fi + return 0 +} + function lookup-version() { FILE=$1 VERSION_DATA=$(cat "$FILE") diff --git a/EdgeAdmin/internal/web/actions/default/db/clickhouse.go b/EdgeAdmin/internal/web/actions/default/db/clickhouse.go index 4303362..d9a193e 100644 --- a/EdgeAdmin/internal/web/actions/default/db/clickhouse.go +++ b/EdgeAdmin/internal/web/actions/default/db/clickhouse.go @@ -3,13 +3,18 @@ package db import ( + "crypto/tls" "encoding/json" + "fmt" + "net/http" + "strings" + "time" + "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" "github.com/TeaOSLab/EdgeCommon/pkg/langs/codes" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" "github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs" "github.com/iwind/TeaGo/actions" - "strings" ) const clickhouseConfigCode = "clickhouseConfig" @@ -29,29 +34,37 @@ func (this *ClickHouseAction) RunGet(params struct{}) { this.ErrorPage(err) return } - cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default", Scheme: "http"} + cfg := &systemconfigs.ClickHouseSetting{Port: 8443, Database: "default", Scheme: "https"} if len(resp.ValueJSON) > 0 { _ = json.Unmarshal(resp.ValueJSON, cfg) } if cfg.Port <= 0 { - cfg.Port = 8123 + cfg.Port = 8443 } if cfg.Database == "" { cfg.Database = "default" } if strings.TrimSpace(cfg.Scheme) == "" { - cfg.Scheme = "http" + cfg.Scheme = "https" } this.Data["config"] = map[string]interface{}{ - "host": cfg.Host, - "port": cfg.Port, - "user": cfg.User, - "password": cfg.Password, - "database": cfg.Database, - "scheme": cfg.Scheme, - "tlsSkipVerify": cfg.TLSSkipVerify, - "tlsServerName": cfg.TLSServerName, + "host": cfg.Host, + "port": cfg.Port, + "user": cfg.User, + "password": cfg.Password, + "database": cfg.Database, + "scheme": cfg.Scheme, } + + // 自动检测连接状态 + connStatus := "unconfigured" // unconfigured / connected / disconnected + connError := "" + if strings.TrimSpace(cfg.Host) != "" { + connStatus, connError = this.probeClickHouse(cfg) + } + this.Data["connStatus"] = connStatus + this.Data["connError"] = connError + this.Show() } @@ -62,20 +75,18 @@ func (this *ClickHouseAction) RunPost(params struct { Password string Database string Scheme string - TLSSkipVerify bool - TLSServerName string Must *actions.Must }) { defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0) - if params.Port <= 0 { - params.Port = 8123 - } if params.Database == "" { params.Database = "default" } - if params.Scheme != "https" { - params.Scheme = "http" + if params.Scheme != "http" { + params.Scheme = "https" + } + if params.Port <= 0 { + params.Port = 8443 } password := params.Password if password == "" { @@ -94,8 +105,8 @@ func (this *ClickHouseAction) RunPost(params struct { Password: password, Database: params.Database, Scheme: params.Scheme, - TLSSkipVerify: params.TLSSkipVerify, - TLSServerName: strings.TrimSpace(params.TLSServerName), + TLSSkipVerify: true, + TLSServerName: "", } valueJSON, err := json.Marshal(cfg) if err != nil { @@ -112,3 +123,51 @@ func (this *ClickHouseAction) RunPost(params struct { } this.Success() } + +// probeClickHouse 快速检测 ClickHouse 连接状态（SELECT 1） +func (this *ClickHouseAction) probeClickHouse(cfg *systemconfigs.ClickHouseSetting) (status string, errMsg string) { + scheme := strings.ToLower(strings.TrimSpace(cfg.Scheme)) + if scheme == "" { + scheme = "https" + } + port := cfg.Port + if port <= 0 { + port = 8443 + } + db := cfg.Database + if db == "" { + db = "default" + } + + testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s", scheme, cfg.Host, port, db) + + transport := &http.Transport{} + if scheme == "https" { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + client := &http.Client{ + Timeout: 3 * time.Second, + Transport: transport, + } + + req, err := http.NewRequest(http.MethodGet, testURL, nil) + if err != nil { + return "disconnected", err.Error() + } + if cfg.User != "" || cfg.Password != "" { + req.SetBasicAuth(cfg.User, cfg.Password) + } + + resp, err := client.Do(req) + if err != nil { + return "disconnected", err.Error() + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "disconnected", fmt.Sprintf("HTTP %d", resp.StatusCode) + } + return "connected", "" +} diff --git a/EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go b/EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go index f1c6575..cec7f0a 100644 --- a/EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go +++ b/EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go @@ -17,7 +17,7 @@ func (this *ClickHouseAction) Init() { func (this *ClickHouseAction) RunGet(params struct{}) { this.Data["mainTab"] = "clickhouse" this.Data["config"] = map[string]interface{}{ - "host": "", "port": 8123, "user": "", "password": "", "database": "default", + "host": "", "port": 8443, "user": "", "password": "", "database": "default", "scheme": "https", } this.Show() } diff --git a/EdgeAdmin/internal/web/actions/default/db/init.go b/EdgeAdmin/internal/web/actions/default/db/init.go index 2289818..34dd093 100644 --- a/EdgeAdmin/internal/web/actions/default/db/init.go +++ b/EdgeAdmin/internal/web/actions/default/db/init.go @@ -25,6 +25,7 @@ func init() { Get("/logs", new(LogsAction)). Post("/status", new(StatusAction)). GetPost("/clickhouse", new(ClickHouseAction)). + Post("/testClickhouse", new(TestClickHouseAction)). EndAll() }) } diff --git a/EdgeAdmin/internal/web/actions/default/db/testClickhouse.go b/EdgeAdmin/internal/web/actions/default/db/testClickhouse.go new file mode 100644 index 0000000..27ccb3b --- /dev/null +++ b/EdgeAdmin/internal/web/actions/default/db/testClickhouse.go @@ -0,0 +1,92 @@ +//go:build plus + +package db + +import ( + "crypto/tls" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" + "github.com/iwind/TeaGo/actions" +) + +type TestClickHouseAction struct { + actionutils.ParentAction +} + +func (this *TestClickHouseAction) Init() { + this.Nav("db", "db", "clickhouse") +} + +func (this *TestClickHouseAction) RunPost(params struct { + Host string + Port int + User string + Password string + Database string + Scheme string + + Must *actions.Must +}) { + params.Must. + Field("host", params.Host). + Require("请输入 ClickHouse 连接地址") + + if params.Database == "" { + params.Database = "default" + } + scheme := "https" + if strings.EqualFold(params.Scheme, "http") { + scheme = "http" + } + if params.Port <= 0 { + params.Port = 8443 + } + + // 构造测试请求 + testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s", + scheme, params.Host, params.Port, params.Database) + + transport := &http.Transport{} + if scheme == "https" { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: transport, + } + + req, err := http.NewRequest(http.MethodGet, testURL, nil) + if err != nil { + this.Fail("请求构造失败: " + err.Error()) + return + } + if params.User != "" || params.Password != "" { + req.SetBasicAuth(params.User, params.Password) + } + + resp, err := client.Do(req) + if err != nil { + this.Fail("连接失败: " + err.Error()) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + msg := strings.TrimSpace(string(body)) + if len(msg) > 200 { + msg = msg[:200] + "..." + } + this.Fail(fmt.Sprintf("ClickHouse 返回 HTTP %d: %s", resp.StatusCode, msg)) + return + } + + this.Success() +} diff --git a/EdgeAdmin/internal/web/actions/default/db/testClickhouse_stub.go b/EdgeAdmin/internal/web/actions/default/db/testClickhouse_stub.go new file mode 100644 index 0000000..fb06f3e --- /dev/null +++ b/EdgeAdmin/internal/web/actions/default/db/testClickhouse_stub.go @@ -0,0 +1,23 @@ +//go:build !plus + +package db + +import ( + "github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils" + "github.com/iwind/TeaGo/actions" +) + +type TestClickHouseAction struct { + actionutils.ParentAction +} + +func (this *TestClickHouseAction) Init() { + this.Nav("db", "db", "clickhouse") +} + +func (this *TestClickHouseAction) RunPost(params struct { + Host string + Must *actions.Must +}) { + this.Fail("请使用商业版以测试 ClickHouse 连接") +} diff --git a/EdgeAdmin/web/public/js/components.js b/EdgeAdmin/web/public/js/components.js index 99daa3f..5238a9f 100644 --- a/EdgeAdmin/web/public/js/components.js +++ b/EdgeAdmin/web/public/js/components.js @@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
[{{accessLog.node.name}}节点] - - [网站] - [网站] + [{{accessLog.region}}] {{accessLog.remoteAddr}} [{{accessLog.timeLocal}}] "{{accessLog.requestMethod}} {{accessLog.scheme}}://{{accessLog.host}}{{accessLog.requestURI}} {{accessLog.proto}}" {{accessLog.status}} @@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", { - 耗时:{{formatCost(accessLog.requestTime)}} ms   ({{accessLog.humanTime}}) + 网站  
` diff --git a/EdgeAdmin/web/public/js/components.src.js b/EdgeAdmin/web/public/js/components.src.js index 99daa3f..5238a9f 100644 --- a/EdgeAdmin/web/public/js/components.src.js +++ b/EdgeAdmin/web/public/js/components.src.js @@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
[{{accessLog.node.name}}节点] - - [网站] - [网站] + [{{accessLog.region}}] {{accessLog.remoteAddr}} [{{accessLog.timeLocal}}] "{{accessLog.requestMethod}} {{accessLog.scheme}}://{{accessLog.host}}{{accessLog.requestURI}} {{accessLog.proto}}" {{accessLog.status}} @@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", { - 耗时:{{formatCost(accessLog.requestTime)}} ms   ({{accessLog.humanTime}}) + 网站  
` diff --git a/EdgeAdmin/web/views/@default/db/clickHouse.html b/EdgeAdmin/web/views/@default/db/clickHouse.html index e0e382b..ca9fb63 100644 --- a/EdgeAdmin/web/views/@default/db/clickHouse.html +++ b/EdgeAdmin/web/views/@default/db/clickHouse.html @@ -1,70 +1,76 @@ {$layout} {$template "menu"} -

ClickHouse 配置

-

用于访问日志列表查询（logs_ingest 表）。配置后，访问日志列表将优先从 ClickHouse 读取；不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。

+
+

ClickHouse 配置

+ + 已连接 + + + 已断开 + + + 未配置 + +
+

配置后，需要在网站列表-日志策略中创建"文件+ClickHouse"的策略，才能将访问日志绕过api组件直接由边缘节点发送给ClickHouse。

-
+ - - - - - - - -
连接地址（Host） - +

ClickHouse 服务器地址。
协议（Scheme） - + + -

默认 HTTP；选择 HTTPS 时将启用 TLS 连接。

+

默认 HTTPS；当前后台固定跳过证书校验。
端口（Port） - -

接口端口，HTTP 默认 8123，HTTPS 常用 8443（以你的 ClickHouse 实际配置为准）。

+ +

接口端口默认 8443（HTTPS）；请与 ClickHouse 实际开放端口保持一致。
用户名（User） - +
密码（Password） - +

留空则不修改已保存的密码。
TLS 跳过证书校验 - -

仅测试环境建议开启；生产建议关闭并使用受信任证书。

-
TLS Server Name - -

可选；当 ClickHouse 证书域名与连接 Host 不一致时使用。

-
数据库名（Database） - +

logs_ingest 表所在库，默认 default。
- -
+
+ + + {{testResult}} +
+ \ No newline at end of file diff --git a/EdgeAdmin/web/views/@default/db/clickHouse.js b/EdgeAdmin/web/views/@default/db/clickHouse.js index 3ccfaf1..db1a608 100644 --- a/EdgeAdmin/web/views/@default/db/clickHouse.js +++ b/EdgeAdmin/web/views/@default/db/clickHouse.js @@ -1,4 +1,22 @@ Tea.context(function () { + var config = this.config || {} + this.form = { + host: config.host || "", + scheme: config.scheme || "https", + port: config.port > 0 ? config.port : 8443, + user: config.user || "", + password: "", + database: config.database || "default", + } + + this.isTesting = false + this.testResult = "" + this.testOk = false + + // 页面加载时的连接状态（后端自动检测） + this.connStatus = this.connStatus || "unconfigured" + this.connError = this.connError || "" + this.success = function () { teaweb.success("保存成功") } @@ -9,4 +27,50 @@ Tea.context(function () { Tea.Vue.success() }) } + this.testConnection = function () { + var that = Tea.Vue + that.isTesting = true + that.testResult = "" + + var form = document.querySelector("form") + var fd = new FormData(form) + fd.set("host", that.form.host || "") + fd.set("scheme", that.form.scheme || "https") + fd.set("port", String(that.form.port > 0 ? that.form.port : 8443)) + fd.set("user", that.form.user || "") + fd.set("password", that.form.password || "") + fd.set("database", that.form.database || "default") + + var xhr = new XMLHttpRequest() + xhr.open("POST", Tea.url("/db/testClickhouse"), true) + xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest") + xhr.timeout = 10000 + xhr.onload = function () { + that.isTesting = false + try { + var resp = JSON.parse(xhr.responseText) + if (resp.code === 200) { + that.testOk = true + that.testResult = "✅ 连接成功" + } else { + that.testOk = false + that.testResult = "❌ " + (resp.message || "连接失败") + } + } catch (e) { + that.testOk = false + that.testResult = "❌ 响应解析失败" + } + } + xhr.onerror = function () { + that.isTesting = false + that.testOk = false + that.testResult = "❌ 网络请求失败" + } + xhr.ontimeout = function () { + that.isTesting = false + that.testOk = false + that.testResult = "❌ 请求超时" + } + xhr.send(fd) + } }) diff --git a/EdgeCommon/pkg/dnsconfigs/ns_node_config.go b/EdgeCommon/pkg/dnsconfigs/ns_node_config.go index 64d4188..458c4ed 100644 --- a/EdgeCommon/pkg/dnsconfigs/ns_node_config.go +++ b/EdgeCommon/pkg/dnsconfigs/ns_node_config.go @@ -19,6 +19,7 @@ type NSNodeConfig struct { AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"` AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"` AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"` + AccessLogRotate *serverconfigs.AccessLogRotateConfig `yaml:"accessLogRotate" json:"accessLogRotate"` RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"` DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"` AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"` diff --git a/EdgeCommon/pkg/serverconfigs/access_log_storage_file.go b/EdgeCommon/pkg/serverconfigs/access_log_storage_file.go index 3e91534..8d068eb 100644 --- a/EdgeCommon/pkg/serverconfigs/access_log_storage_file.go +++ b/EdgeCommon/pkg/serverconfigs/access_log_storage_file.go @@ -2,8 +2,65 @@ package serverconfigs +const ( + DefaultAccessLogRotateMaxSizeMB = 256 + DefaultAccessLogRotateMaxBackups = 14 + DefaultAccessLogRotateMaxAgeDays = 7 +) + +// AccessLogRotateConfig 文件轮转配置。 +type AccessLogRotateConfig struct { + MaxSizeMB int `yaml:"maxSizeMB" json:"maxSizeMB"` // 单文件最大大小（MB） + MaxBackups int `yaml:"maxBackups" json:"maxBackups"` // 保留历史文件数 + MaxAgeDays int `yaml:"maxAgeDays" json:"maxAgeDays"` // 保留天数 + Compress *bool `yaml:"compress" json:"compress"` // 是否压缩历史文件 + LocalTime *bool `yaml:"localTime" json:"localTime"` // 轮转时间使用本地时区 +} + // AccessLogFileStorageConfig 文件存储配置 type AccessLogFileStorageConfig struct { - Path string `yaml:"path" json:"path"` // 文件路径，支持变量：${year|month|week|day|hour|minute|second} - AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录 + Path string `yaml:"path" json:"path"` // 文件路径，支持变量：${year|month|week|day|hour|minute|second} + AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录 + Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 文件轮转配置 +} + +// NewDefaultAccessLogRotateConfig 默认轮转配置。 +func NewDefaultAccessLogRotateConfig() *AccessLogRotateConfig { + compress := false + localTime := true + return &AccessLogRotateConfig{ + MaxSizeMB: DefaultAccessLogRotateMaxSizeMB, + MaxBackups: DefaultAccessLogRotateMaxBackups, + MaxAgeDays: DefaultAccessLogRotateMaxAgeDays, + Compress: &compress, + LocalTime: &localTime, + } +} + +// Normalize 归一化轮转配置，空值/非法值回退默认。 +func (c *AccessLogRotateConfig) Normalize() *AccessLogRotateConfig { + defaultConfig := NewDefaultAccessLogRotateConfig() + if c == nil { + return defaultConfig + } + + if c.MaxSizeMB > 0 { + defaultConfig.MaxSizeMB = c.MaxSizeMB + } + if c.MaxBackups > 0 { + defaultConfig.MaxBackups = c.MaxBackups + } + if c.MaxAgeDays > 0 { + defaultConfig.MaxAgeDays = c.MaxAgeDays + } + if c.Compress != nil { + v := *c.Compress + defaultConfig.Compress = &v + } + if c.LocalTime != nil { + v := *c.LocalTime + defaultConfig.LocalTime = &v + } + + return defaultConfig } diff --git a/EdgeCommon/pkg/serverconfigs/global_server_config.go b/EdgeCommon/pkg/serverconfigs/global_server_config.go index 3cb65e5..2b84f4a 100644 --- a/EdgeCommon/pkg/serverconfigs/global_server_config.go +++ b/EdgeCommon/pkg/serverconfigs/global_server_config.go @@ -26,6 +26,7 @@ func NewGlobalServerConfig() *GlobalServerConfig { config.HTTPAccessLog.EnableResponseHeaders = true config.HTTPAccessLog.EnableCookies = true config.HTTPAccessLog.EnableServerNotFound = true + config.HTTPAccessLog.Rotate = NewDefaultAccessLogRotateConfig() config.Log.RecordServerError = false @@ -79,6 +80,7 @@ type GlobalServerConfig struct { EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志 WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标：文件/MySQL/ClickHouse（双写/单写） FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径（用于节点侧复用） + Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 本地日志轮转配置（lumberjack） } `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置 Stat struct { diff --git a/EdgeDNS/build/build.sh b/EdgeDNS/build/build.sh index 0beb3e2..4d5ea3c 100644 --- a/EdgeDNS/build/build.sh +++ b/EdgeDNS/build/build.sh @@ -107,11 +107,12 @@ function copy_fluent_bit_assets() { echo "[error] fluent-bit source directory not found: $FLUENT_ROOT" return 1 fi + verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1 rm -rf "$FLUENT_DIST" mkdir -p "$FLUENT_DIST" - for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do + for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do if [ -f "$FLUENT_ROOT/$file" ]; then cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/" fi @@ -129,6 +130,43 @@ function copy_fluent_bit_assets() { fi fi + rm -f "$FLUENT_DIST/.gitignore" + rm -f "$FLUENT_DIST"/logs.db* + rm -rf "$FLUENT_DIST/storage" + + return 0 +} + +function verify_fluent_bit_package_matrix() { + FLUENT_ROOT=$1 + ARCH=$2 + REQUIRED_FILES=() + if [ "$ARCH" = "amd64" ]; then + REQUIRED_FILES=( + "packages/linux-amd64/fluent-bit_4.2.2_amd64.deb" + "packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm" + ) + elif [ "$ARCH" = "arm64" ]; then + REQUIRED_FILES=( + "packages/linux-arm64/fluent-bit_4.2.2_arm64.deb" + "packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm" + ) + else + echo "[error] unsupported arch for fluent-bit package validation: $ARCH" + return 1 + fi + + MISSING=0 + for FILE in "${REQUIRED_FILES[@]}"; do + if [ ! -f "$FLUENT_ROOT/$FILE" ]; then + echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE" + MISSING=1 + fi + done + + if [ "$MISSING" -ne 0 ]; then + return 1 + fi return 0 } diff --git a/EdgeDNS/go.mod b/EdgeDNS/go.mod index b470d94..899785b 100644 --- a/EdgeDNS/go.mod +++ b/EdgeDNS/go.mod @@ -13,6 +13,7 @@ require ( github.com/mdlayher/netlink v1.7.2 github.com/miekg/dns v1.1.58 github.com/shirou/gopsutil/v3 v3.24.2 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 golang.org/x/sys v0.38.0 google.golang.org/grpc v1.78.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/EdgeDNS/go.sum b/EdgeDNS/go.sum index ce34f7c..c152b07 100644 --- a/EdgeDNS/go.sum +++ b/EdgeDNS/go.sum @@ -111,6 +111,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/EdgeDNS/internal/accesslogs/dns_file_writer.go b/EdgeDNS/internal/accesslogs/dns_file_writer.go index 35b5500..d9ac5da 100644 --- a/EdgeDNS/internal/accesslogs/dns_file_writer.go +++ b/EdgeDNS/internal/accesslogs/dns_file_writer.go @@ -9,7 +9,9 @@ import ( "sync" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeDNS/internal/remotelogs" + "gopkg.in/natefinch/lumberjack.v2" ) const ( @@ -30,18 +32,23 @@ func SharedDNSFileWriter() *DNSFileWriter { return sharedDNSFileWriter } -// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集. +// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件，供 Fluent Bit 采集。 +// 文件轮转由 lumberjack 内建完成。 type DNSFileWriter struct { - dir string - mu sync.Mutex - file *os.File - inited bool + dir string + mu sync.Mutex + file *lumberjack.Logger + rotateConfig *serverconfigs.AccessLogRotateConfig + inited bool } // NewDNSFileWriter 创建 DNS 本地日志写入器. func NewDNSFileWriter() *DNSFileWriter { logDir := resolveDefaultDNSLogDir() - return &DNSFileWriter{dir: logDir} + return &DNSFileWriter{ + dir: logDir, + rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(), + } } func resolveDefaultDNSLogDir() string { @@ -102,6 +109,25 @@ func (w *DNSFileWriter) SetDir(dir string) { w.dir = dir } +// SetRotateConfig 更新日志轮转配置并重建 writer。 +func (w *DNSFileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) { + normalized := config.Normalize() + + w.mu.Lock() + defer w.mu.Unlock() + + if equalDNSRotateConfig(w.rotateConfig, normalized) { + return + } + + if w.file != nil { + _ = w.file.Close() + w.file = nil + } + w.inited = false + w.rotateConfig = normalized +} + // EnsureInit 在启动时预创建目录与 access.log. func (w *DNSFileWriter) EnsureInit() error { if w.dir == "" { @@ -127,13 +153,16 @@ func (w *DNSFileWriter) init() error { return err } - fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error()) - return err + rotateConfig := w.rotateConfig.Normalize() + w.file = &lumberjack.Logger{ + Filename: filepath.Join(w.dir, "access.log"), + MaxSize: rotateConfig.MaxSizeMB, + MaxBackups: rotateConfig.MaxBackups, + MaxAge: rotateConfig.MaxAgeDays, + Compress: *rotateConfig.Compress, + LocalTime: *rotateConfig.LocalTime, } - w.file = fp w.inited = true return nil } @@ -148,14 +177,14 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) { } w.mu.Lock() - fp := w.file + file := w.file w.mu.Unlock() - if fp == nil { + if file == nil { return } - for _, log := range logs { - ingestLog := FromNSAccessLog(log, clusterId) + for _, logItem := range logs { + ingestLog := FromNSAccessLog(logItem, clusterId) if ingestLog == nil { continue } @@ -163,11 +192,11 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) { if err != nil { continue } - _, _ = fp.Write(append(line, '\n')) + _, _ = file.Write(append(line, '\n')) } } -// Reopen 关闭并重新打开日志文件（配合 logrotate）. +// Reopen 关闭并重建日志 writer（供兼容调用）. func (w *DNSFileWriter) Reopen() error { w.mu.Lock() if w.file != nil { @@ -197,3 +226,14 @@ func (w *DNSFileWriter) Close() error { } return nil } + +func equalDNSRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool { + if left == nil || right == nil { + return left == right + } + return left.MaxSizeMB == right.MaxSizeMB && + left.MaxBackups == right.MaxBackups && + left.MaxAgeDays == right.MaxAgeDays && + *left.Compress == *right.Compress && + *left.LocalTime == *right.LocalTime +} diff --git a/EdgeDNS/internal/nodes/manager_node_config.go b/EdgeDNS/internal/nodes/manager_node_config.go index 5066b07..fd407d1 100644 --- a/EdgeDNS/internal/nodes/manager_node_config.go +++ b/EdgeDNS/internal/nodes/manager_node_config.go @@ -110,6 +110,7 @@ func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) { teaconst.IsPlus = config.IsPlus accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath) + accesslogs.SharedDNSFileWriter().SetRotateConfig(config.AccessLogRotate) needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse if needWriteFile { diff --git a/EdgeNode/build/build.sh b/EdgeNode/build/build.sh index a77ef1f..f648399 100644 --- a/EdgeNode/build/build.sh +++ b/EdgeNode/build/build.sh @@ -181,11 +181,12 @@ function copy_fluent_bit_assets() { echo "[error] fluent-bit source directory not found: $FLUENT_ROOT" return 1 fi + verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1 rm -rf "$FLUENT_DIST" mkdir -p "$FLUENT_DIST" - for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do + for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do if [ -f "$FLUENT_ROOT/$file" ]; then cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/" fi @@ -203,6 +204,43 @@ function copy_fluent_bit_assets() { fi fi + rm -f "$FLUENT_DIST/.gitignore" + rm -f "$FLUENT_DIST"/logs.db* + rm -rf "$FLUENT_DIST/storage" + + return 0 +} + +function verify_fluent_bit_package_matrix() { + FLUENT_ROOT=$1 + ARCH=$2 + REQUIRED_FILES=() + if [ "$ARCH" = "amd64" ]; then + REQUIRED_FILES=( + "packages/linux-amd64/fluent-bit_4.2.2_amd64.deb" + "packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm" + ) + elif [ "$ARCH" = "arm64" ]; then + REQUIRED_FILES=( + "packages/linux-arm64/fluent-bit_4.2.2_arm64.deb" + "packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm" + ) + else + echo "[error] unsupported arch for fluent-bit package validation: $ARCH" + return 1 + fi + + MISSING=0 + for FILE in "${REQUIRED_FILES[@]}"; do + if [ ! -f "$FLUENT_ROOT/$FILE" ]; then + echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE" + MISSING=1 + fi + done + + if [ "$MISSING" -ne 0 ]; then + return 1 + fi return 0 } diff --git a/EdgeNode/go.mod b/EdgeNode/go.mod index efb3cb6..797b2d3 100644 --- a/EdgeNode/go.mod +++ b/EdgeNode/go.mod @@ -38,6 +38,7 @@ require ( github.com/shirou/gopsutil/v3 v3.22.2 github.com/tdewolff/minify/v2 v2.20.20 github.com/tencentyun/cos-go-sdk-v5 v0.7.41 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f golang.org/x/image v0.15.0 golang.org/x/net v0.47.0 diff --git a/EdgeNode/go.sum b/EdgeNode/go.sum index 32daf30..ba857d7 100644 --- a/EdgeNode/go.sum +++ b/EdgeNode/go.sum @@ -305,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/EdgeNode/internal/accesslogs/file_writer.go b/EdgeNode/internal/accesslogs/file_writer.go index 264df8f..f0f16b6 100644 --- a/EdgeNode/internal/accesslogs/file_writer.go +++ b/EdgeNode/internal/accesslogs/file_writer.go @@ -9,7 +9,9 @@ import ( "sync" "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb" + "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "gopkg.in/natefinch/lumberjack.v2" ) var ( @@ -30,20 +32,23 @@ const ( envLogDir = "EDGE_LOG_DIR" ) -// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 logrotate 与 Fluent Bit 采集 +// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件，便于 Fluent Bit 采集。 +// 文件轮转由 lumberjack 内建完成。 type FileWriter struct { - dir string - mu sync.Mutex - files map[string]*os.File // access.log, waf.log, error.log - inited bool + dir string + mu sync.Mutex + files map[string]*lumberjack.Logger // access.log, waf.log, error.log + rotateConfig *serverconfigs.AccessLogRotateConfig + inited bool } // NewFileWriter 创建本地日志文件写入器 func NewFileWriter() *FileWriter { dir := resolveDefaultLogDir() return &FileWriter{ - dir: dir, - files: make(map[string]*os.File), + dir: dir, + files: make(map[string]*lumberjack.Logger), + rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(), } } @@ -97,9 +102,9 @@ func (w *FileWriter) SetDir(dir string) { return } - for name, f := range w.files { - if f != nil { - _ = f.Close() + for name, file := range w.files { + if file != nil { + _ = file.Close() } w.files[name] = nil } @@ -107,6 +112,27 @@ func (w *FileWriter) SetDir(dir string) { w.dir = dir } +// SetRotateConfig 更新日志轮转配置并重建 writer。 +func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) { + normalized := config.Normalize() + + w.mu.Lock() + defer w.mu.Unlock() + + if equalRotateConfig(w.rotateConfig, normalized) { + return + } + + for name, file := range w.files { + if file != nil { + _ = file.Close() + } + w.files[name] = nil + } + w.inited = false + w.rotateConfig = normalized +} + // IsEnabled 是否启用落盘（目录非空即视为启用） func (w *FileWriter) IsEnabled() bool { return w.dir != "" @@ -138,17 +164,24 @@ func (w *FileWriter) init() error { if w.files[name] != nil { continue } - fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error()) - continue - } - w.files[name] = fp + w.files[name] = w.newLogger(name) } w.inited = true return nil } +func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger { + rotateConfig := w.rotateConfig.Normalize() + return &lumberjack.Logger{ + Filename: filepath.Join(w.dir, fileName), + MaxSize: rotateConfig.MaxSizeMB, + MaxBackups: rotateConfig.MaxBackups, + MaxAge: rotateConfig.MaxAgeDays, + Compress: *rotateConfig.Compress, + LocalTime: *rotateConfig.LocalTime, + } +} + // Write 将一条访问日志按 log_type 写入对应文件（access.log / waf.log / error.log） func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) { if w.dir == "" { @@ -173,13 +206,12 @@ func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) { fileName = "access.log" } w.mu.Lock() - fp := w.files[fileName] + file := w.files[fileName] w.mu.Unlock() - if fp == nil { + if file == nil { return } - // 单行写入，末尾换行，便于 Fluent Bit / JSON 解析 - _, err = fp.Write(append(line, '\n')) + _, err = file.Write(append(line, '\n')) if err != nil { remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error()) } @@ -194,49 +226,49 @@ func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) { return } w.mu.Lock() - accessFp := w.files["access.log"] - wafFp := w.files["waf.log"] - errorFp := w.files["error.log"] + accessFile := w.files["access.log"] + wafFile := w.files["waf.log"] + errorFile := w.files["error.log"] w.mu.Unlock() - if accessFp == nil && wafFp == nil && errorFp == nil { + if accessFile == nil && wafFile == nil && errorFile == nil { return } - for _, l := range logs { - ingest, logType := FromHTTPAccessLog(l, clusterId) + for _, logItem := range logs { + ingest, logType := FromHTTPAccessLog(logItem, clusterId) line, err := json.Marshal(ingest) if err != nil { continue } line = append(line, '\n') - var fp *os.File + var file *lumberjack.Logger switch logType { case LogTypeWAF: - fp = wafFp + file = wafFile case LogTypeError: - fp = errorFp + file = errorFile default: - fp = accessFp + file = accessFile } - if fp != nil { - _, _ = fp.Write(line) + if file != nil { + _, _ = file.Write(line) } } } -// Reopen 关闭并重新打开所有日志文件（供 logrotate copytruncate 或 SIGHUP 后重开句柄） +// Reopen 关闭并重建所有日志 writer（供 SIGHUP 兼容调用）。 func (w *FileWriter) Reopen() error { if w.dir == "" { return nil } w.mu.Lock() - defer w.mu.Unlock() - for name, f := range w.files { - if f != nil { - _ = f.Close() + for name, file := range w.files { + if file != nil { + _ = file.Close() w.files[name] = nil } } w.inited = false + w.mu.Unlock() return w.init() } @@ -245,9 +277,9 @@ func (w *FileWriter) Close() error { w.mu.Lock() defer w.mu.Unlock() var lastErr error - for name, f := range w.files { - if f != nil { - if err := f.Close(); err != nil { + for name, file := range w.files { + if file != nil { + if err := file.Close(); err != nil { lastErr = err remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err)) } @@ -257,3 +289,14 @@ func (w *FileWriter) Close() error { w.inited = false return lastErr } + +func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool { + if left == nil || right == nil { + return left == right + } + return left.MaxSizeMB == right.MaxSizeMB && + left.MaxBackups == right.MaxBackups && + left.MaxAgeDays == right.MaxAgeDays && + *left.Compress == *right.Compress && + *left.LocalTime == *right.LocalTime +} diff --git a/EdgeNode/internal/nodes/node.go b/EdgeNode/internal/nodes/node.go index 77180c8..4eee53d 100644 --- a/EdgeNode/internal/nodes/node.go +++ b/EdgeNode/internal/nodes/node.go @@ -577,7 +577,7 @@ func (this *Node) listenSignals() { goman.New(func() { for sig := range queue { if sig == syscall.SIGHUP { - // 供 logrotate 等旋转日志后重开句柄 + // 兼容 SIGHUP：重建本地日志 writer if err := accesslogs.SharedFileWriter().Reopen(); err != nil { remotelogs.Error("NODE", "access log file reopen: "+err.Error()) } @@ -890,6 +890,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) { var accessLogFilePath string if config != nil && config.GlobalServerConfig != nil { accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath + accesslogs.SharedFileWriter().SetRotateConfig(config.GlobalServerConfig.HTTPAccessLog.Rotate) } accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath) diff --git a/config.xml b/config.xml new file mode 100644 index 0000000..7097024 --- /dev/null +++ b/config.xml @@ -0,0 +1,1984 @@ + + + + + warning + + + + + + + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + https://{bucket}.s3.amazonaws.com + + + https://storage.googleapis.com/{bucket} + + + https://{bucket}.oss.aliyuncs.com + + + + + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + origin, x-requested-with, x-clickhouse-format, x-clickhouse-user, x-clickhouse-key, Authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+ + + + + + + 8123 + + + 9000 + + + + + + 9004 + + + 9005 + + + 8443 + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + :: + + + + + + + + + + + + + + + + + + + + + + + + 10 + + + + + + + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + + none + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + + + + 0 + 2 + fair_round_robin + + + 1000 + + + 1500000000 + + + + 10000 + + + + + + true + + + 0.9 + + + 4194304 + + + 0 + + + + + + 100000000 + + + + + 100000000 + + + + + + + + + + + + + + + 106700800 + + + + + + /var/lib/clickhouse/caches/ + + false + + + /var/lib/clickhouse/ + + + + + + + + + + /var/lib/clickhouse/tmp/ + + + 1 + 1 + 1 + + + sha256_password + + + 12 + + + + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + + true + + + true + + + true + + + true + + + true + + + false + + + 600 + + + + default + + + SQL_ + + + + + + + + + default + + + + + + + + + true + + 5000000000 + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/ + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + + + 7500 + + 1048576 + + 8192 + + 524288 + + false + + + + + + + + false + system + trace_log
+ + toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + + false + true + + + + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + system + background_schedule_pool_log
+ toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false + + + 30 + + + + + + + + + + + system + error_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false + + + + + system + instrumentation_trace_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false + + + + + system + query_metric_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false + + + + + + + system + iceberg_metadata_log
+ 2000 + 1048576 + 8192 + 524288 + false + + + + system + delta_lake_metadata_log
+ 2000 + 1048576 + 8192 + 524288 + false + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us) + + system + opentelemetry_span_log
+ 7500 + 1048576 + 8192 + 524288 + false + + + + + + system + crash_log
+ + + 1000 + 1024 + 1024 + 512 + true + + + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false + event_date + INTERVAL 30 DAY DELETE + + + + + system + asynchronous_insert_log
+ + 7500 + 1048576 + 8192 + 524288 + false + event_date + event_date + INTERVAL 3 DAY + + + + + system + backup_log
+ toYYYYMM(event_date) + 7500 + + + + + system + s3queue_log
+ toYYYYMM(event_date) + 7500 + + + + + system + blob_storage_log
+ toYYYYMM(event_date) + 7500 + event_date + INTERVAL 30 DAY + + + + + system + aggregated_zookeeper_log
+ toYYYYMM(event_date) + 1000 + event_date + INTERVAL 30 DAY + + + + + system + zookeeper_connection_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY + + + + + + + + + + + + + + *_dictionary.*ml + + + true + + + true + + + *_function.*ml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /clickhouse/task_queue/ddl + + /clickhouse/task_queue/replicas + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + /usr/share/clickhouse/protos/ + + + + + + + + + + + true + true + https://crash.clickhouse.com/ + + + + + + + + + + + + + + + + + + + + + + + + + + backups + + + true + + + + + + + + + + + + + + + + + + + + diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..1b55a81 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,348 @@ +#!/bin/bash +# CloudWAF 部署脚本 - Gname DNS 提供商更新 +# 使用方法: ./deploy.sh [zip文件路径] + +set -e # 遇到错误立即退出 + +# ========== 配置区域 ========== +# 支持环境变量和配置文件 +# 从环境变量读取 +INSTALL_BASE_DIR="${BRAND_INSTALL_PATH:-/usr/local/goedge}" + +# 从配置文件读取（如果存在） +if [ -f "/etc/goedge/brand.conf" ]; then + source /etc/goedge/brand.conf + if [ -n "$BRAND_INSTALL_PATH" ]; then + INSTALL_BASE_DIR="$BRAND_INSTALL_PATH" + fi +fi + +# 如果通过参数传入 zip 文件路径，则使用参数；否则使用默认路径 +ZIP_FILE_RAW="${1:-/tmp/edge-admin-linux-amd64-plus-v1.3.8.zip}" +# 获取绝对路径，防止 cd 到临时目录后找不到文件 +if [[ "$ZIP_FILE_RAW" = /* ]]; then + ZIP_FILE="$ZIP_FILE_RAW" +else + ZIP_FILE="$(pwd)/$ZIP_FILE_RAW" +fi + +TARGET_DIR="${INSTALL_BASE_DIR}/edge-admin" +BACKUP_DIR="${INSTALL_BASE_DIR}/backup_$(date +%Y%m%d_%H%M%S)" +TEMP_DIR="/tmp/edge-admin-update-$(date +%s)" + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 日志函数 +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# ========== 检查前置条件 ========== +check_prerequisites() { + log_info "检查前置条件..." + + # 检查 zip 文件是否存在 + if [ ! -f "$ZIP_FILE" ]; then + log_error "未找到更新包: $ZIP_FILE" + log_info "使用方法: $0 [zip文件路径]" + exit 1 + fi + + # 检查目标目录是否存在 + if [ ! -d "$TARGET_DIR" ]; then + log_error "目标目录不存在: $TARGET_DIR" + exit 1 + fi + + # 检查是否有足够的磁盘空间（至少 500MB） + available_space=$(df "$TARGET_DIR" | tail -1 | awk '{print $4}') + if [ "$available_space" -lt 512000 ]; then + log_warn "可用磁盘空间不足 500MB，可能影响部署" + fi + + log_info "前置条件检查通过" +} + +# ========== 1. 备份关键目录 ========== +backup_files() { + log_info "==========================================" + log_info "步骤 1: 备份关键目录" + log_info "==========================================" + + mkdir -p "$BACKUP_DIR" + + # 备份 EdgeAdmin 配置 + if [ -d "$TARGET_DIR/configs" ]; then + log_info "备份 EdgeAdmin 配置文件..." + cp -r "$TARGET_DIR/configs" "$BACKUP_DIR/edge-admin-configs" + log_info "✅ EdgeAdmin 配置已备份" + else + log_warn "EdgeAdmin configs 目录不存在，跳过备份" + fi + + # 备份 EdgeAdmin 日志（可选） + if [ -d "$TARGET_DIR/logs" ]; then + log_info "备份 EdgeAdmin 日志文件..." + cp -r "$TARGET_DIR/logs" "$BACKUP_DIR/edge-admin-logs" 2>/dev/null || true + fi + + # 备份 EdgeAPI 配置和数据 + if [ -d "$TARGET_DIR/edge-api" ]; then + if [ -d "$TARGET_DIR/edge-api/configs" ]; then + log_info "备份 EdgeAPI 配置文件..." + cp -r "$TARGET_DIR/edge-api/configs" "$BACKUP_DIR/edge-api-configs" + log_info "✅ EdgeAPI 配置已备份" + fi + + if [ -d "$TARGET_DIR/edge-api/logs" ]; then + log_info "备份 EdgeAPI 日志文件..." + cp -r "$TARGET_DIR/edge-api/logs" "$BACKUP_DIR/edge-api-logs" 2>/dev/null || true + fi + + if [ -d "$TARGET_DIR/edge-api/data" ]; then + log_info "备份 EdgeAPI 数据文件..." + cp -r "$TARGET_DIR/edge-api/data" "$BACKUP_DIR/edge-api-data" 2>/dev/null || true + fi + fi + + log_info "✅ 备份完成，备份位置: $BACKUP_DIR" + echo "" +} + +# ========== 2. 停止服务 ========== +stop_services() { + log_info "==========================================" + log_info "步骤 2: 停止服务" + log_info "==========================================" + + if [ -f "$TARGET_DIR/bin/edge-admin" ]; then + log_info "停止 EdgeAdmin 服务..." + "$TARGET_DIR/bin/edge-admin" stop || true + sleep 2 + + # 检查是否还有进程在运行 (精确匹配二进制路径，避免杀死解析 zip 路径的脚本自身) + if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then + log_warn "检测到 edge-admin 进程仍在运行，尝试强制停止..." + pkill -9 -f "$TARGET_DIR/bin/edge-admin" || true + sleep 1 + fi + + log_info "✅ 服务已停止" + else + log_warn "未找到 edge-admin 可执行文件，跳过停止步骤" + fi + echo "" +} + +# ========== 3. 解压新版本 ========== +extract_package() { + log_info "==========================================" + log_info "步骤 3: 解压新版本" + log_info "==========================================" + + log_info "解压更新包到临时目录: $TEMP_DIR" + mkdir -p "$TEMP_DIR" + cd "$TEMP_DIR" + + if ! unzip -q "$ZIP_FILE"; then + log_error "解压失败，请检查 zip 文件是否损坏" + exit 1 + fi + + log_info "✅ 解压完成" + echo "" +} + +# ========== 4. 替换文件 ========== +replace_files() { + log_info "==========================================" + log_info "步骤 4: 替换文件" + log_info "==========================================" + + # 替换 EdgeAdmin bin + if [ -d "$TEMP_DIR/edge-admin/bin" ]; then + log_info "替换 EdgeAdmin 可执行文件..." + cp -r "$TEMP_DIR/edge-admin/bin"/* "$TARGET_DIR/bin/" + log_info "✅ EdgeAdmin bin 已更新" + fi + + # 替换 EdgeAdmin web（排除 tmp） + if [ -d "$TEMP_DIR/edge-admin/web" ]; then + log_info "替换 EdgeAdmin 前端文件..." + if command -v rsync > /dev/null; then + rsync -av --exclude='tmp' \ + "$TEMP_DIR/edge-admin/web/" "$TARGET_DIR/web/" + else + # 如果没有 rsync，使用 cp + cp -r "$TEMP_DIR/edge-admin/web"/* "$TARGET_DIR/web/" 2>/dev/null || true + rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true + fi + # 清空 tmp 目录 + rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true + log_info "✅ EdgeAdmin web 已更新" + fi + + # 替换 EdgeAPI 文件 + if [ -d "$TEMP_DIR/edge-admin/edge-api" ]; then + log_info "替换 EdgeAPI 文件..." + + # 确保 edge-api 目录存在 + mkdir -p "$TARGET_DIR/edge-api" + + # 替换 bin + if [ -d "$TEMP_DIR/edge-admin/edge-api/bin" ]; then + mkdir -p "$TARGET_DIR/edge-api/bin" + cp -r "$TEMP_DIR/edge-admin/edge-api/bin"/* \ + "$TARGET_DIR/edge-api/bin/" 2>/dev/null || true + log_info "✅ EdgeAPI bin 已更新" + fi + + # 替换 deploy（节点安装包） + if [ -d "$TEMP_DIR/edge-admin/edge-api/deploy" ]; then + mkdir -p "$TARGET_DIR/edge-api/deploy" + cp -r "$TEMP_DIR/edge-admin/edge-api/deploy"/* \ + "$TARGET_DIR/edge-api/deploy/" 2>/dev/null || true + log_info "✅ EdgeAPI deploy 已更新" + fi + + # 替换 installers（安装工具） + if [ -d "$TEMP_DIR/edge-admin/edge-api/installers" ]; then + mkdir -p "$TARGET_DIR/edge-api/installers" + cp -r "$TEMP_DIR/edge-admin/edge-api/installers"/* \ + "$TARGET_DIR/edge-api/installers/" 2>/dev/null || true + log_info "✅ EdgeAPI installers 已更新" + fi + fi + + log_info "✅ 文件替换完成" + echo "" +} + +# ========== 5. 恢复配置文件 ========== +restore_configs() { + log_info "==========================================" + log_info "步骤 5: 恢复配置文件" + log_info "==========================================" + + # 恢复 EdgeAdmin 配置 + if [ -d "$BACKUP_DIR/edge-admin-configs" ]; then + log_info "恢复 EdgeAdmin 配置文件..." + cp -r "$BACKUP_DIR/edge-admin-configs"/* "$TARGET_DIR/configs/" + log_info "✅ EdgeAdmin 配置已恢复" + else + log_warn "未找到 EdgeAdmin 配置备份，请手动检查配置文件" + fi + + # 恢复 EdgeAPI 配置 + if [ -d "$BACKUP_DIR/edge-api-configs" ]; then + log_info "恢复 EdgeAPI 配置文件..." + mkdir -p "$TARGET_DIR/edge-api/configs" + cp -r "$BACKUP_DIR/edge-api-configs"/* "$TARGET_DIR/edge-api/configs/" 2>/dev/null || true + log_info "✅ EdgeAPI 配置已恢复" + else + log_warn "未找到 EdgeAPI 配置备份，如果存在 edge-api 目录，请手动检查配置文件" + fi + + log_info "✅ 配置文件恢复完成" + echo "" +} + +# ========== 6. 清理临时文件 ========== +cleanup() { + log_info "==========================================" + log_info "步骤 6: 清理临时文件" + log_info "==========================================" + + rm -rf "$TEMP_DIR" + log_info "✅ 临时文件已清理" + echo "" +} + +# ========== 7. 启动服务 ========== +start_services() { + log_info "==========================================" + log_info "步骤 7: 启动服务" + log_info "==========================================" + + if [ -f "$TARGET_DIR/bin/edge-admin" ]; then + log_info "启动 EdgeAdmin 服务..." + "$TARGET_DIR/bin/edge-admin" start + sleep 3 + + # 检查服务状态 + if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then + log_info "✅ 服务启动成功" + else + log_warn "服务可能未正常启动，请检查日志" + log_info "查看日志: tail -f $TARGET_DIR/logs/run.log" + fi + else + log_error "未找到 edge-admin 可执行文件" + exit 1 + fi + echo "" +} + +# ========== 主函数 ========== +main() { + echo "" + log_info "==========================================" + log_info "CloudWAF 部署脚本 - Gname DNS 提供商更新" + log_info "==========================================" + echo "" + + log_info "部署配置:" + log_info " ZIP 文件: $ZIP_FILE" + log_info " 目标目录: $TARGET_DIR" + log_info " 备份目录: $BACKUP_DIR" + echo "" + + # 确认操作 + read -p "确认开始部署? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "部署已取消" + exit 0 + fi + echo "" + + # 执行部署步骤 + check_prerequisites + backup_files + stop_services + extract_package + replace_files + restore_configs + cleanup + start_services + + # 完成 + log_info "==========================================" + log_info "✅ 部署完成！" + log_info "==========================================" + echo "" + log_info "备份位置: $BACKUP_DIR" + log_info "请检查服务运行状态和日志" + echo "" + log_info "验证步骤:" + log_info "1. 检查服务进程: ps aux | grep \"$TARGET_DIR/bin/edge-admin\"" + log_info "2. 检查服务日志: tail -f $TARGET_DIR/logs/run.log" + log_info "3. 访问管理后台，测试 Gname DNS 提供商功能" + echo "" +} + +# 执行主函数 +main + diff --git a/deploy/clickhouse/README.md b/deploy/clickhouse/README.md new file mode 100644 index 0000000..0767a56 --- /dev/null +++ b/deploy/clickhouse/README.md @@ -0,0 +1,111 @@ +# ClickHouse + Fluent Bit 使用手册（Ubuntu 22.04 / Amazon Linux 2023） + +## 1. 支持范围 + +- Ubuntu 22.04 +- Amazon Linux 2023（AWS） + +安装脚本：`install_clickhouse_linux.sh`（自动识别上述系统）。 + +## 2. 安装 ClickHouse + +```bash +cd /path/to/waf-platform/deploy/clickhouse +chmod +x install_clickhouse_linux.sh +sudo ./install_clickhouse_linux.sh +``` + +可选：安装时初始化 `default` 用户密码： + +```bash +sudo CLICKHOUSE_DEFAULT_PASSWORD='YourStrongPassword' ./install_clickhouse_linux.sh +``` + +## 3. 开启 HTTPS（默认仅 crt+key） + +脚本默认生成 `server.crt + server.key`（带 SAN）并启用 8443： + +```bash +cd /path/to/waf-platform/deploy/clickhouse +chmod +x configure_clickhouse_https.sh +sudo CH_HTTPS_PORT=8443 \ + CH_CERT_CN=clickhouse.example.com \ + CH_CERT_DNS=clickhouse.example.com \ + CH_CERT_IP= \ + ./configure_clickhouse_https.sh +``` + +使用已有证书： + +```bash +sudo SRC_CERT=/path/to/server.crt \ + SRC_KEY=/path/to/server.key \ + CH_HTTPS_PORT=8443 \ + ./configure_clickhouse_https.sh +``` + +## 4. 初始化日志表（含优化） + +```bash +cd /path/to/waf-platform/deploy/clickhouse +chmod +x init_waf_logs_tables.sh +sudo CH_HOST=127.0.0.1 \ + CH_PORT=9000 \ + CH_USER=default \ + CH_PASSWORD='YourStrongPassword' \ + CH_DATABASE=default \ + ./init_waf_logs_tables.sh +``` + +说明： +- `init_waf_logs_tables.sql` 已内置主要优化（`CODEC`、`LowCardinality`、跳数索引）。 +- `optimize_schema.sql` 主要用于历史表补齐优化，不是首次建表必需步骤。 + +## 5. 平台侧配置（EdgeAdmin） + +在 ClickHouse 设置页配置： + +- Host：ClickHouse 地址 +- Port：`8443` +- Database：`default` +- Scheme：`https` + +当前实现说明： +- 前端不再提供 `TLS跳过校验` 和 `TLS Server Name` 配置项。 +- 后端固定 `TLSSkipVerify=true`（默认不校验证书）。 + +保存后点击“测试连接”。 + +## 6. Fluent Bit 配置方式 + +推荐平台托管模式（在线安装/升级 Node、DNS 时自动下发）： + +- `/etc/fluent-bit/fluent-bit.conf` +- `/etc/fluent-bit/.edge-managed.env` +- `/etc/fluent-bit/.edge-managed.json` + +检查状态： + +```bash +sudo systemctl status fluent-bit --no-pager +sudo cat /etc/fluent-bit/.edge-managed.json +``` + +## 7. 验证与排障 + +查看 Fluent Bit 日志： + +```bash +sudo journalctl -u fluent-bit -f +``` + +查看写入： + +```sql +SELECT count() FROM default.logs_ingest; +SELECT count() FROM default.dns_logs_ingest; +``` + +常见错误： +- `connection refused`：8443 未监听或网络未放行。 +- `legacy Common Name`：证书缺 SAN，需重签。 diff --git a/deploy/clickhouse/configure_clickhouse_https.sh b/deploy/clickhouse/configure_clickhouse_https.sh new file mode 100644 index 0000000..783cc37 --- /dev/null +++ b/deploy/clickhouse/configure_clickhouse_https.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "[ERROR] please run as root" + exit 1 +fi + +if [[ ! -f /etc/os-release ]]; then + echo "[ERROR] /etc/os-release not found" + exit 1 +fi + +# shellcheck disable=SC1091 +source /etc/os-release +os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')" +os_ver="${VERSION_ID:-}" +is_ubuntu22=false +is_amzn2023=false + +if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then + is_ubuntu22=true +fi +if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then + is_amzn2023=true +fi + +if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then + echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}" + exit 1 +fi + +if ! command -v openssl >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then + if [[ "${is_ubuntu22}" == "true" ]]; then + apt-get update -y + DEBIAN_FRONTEND=noninteractive apt-get install -y openssl curl ca-certificates + else + dnf makecache -y + dnf install -y openssl curl ca-certificates + fi +fi + +CH_HTTPS_PORT="${CH_HTTPS_PORT:-8443}" +CH_LISTEN_HOST="${CH_LISTEN_HOST:-::}" +CH_CERT_CN="${CH_CERT_CN:-$(hostname -f 2>/dev/null || hostname)}" +CH_CERT_DNS="${CH_CERT_DNS:-}" +CH_CERT_IP="${CH_CERT_IP:-}" +CH_CERT_DAYS="${CH_CERT_DAYS:-825}" +CH_GENERATE_CA="${CH_GENERATE_CA:-false}" + +SRC_CERT="${SRC_CERT:-}" +SRC_KEY="${SRC_KEY:-}" +SRC_CA="${SRC_CA:-}" + +CH_DIR="/etc/clickhouse-server" +CH_CONFIG_D_DIR="${CH_DIR}/config.d" +PKI_DIR="${CH_DIR}/pki" +SERVER_CERT="${CH_DIR}/server.crt" +SERVER_KEY="${CH_DIR}/server.key" +CA_CERT="${CH_DIR}/ca.crt" +OVERRIDE_FILE="${CH_CONFIG_D_DIR}/waf-https.xml" + +mkdir -p "${CH_CONFIG_D_DIR}" "${PKI_DIR}" + +split_csv() { + local raw="$1" + if [[ -z "${raw}" ]]; then + return 0 + fi + IFS=',' read -r -a arr <<<"${raw}" + for item in "${arr[@]}"; do + item="$(echo "${item}" | xargs)" + if [[ -n "${item}" ]]; then + echo "${item}" + fi + done +} + +build_san_line() { + local san_entries=() + while IFS= read -r dns_item; do + san_entries+=("DNS:${dns_item}") + done < <(split_csv "${CH_CERT_DNS}") + while IFS= read -r ip_item; do + san_entries+=("IP:${ip_item}") + done < <(split_csv "${CH_CERT_IP}") + + if [[ ${#san_entries[@]} -eq 0 ]]; then + san_entries+=("DNS:${CH_CERT_CN}") + fi + + local san_line + san_line="$(IFS=,; echo "${san_entries[*]}")" + echo "${san_line}" +} + +generate_self_signed_cert() { + echo "[INFO] generating self-signed server certificate (crt+key only) ..." + local server_key="${PKI_DIR}/server.key" + local server_csr="${PKI_DIR}/server.csr" + local server_crt="${PKI_DIR}/server.crt" + local ext_file="${PKI_DIR}/server.ext" + local san_line + san_line="$(build_san_line)" + + openssl genrsa -out "${server_key}" 2048 + openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}" + + cat >"${ext_file}" <"${ext_file}" <"${OVERRIDE_FILE}" < + ${CH_HTTPS_PORT} + ${CH_LISTEN_HOST} + + + ${SERVER_CERT} + ${SERVER_KEY} + none + true + true + sslv2,sslv3 + true + + RejectCertificateHandler + + + + +EOF + +echo "[INFO] restarting clickhouse-server ..." +systemctl restart clickhouse-server +sleep 2 + +echo "[INFO] service status ..." +systemctl --no-pager -l status clickhouse-server | sed -n '1,15p' + +echo "[INFO] verifying HTTPS endpoint ..." +curl -sk "https://127.0.0.1:${CH_HTTPS_PORT}/?query=SELECT%201" || true +echo + +echo "[OK] ClickHouse HTTPS setup finished" +echo " HTTPS port : ${CH_HTTPS_PORT}" +echo " cert file : ${SERVER_CERT}" +echo " key file : ${SERVER_KEY}" +if [[ -f "${CA_CERT}" ]]; then + echo " CA file : ${CA_CERT}" + echo " import this CA file into API/Fluent Bit hosts if tls.verify=On" +fi diff --git a/deploy/clickhouse/init_waf_logs_tables.sh b/deploy/clickhouse/init_waf_logs_tables.sh new file mode 100644 index 0000000..0f524cc --- /dev/null +++ b/deploy/clickhouse/init_waf_logs_tables.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SQL_FILE="${SCRIPT_DIR}/init_waf_logs_tables.sql" + +if [[ ! -f "${SQL_FILE}" ]]; then + echo "[ERROR] SQL file not found: ${SQL_FILE}" + exit 1 +fi + +if ! command -v clickhouse-client >/dev/null 2>&1; then + echo "[ERROR] clickhouse-client not found. Please install ClickHouse client first." + exit 1 +fi + +CH_HOST="${CH_HOST:-127.0.0.1}" +CH_PORT="${CH_PORT:-9000}" +CH_USER="${CH_USER:-default}" +CH_PASSWORD="${CH_PASSWORD:-}" +CH_DATABASE="${CH_DATABASE:-default}" + +args=(--host "${CH_HOST}" --port "${CH_PORT}" --user "${CH_USER}") +if [[ -n "${CH_PASSWORD}" ]]; then + args+=(--password "${CH_PASSWORD}") +fi + +echo "[INFO] creating database if not exists: ${CH_DATABASE}" +clickhouse-client "${args[@]}" --query "CREATE DATABASE IF NOT EXISTS ${CH_DATABASE}" + +echo "[INFO] initializing tables in database: ${CH_DATABASE}" +clickhouse-client "${args[@]}" --database "${CH_DATABASE}" < "${SQL_FILE}" + +echo "[INFO] checking table status ..." +clickhouse-client "${args[@]}" --database "${CH_DATABASE}" --query \ + "SELECT name, engine FROM system.tables WHERE database='${CH_DATABASE}' AND name IN ('logs_ingest','dns_logs_ingest') ORDER BY name" + +echo "[OK] ClickHouse ingest tables are ready in database '${CH_DATABASE}'" diff --git a/deploy/clickhouse/init_waf_logs_tables.sql b/deploy/clickhouse/init_waf_logs_tables.sql new file mode 100644 index 0000000..239e024 --- /dev/null +++ b/deploy/clickhouse/init_waf_logs_tables.sql @@ -0,0 +1,69 @@ +-- Initialize HTTP and DNS ingest tables for GoEdge access logs. +-- Run with: +-- clickhouse-client --database < init_waf_logs_tables.sql + +CREATE TABLE IF NOT EXISTS logs_ingest +( + timestamp DateTime CODEC(DoubleDelta, ZSTD(1)), + node_id UInt64, + cluster_id UInt64, + server_id UInt64, + host LowCardinality(String), + ip String, + method LowCardinality(String), + path String CODEC(ZSTD(1)), + status UInt16, + bytes_in UInt64 CODEC(Delta, ZSTD(1)), + bytes_out UInt64 CODEC(Delta, ZSTD(1)), + cost_ms UInt32 CODEC(Delta, ZSTD(1)), + ua String CODEC(ZSTD(1)), + referer String CODEC(ZSTD(1)), + log_type LowCardinality(String), + trace_id String, + firewall_policy_id UInt64 DEFAULT 0, + firewall_rule_group_id UInt64 DEFAULT 0, + firewall_rule_set_id UInt64 DEFAULT 0, + firewall_rule_id UInt64 DEFAULT 0, + request_headers String CODEC(ZSTD(3)) DEFAULT '', + request_body String CODEC(ZSTD(3)) DEFAULT '', + response_headers String CODEC(ZSTD(3)) DEFAULT '', + response_body String CODEC(ZSTD(3)) DEFAULT '', + INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4, + INDEX idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4, + INDEX idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4, + INDEX idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4, + INDEX idx_status status TYPE minmax GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (timestamp, node_id, server_id, trace_id) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS dns_logs_ingest +( + timestamp DateTime CODEC(DoubleDelta, ZSTD(1)), + request_id String, + node_id UInt64, + cluster_id UInt64, + domain_id UInt64, + record_id UInt64, + remote_addr String, + question_name String, + question_type LowCardinality(String), + record_name String, + record_type LowCardinality(String), + record_value String, + networking LowCardinality(String), + is_recursive UInt8, + error String CODEC(ZSTD(1)), + ns_route_codes Array(String), + content_json String CODEC(ZSTD(3)) DEFAULT '', + INDEX idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4, + INDEX idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4, + INDEX idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4, + INDEX idx_domain_id domain_id TYPE minmax GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (timestamp, request_id, node_id) +SETTINGS index_granularity = 8192; diff --git a/deploy/clickhouse/install_clickhouse_linux.sh b/deploy/clickhouse/install_clickhouse_linux.sh new file mode 100644 index 0000000..729008b --- /dev/null +++ b/deploy/clickhouse/install_clickhouse_linux.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "[ERROR] please run as root" + exit 1 +fi + +if [[ ! -f /etc/os-release ]]; then + echo "[ERROR] /etc/os-release not found" + exit 1 +fi + +# shellcheck disable=SC1091 +source /etc/os-release +os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')" +os_ver="${VERSION_ID:-}" +is_ubuntu22=false +is_amzn2023=false + +if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then + is_ubuntu22=true +fi +if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then + is_amzn2023=true +fi + +if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then + echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}" + exit 1 +fi + +if [[ "${is_ubuntu22}" == "true" ]]; then + echo "[INFO] detected Ubuntu 22.04" + echo "[INFO] installing prerequisites ..." + apt-get update -y + DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates gnupg apt-transport-https lsb-release + + echo "[INFO] configuring ClickHouse apt repository ..." + install -d -m 0755 /etc/apt/keyrings + if [[ ! -f /etc/apt/keyrings/clickhouse.gpg ]]; then + curl -fsSL https://packages.clickhouse.com/CLICKHOUSE-KEY.GPG | gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg + fi + + cat >/etc/apt/sources.list.d/clickhouse.list <<'EOF' +deb [signed-by=/etc/apt/keyrings/clickhouse.gpg arch=amd64,arm64] https://packages.clickhouse.com/deb stable main +EOF + + echo "[INFO] installing clickhouse-server and clickhouse-client ..." + apt-get update -y + DEBIAN_FRONTEND=noninteractive apt-get install -y clickhouse-server clickhouse-client clickhouse-common-static +fi + +if [[ "${is_amzn2023}" == "true" ]]; then + echo "[INFO] detected Amazon Linux 2023" + echo "[INFO] installing prerequisites ..." + dnf makecache -y + dnf install -y curl ca-certificates gnupg2 dnf-plugins-core + + echo "[INFO] configuring ClickHouse yum repository ..." + cat >/etc/yum.repos.d/clickhouse.repo <<'EOF' +[clickhouse-stable] +name=ClickHouse Stable Repository +baseurl=https://packages.clickhouse.com/rpm/stable/$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.clickhouse.com/rpm/stable/repodata/repomd.xml.key + https://packages.clickhouse.com/rpm/clickhouse-static.key +EOF + + echo "[INFO] installing clickhouse-server and clickhouse-client ..." + dnf clean all + dnf makecache -y + if ! dnf install -y clickhouse-server clickhouse-client clickhouse-common-static; then + dnf install -y clickhouse-server clickhouse-client + fi +fi + +echo "[INFO] enabling clickhouse-server ..." +systemctl enable clickhouse-server >/dev/null 2>&1 || true +systemctl restart clickhouse-server +sleep 2 + +if [[ -n "${CLICKHOUSE_DEFAULT_PASSWORD:-}" ]]; then + echo "[INFO] setting default user password ..." + if [[ "${CLICKHOUSE_DEFAULT_PASSWORD}" == *"'"* ]]; then + echo "[ERROR] CLICKHOUSE_DEFAULT_PASSWORD contains single quote, please set password manually with clickhouse-client" + exit 1 + fi + clickhouse-client --query "ALTER USER default IDENTIFIED WITH plaintext_password BY '${CLICKHOUSE_DEFAULT_PASSWORD}'" +fi + +echo "[INFO] health check ..." +clickhouse-client --query "SELECT version()" +echo "[OK] ClickHouse install completed: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}" diff --git a/deploy/clickhouse/optimize_schema.sql b/deploy/clickhouse/optimize_schema.sql new file mode 100644 index 0000000..2ad8586 --- /dev/null +++ b/deploy/clickhouse/optimize_schema.sql @@ -0,0 +1,123 @@ +-- ============================================================================= +-- ClickHouse logs_ingest 表优化脚本 +-- +-- 说明： +-- - 所有 ALTER 操作均为在线操作，无需停服 +-- - 建议按阶段顺序执行，每阶段执行后观察 system.parts 确认生效 +-- - 压缩编解码器变更仅影响新写入的 part，存量数据需等 merge 或手动 OPTIMIZE +-- +-- 执行方式： +-- clickhouse-client --host 127.0.0.1 --port 9000 --user default --password 'xxx' < optimize_schema.sql +-- ============================================================================= + +-- ============================================= +-- 阶段 1：大字段压缩优化（效果最显著） +-- ============================================= + +-- 大文本字段改用 ZSTD(3)，对 JSON / HTTP 文本压缩率远优于默认 LZ4 +-- 预期效果：磁盘占用减少 40%-60% +ALTER TABLE logs_ingest MODIFY COLUMN request_headers String CODEC(ZSTD(3)); +ALTER TABLE logs_ingest MODIFY COLUMN request_body String CODEC(ZSTD(3)); +ALTER TABLE logs_ingest MODIFY COLUMN response_headers String CODEC(ZSTD(3)); +ALTER TABLE logs_ingest MODIFY COLUMN response_body String CODEC(ZSTD(3)); + +-- 中等长度文本字段用 ZSTD(1)，平衡压缩率与 CPU 开销 +ALTER TABLE logs_ingest MODIFY COLUMN ua String CODEC(ZSTD(1)); +ALTER TABLE logs_ingest MODIFY COLUMN path String CODEC(ZSTD(1)); +ALTER TABLE logs_ingest MODIFY COLUMN referer String CODEC(ZSTD(1)); + +-- 低基数字段改用 LowCardinality（内存+磁盘双降） +-- method 的基数极低（GET/POST/PUT/DELETE 等），host 基数取决于站点数量 +ALTER TABLE logs_ingest MODIFY COLUMN method LowCardinality(String); +ALTER TABLE logs_ingest MODIFY COLUMN log_type LowCardinality(String); +ALTER TABLE logs_ingest MODIFY COLUMN host LowCardinality(String); + +-- 数值字段使用 Delta + ZSTD 编码（利用相邻行的时间/大小相关性） +ALTER TABLE logs_ingest MODIFY COLUMN bytes_in UInt64 CODEC(Delta, ZSTD(1)); +ALTER TABLE logs_ingest MODIFY COLUMN bytes_out UInt64 CODEC(Delta, ZSTD(1)); +ALTER TABLE logs_ingest MODIFY COLUMN cost_ms UInt32 CODEC(Delta, ZSTD(1)); + +-- ============================================= +-- 阶段 2：添加 Skipping Index（加速高频过滤查询） +-- ============================================= + +-- trace_id 精确查找（查看日志详情 FindByTraceId） +-- bloom_filter(0.01) = 1% 误判率，GRANULARITY 4 = 每 4 个 granule 一个 bloom block +ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4; + +-- IP 精确查找 +ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4; + +-- host 模糊查询支持（tokenbf_v1 对 LIKE '%xxx%' 有效） +ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4; + +-- firewall_policy_id 过滤（HasFirewallPolicy: > 0） +ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4; + +-- status 范围过滤（HasError: status >= 400） +ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_status status TYPE minmax GRANULARITY 4; + +-- ============================================= +-- 阶段 3：物化索引到现有数据（对存量数据生效） +-- ============================================= +-- 注意：MATERIALIZE INDEX 会触发后台 mutation，大表可能需要一定时间 +-- 可通过 SELECT * FROM system.mutations WHERE is_done = 0 监控进度 + +ALTER TABLE logs_ingest MATERIALIZE INDEX idx_trace_id; +ALTER TABLE logs_ingest MATERIALIZE INDEX idx_ip; +ALTER TABLE logs_ingest MATERIALIZE INDEX idx_host; +ALTER TABLE logs_ingest MATERIALIZE INDEX idx_fw_policy; +ALTER TABLE logs_ingest MATERIALIZE INDEX idx_status; + + +-- ============================================================================= +-- dns_logs_ingest 表优化（DNS 日志表） +-- ============================================================================= + +-- 大文本字段压缩 +ALTER TABLE dns_logs_ingest MODIFY COLUMN content_json String CODEC(ZSTD(3)); +ALTER TABLE dns_logs_ingest MODIFY COLUMN error String CODEC(ZSTD(1)); + +-- 低基数字段 +ALTER TABLE dns_logs_ingest MODIFY COLUMN question_type LowCardinality(String); +ALTER TABLE dns_logs_ingest MODIFY COLUMN record_type LowCardinality(String); +ALTER TABLE dns_logs_ingest MODIFY COLUMN networking LowCardinality(String); + +-- request_id 精确查找 +ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4; + +-- remote_addr 精确查找 +ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4; + +-- question_name 模糊查询 +ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4; + +-- domain_id 过滤 +ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_domain_id domain_id TYPE minmax GRANULARITY 4; + +-- 物化索引到现有数据 +ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_request_id; +ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_remote_addr; +ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_question_name; +ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_domain_id; + + +-- ============================================================================= +-- 验证命令（执行完上述 ALTER 后运行） +-- ============================================================================= + +-- 查看列的压缩编解码器 +-- SELECT name, type, compression_codec FROM system.columns WHERE table = 'logs_ingest' AND database = currentDatabase(); + +-- 查看表的压缩率 +-- SELECT table, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed, round(sum(data_uncompressed_bytes) / sum(data_compressed_bytes), 2) AS ratio FROM system.columns WHERE table IN ('logs_ingest', 'dns_logs_ingest') GROUP BY table; + +-- 查看各列占用的磁盘空间（找出最大的列） +-- SELECT name, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed FROM system.columns WHERE table = 'logs_ingest' GROUP BY name ORDER BY sum(data_compressed_bytes) DESC; + +-- 查看 mutation 进度 +-- SELECT database, table, mutation_id, command, is_done, parts_to_do FROM system.mutations WHERE is_done = 0; + +-- 强制触发 merge（可选，让压缩编解码器变更对存量数据生效） +-- OPTIMIZE TABLE logs_ingest FINAL; +-- OPTIMIZE TABLE dns_logs_ingest FINAL; diff --git a/deploy/fluent-bit/README.md b/deploy/fluent-bit/README.md index 9c77d50..3706623 100644 --- a/deploy/fluent-bit/README.md +++ b/deploy/fluent-bit/README.md @@ -21,7 +21,14 @@ - **边缘节点（EdgeNode）** 已开启本地日志落盘，目录优先取“公用访问日志策略”的文件 `path`（取目录），为空时回退 `EDGE_LOG_DIR`，再回退默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`（JSON Lines）。 - **DNS 节点（EdgeDNS）** 已开启本地日志落盘，目录优先取“公用访问日志策略”的文件 `path`（取目录），为空时回退 `EDGE_DNS_LOG_DIR`，再回退默认 `/var/log/edge/edge-dns`；生成 `access.log`（JSON Lines）。 - **ClickHouse** 已安装并可访问（单机或集群），且已创建好 `logs_ingest` 表（见下文「五、ClickHouse 建表」）。 -- 若 Fluent Bit 与 ClickHouse 不在同一台机，需保证网络可达（默认 HTTP 端口 8123）。 +- 若 Fluent Bit 与 ClickHouse 不在同一台机，需保证网络可达（默认 HTTPS 端口 8443）。 +- 日志轮转默认由 Node/DNS 内建 `lumberjack` 执行： + - `maxSizeMB=256` + - `maxBackups=14` + - `maxAgeDays=7` + - `compress=false` + - `localTime=true` + 可通过公用日志策略 `file.rotate` 调整。 --- @@ -71,7 +78,7 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/ 编辑 `clickhouse-upstream.conf`，按实际环境填写 ClickHouse 的 Host/Port： -- **单机**：保留一个 `[NODE]`，改 `Host`、`Port`（默认 8123）。 +- **单机**：保留一个 `[NODE]`，改 `Host`、`Port`（默认 8443）。 - **集群**：复制多段 `[NODE]`，每段一个节点，例如： ```ini @@ -81,12 +88,12 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/ [NODE] Name node-01 Host 192.168.1.10 - Port 8123 + Port 8443 [NODE] Name node-02 Host 192.168.1.11 - Port 8123 + Port 8443 ``` ### 3.3 ClickHouse 账号密码（有密码时必做） @@ -319,9 +326,9 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch | 组件 | 说明 | |------|------| -| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`（取目录）；若为空回退 `EDGE_LOG_DIR`，再回退默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`；支持 SIGHUP 重开句柄，可与 logrotate 的 `copytruncate` 配合。 | +| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`（取目录）；若为空回退 `EDGE_LOG_DIR`，再回退默认 `/var/log/edge/edge-node`；生成 `access.log`、`waf.log`、`error.log`；内建 lumberjack 轮转（默认 256MB/14份/7天，可按策略调整），仍支持 SIGHUP 重建 writer。 | | **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`（取目录）；若为空回退 `EDGE_DNS_LOG_DIR`，再回退默认 `/var/log/edge/edge-dns`；生成 `access.log`（JSON Lines），由 Fluent Bit 采集写入 `dns_logs_ingest`。 | -| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转，避免磁盘占满。 | +| **logrotate** | 可选的历史兼容方案（已非必需）；默认建议使用节点内建 lumberjack 轮转。 | | **平台（EdgeAPI）** | 配置 ClickHouse 只读连接（`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`）；当请求带 `Day` 且已配置 ClickHouse 时，访问日志列表查询走 ClickHouse。 | --- @@ -411,3 +418,54 @@ sudo systemctl restart fluent-bit ``` 回滚后恢复原 HTTP 模式，不影响平台 API/管理端配置。 + +--- + +## 十、平台托管模式（推荐） + +从 `v1.4.7` 开始，Node/DNS 在线安装流程会由平台托管 Fluent Bit，默认不再要求逐台手改 `/etc/fluent-bit/fluent-bit.conf`。 + +### 10.1 托管行为 + +- 安装器优先使用发布包内置离线包（不走 `curl | sh`）。 +- 首次安装后写入： + - `/etc/fluent-bit/fluent-bit.conf` + - `/etc/fluent-bit/parsers.conf` + - `/etc/fluent-bit/.edge-managed.env` + - `/etc/fluent-bit/.edge-managed.json` +- 配置发生变化时按 `hash` 幂等更新，仅在内容变化时重启服务。 +- Node 与 DNS 同机安装时会自动合并角色，输出单份配置。 + +### 10.2 托管元数据 + +平台会维护 `/etc/fluent-bit/.edge-managed.json`，核心字段： + +- `roles`: 当前机器启用角色（`node`/`dns`） +- `hash`: 当前托管配置摘要 +- `sourceVersion`: 平台版本号 +- `updatedAt`: 最近更新时间戳 + +### 10.3 支持矩阵（离线包） + +当前固定支持以下平台键： + +- `ubuntu22.04-amd64` +- `ubuntu22.04-arm64` +- `amzn2023-amd64` +- `amzn2023-arm64` + +构建阶段会校验矩阵包是否齐全，缺失会直接失败并打印期望文件路径。 + +### 10.4 手工配置兼容 + +- 若现有 `fluent-bit.conf` 不是平台托管文件（不含 `managed-by-edgeapi` 标记），安装器不会强制覆盖，会返回明确错误提示。 +- 需要切到托管模式时，先备份旧配置，再由平台触发一次安装/更新任务。 + +### 10.5 Resource Profile Notes (New) + +- Managed default is now tuned for `2C4G` nodes (conservative and stable). +- Additional sample profiles are provided for larger nodes: + - `deploy/fluent-bit/fluent-bit-sample-4c8g.conf` + - `deploy/fluent-bit/fluent-bit-sample-8c16g.conf` +- These sample files are for benchmark/reference only and are not auto-applied by installer. +- To use higher profiles in managed mode, sync those parameters into `EdgeAPI/internal/installers/fluent_bit.go` and then trigger node reinstall/upgrade. diff --git a/deploy/fluent-bit/clickhouse-upstream.conf b/deploy/fluent-bit/clickhouse-upstream.conf index 101700c..b31f443 100644 --- a/deploy/fluent-bit/clickhouse-upstream.conf +++ b/deploy/fluent-bit/clickhouse-upstream.conf @@ -8,4 +8,4 @@ [NODE] Name node-01 Host 127.0.0.1 - Port 8123 + Port 8443 diff --git a/deploy/fluent-bit/fluent-bit-dns.conf b/deploy/fluent-bit/fluent-bit-dns.conf index 3a101b3..2e20aa5 100644 --- a/deploy/fluent-bit/fluent-bit-dns.conf +++ b/deploy/fluent-bit/fluent-bit-dns.conf @@ -1,4 +1,4 @@ -# DNS 节点专用：使用 HTTP 输出写入 ClickHouse（无需 out_clickhouse 插件） +# DNS 节点专用：使用 HTTPS 输出写入 ClickHouse（无需 out_clickhouse 插件） # 启动前设置：CH_USER、CH_PASSWORD；若 ClickHouse 不在本机，请修改 Host、Port # Read_from_Head=true：首次启动会发送已有日志；若只采新日志建议改为 false @@ -26,11 +26,15 @@ Name http Match app.dns.logs Host 127.0.0.1 - Port 8123 + Port 8443 URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow Format json_lines http_user ${CH_USER} http_passwd ${CH_PASSWORD} + tls On + tls.verify On + # tls.ca_file /etc/ssl/certs/ca-certificates.crt + # tls.vhost clickhouse.example.com json_date_key timestamp json_date_format epoch Retry_Limit 10 diff --git a/deploy/fluent-bit/fluent-bit-sample-4c8g.conf b/deploy/fluent-bit/fluent-bit-sample-4c8g.conf new file mode 100644 index 0000000..56b2f6a --- /dev/null +++ b/deploy/fluent-bit/fluent-bit-sample-4c8g.conf @@ -0,0 +1,69 @@ +# Sample profile for 4C8G nodes (Node + DNS on same host). +# Replace Host/Port/URI and credentials according to your ClickHouse deployment. + +[SERVICE] + Flush 1 + Log_Level info + Parsers_File parsers.conf + storage.path /var/lib/fluent-bit/storage + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 512MB + +[INPUT] + Name tail + Path /var/log/edge/edge-node/*.log + Tag app.http.logs + Parser json + Refresh_Interval 2 + Read_from_Head false + DB /var/lib/fluent-bit/http-logs.db + storage.type filesystem + Mem_Buf_Limit 256MB + Skip_Long_Lines On + +[INPUT] + Name tail + Path /var/log/edge/edge-dns/*.log + Tag app.dns.logs + Parser json + Refresh_Interval 2 + Read_from_Head false + DB /var/lib/fluent-bit/dns-logs.db + storage.type filesystem + Mem_Buf_Limit 256MB + Skip_Long_Lines On + +[OUTPUT] + Name http + Match app.http.logs + Host 127.0.0.1 + Port 8443 + URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow + Format json_lines + http_user ${CH_USER} + http_passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + workers 2 + net.keepalive On + Retry_Limit False + tls On + tls.verify On + +[OUTPUT] + Name http + Match app.dns.logs + Host 127.0.0.1 + Port 8443 + URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow + Format json_lines + http_user ${CH_USER} + http_passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + workers 2 + net.keepalive On + Retry_Limit False + tls On + tls.verify On diff --git a/deploy/fluent-bit/fluent-bit-sample-8c16g.conf b/deploy/fluent-bit/fluent-bit-sample-8c16g.conf new file mode 100644 index 0000000..36da86f --- /dev/null +++ b/deploy/fluent-bit/fluent-bit-sample-8c16g.conf @@ -0,0 +1,69 @@ +# Sample profile for 8C16G nodes (Node + DNS on same host). +# Replace Host/Port/URI and credentials according to your ClickHouse deployment. + +[SERVICE] + Flush 1 + Log_Level info + Parsers_File parsers.conf + storage.path /var/lib/fluent-bit/storage + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 1024MB + +[INPUT] + Name tail + Path /var/log/edge/edge-node/*.log + Tag app.http.logs + Parser json + Refresh_Interval 1 + Read_from_Head false + DB /var/lib/fluent-bit/http-logs.db + storage.type filesystem + Mem_Buf_Limit 512MB + Skip_Long_Lines On + +[INPUT] + Name tail + Path /var/log/edge/edge-dns/*.log + Tag app.dns.logs + Parser json + Refresh_Interval 1 + Read_from_Head false + DB /var/lib/fluent-bit/dns-logs.db + storage.type filesystem + Mem_Buf_Limit 512MB + Skip_Long_Lines On + +[OUTPUT] + Name http + Match app.http.logs + Host 127.0.0.1 + Port 8443 + URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow + Format json_lines + http_user ${CH_USER} + http_passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + workers 4 + net.keepalive On + Retry_Limit False + tls On + tls.verify On + +[OUTPUT] + Name http + Match app.dns.logs + Host 127.0.0.1 + Port 8443 + URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow + Format json_lines + http_user ${CH_USER} + http_passwd ${CH_PASSWORD} + json_date_key timestamp + json_date_format epoch + workers 4 + net.keepalive On + Retry_Limit False + tls On + tls.verify On diff --git a/deploy/fluent-bit/packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm b/deploy/fluent-bit/packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm new file mode 100644 index 0000000..6a2f7a4 Binary files /dev/null and b/deploy/fluent-bit/packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm differ diff --git a/deploy/fluent-bit/packages/linux-amd64/fluent-bit_4.2.2_amd64.deb b/deploy/fluent-bit/packages/linux-amd64/fluent-bit_4.2.2_amd64.deb new file mode 100644 index 0000000..9ef01a5 Binary files /dev/null and b/deploy/fluent-bit/packages/linux-amd64/fluent-bit_4.2.2_amd64.deb differ diff --git a/访问日志策略配置手册.md b/访问日志策略配置手册.md index 7b30086..cecc4aa 100644 --- a/访问日志策略配置手册.md +++ b/访问日志策略配置手册.md @@ -47,6 +47,32 @@ flowchart TD - `/var/log/edge/edge-node/*.log` 3. ClickHouse 已建表：`logs_ingest`（见 `deploy/fluent-bit/README.md`）。 +### 3.3 本地日志轮转（默认开启） +从当前版本开始，EdgeNode / EdgeDNS 使用内建 `lumberjack` 轮转，不再依赖系统 `logrotate`。 + +默认值： +- `maxSizeMB=256` +- `maxBackups=14` +- `maxAgeDays=7` +- `compress=false` +- `localTime=true` + +可在策略 `file.rotate` 中配置，例如： + +```json +{ + "path": "/var/log/web-access-${date}.log", + "autoCreate": true, + "rotate": { + "maxSizeMB": 256, + "maxBackups": 14, + "maxAgeDays": 7, + "compress": false, + "localTime": true + } +} +``` + --- ## 4. 三种目标模式怎么配