引入lumberjack和fluentbit自动分发
This commit is contained in:
@@ -18,11 +18,9 @@ const (
|
||||
envPassword = "CLICKHOUSE_PASSWORD"
|
||||
envDatabase = "CLICKHOUSE_DATABASE"
|
||||
envScheme = "CLICKHOUSE_SCHEME"
|
||||
envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY"
|
||||
envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME"
|
||||
defaultPort = 8123
|
||||
defaultPort = 8443
|
||||
defaultDB = "default"
|
||||
defaultScheme = "http"
|
||||
defaultScheme = "https"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -62,7 +60,7 @@ func ResetSharedConfig() {
|
||||
}
|
||||
|
||||
func loadConfig() *Config {
|
||||
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme}
|
||||
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme, TLSSkipVerify: true}
|
||||
// 1) 优先从后台页面配置(DB)读取
|
||||
if models.SharedSysSettingDAO != nil {
|
||||
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
|
||||
@@ -72,8 +70,8 @@ func loadConfig() *Config {
|
||||
cfg.Password = dbCfg.Password
|
||||
cfg.Database = dbCfg.Database
|
||||
cfg.Scheme = normalizeScheme(dbCfg.Scheme)
|
||||
cfg.TLSSkipVerify = dbCfg.TLSSkipVerify
|
||||
cfg.TLSServerName = dbCfg.TLSServerName
|
||||
cfg.TLSSkipVerify = true
|
||||
cfg.TLSServerName = ""
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
@@ -93,8 +91,8 @@ func loadConfig() *Config {
|
||||
cfg.Password = ch.Password
|
||||
cfg.Database = ch.Database
|
||||
cfg.Scheme = normalizeScheme(ch.Scheme)
|
||||
cfg.TLSSkipVerify = ch.TLSSkipVerify
|
||||
cfg.TLSServerName = ch.TLSServerName
|
||||
cfg.TLSSkipVerify = true
|
||||
cfg.TLSServerName = ""
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
@@ -112,17 +110,13 @@ func loadConfig() *Config {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
|
||||
cfg.TLSServerName = os.Getenv(envTLSServerName)
|
||||
cfg.TLSServerName = ""
|
||||
if p := os.Getenv(envPort); p != "" {
|
||||
if v, err := strconv.Atoi(p); err == nil {
|
||||
cfg.Port = v
|
||||
}
|
||||
}
|
||||
if v := os.Getenv(envTLSSkipVerify); v != "" {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
cfg.TLSSkipVerify = b
|
||||
}
|
||||
}
|
||||
cfg.TLSSkipVerify = true
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -176,7 +176,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
||||
}
|
||||
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
|
||||
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||
// 列表查询不 SELECT 大字段(request_headers / request_body / response_headers / response_body),
|
||||
// 避免每次翻页读取 GB 级数据。详情查看时通过 FindByTraceId 单独获取。
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||
table, where, orderBy, limit+1)
|
||||
|
||||
var rawRows []map[string]interface{}
|
||||
|
||||
@@ -122,8 +122,10 @@ func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*N
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
// 列表查询不 SELECT content_json 大字段,减少翻页时的数据传输量。
|
||||
// 详情查看时通过 FindByRequestId 单独获取完整信息。
|
||||
query := fmt.Sprintf(
|
||||
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
|
||||
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
|
||||
table,
|
||||
strings.Join(conditions, " AND "),
|
||||
orderDir,
|
||||
|
||||
@@ -20,3 +20,20 @@ func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string {
|
||||
|
||||
return strings.TrimSpace(config.Path)
|
||||
}
|
||||
|
||||
// ParseHTTPAccessLogPolicyRotateConfig 提取访问日志策略中的文件轮转配置(所有 file* 类型有效)。
|
||||
func ParseHTTPAccessLogPolicyRotateConfig(policy *HTTPAccessLogPolicy) *serverconfigs.AccessLogRotateConfig {
|
||||
if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
config := &serverconfigs.AccessLogFileStorageConfig{}
|
||||
if err := json.Unmarshal(policy.Options, config); err != nil {
|
||||
return nil
|
||||
}
|
||||
if config.Rotate == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return config.Rotate.Normalize()
|
||||
}
|
||||
|
||||
@@ -1176,6 +1176,7 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
|
||||
if publicPolicy != nil {
|
||||
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||
config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
||||
config.GlobalServerConfig.HTTPAccessLog.Rotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,6 +479,7 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.
|
||||
if publicPolicy != nil {
|
||||
config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||
config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
||||
config.AccessLogRotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,8 +270,9 @@ func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.Clic
|
||||
}
|
||||
if len(valueJSON) == 0 {
|
||||
return &systemconfigs.ClickHouseSetting{
|
||||
Port: 8123,
|
||||
Port: 8443,
|
||||
Database: "default",
|
||||
Scheme: "https",
|
||||
}, nil
|
||||
}
|
||||
var config = &systemconfigs.ClickHouseSetting{}
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
package installers
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
slashpath "path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
)
|
||||
|
||||
@@ -17,19 +26,45 @@ const (
|
||||
fluentBitStorageDir = "/var/lib/fluent-bit/storage"
|
||||
fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf"
|
||||
fluentBitParsersFile = "/etc/fluent-bit/parsers.conf"
|
||||
fluentBitUpstreamFile = "/etc/fluent-bit/clickhouse-upstream.conf"
|
||||
fluentBitManagedMetaFile = "/etc/fluent-bit/.edge-managed.json"
|
||||
fluentBitManagedEnvFile = "/etc/fluent-bit/.edge-managed.env"
|
||||
fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge"
|
||||
fluentBitDropInDir = "/etc/systemd/system/fluent-bit.service.d"
|
||||
fluentBitDropInFile = "/etc/systemd/system/fluent-bit.service.d/edge-managed.conf"
|
||||
fluentBitServiceName = "fluent-bit"
|
||||
fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit"
|
||||
fluentBitLocalPackagesRoot = "packages"
|
||||
fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log"
|
||||
fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log"
|
||||
fluentBitManagedMarker = "managed-by-edgeapi"
|
||||
fluentBitRoleNode = "node"
|
||||
fluentBitRoleDNS = "dns"
|
||||
)
|
||||
|
||||
var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found")
|
||||
|
||||
// SetupFluentBit 安装 Fluent Bit(仅离线包),并同步配置文件。
|
||||
// 升级场景下不覆盖已有配置;若已有配置与节点角色不兼容,直接报错终止安装。
|
||||
var fluentBitPackageFileMapping = map[string]string{
|
||||
"ubuntu22.04-amd64": "fluent-bit_4.2.2_amd64.deb",
|
||||
"ubuntu22.04-arm64": "fluent-bit_4.2.2_arm64.deb",
|
||||
"amzn2023-amd64": "fluent-bit-4.2.2-1.x86_64.rpm",
|
||||
"amzn2023-arm64": "fluent-bit-4.2.2-1.aarch64.rpm",
|
||||
}
|
||||
|
||||
type fluentBitManagedMeta struct {
|
||||
Roles []string `json:"roles"`
|
||||
Hash string `json:"hash"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
SourceVersion string `json:"sourceVersion"`
|
||||
}
|
||||
|
||||
type fluentBitDesiredConfig struct {
|
||||
Roles []string
|
||||
ClickHouse *systemconfigs.ClickHouseSetting
|
||||
HTTPPathPattern string
|
||||
DNSPathPattern string
|
||||
}
|
||||
|
||||
// SetupFluentBit 安装并托管 Fluent Bit 配置(离线包 + 平台渲染配置)。
|
||||
func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
||||
if this.client == nil {
|
||||
return errors.New("ssh client is nil")
|
||||
@@ -41,55 +76,41 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
||||
}
|
||||
|
||||
tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit"
|
||||
_, _, _ = this.client.Exec("mkdir -p " + tempDir)
|
||||
_, _, _ = this.client.Exec("mkdir -p " + shQuote(tempDir))
|
||||
defer func() {
|
||||
_, _, _ = this.client.Exec("rm -rf " + tempDir)
|
||||
_, _, _ = this.client.Exec("rm -rf " + shQuote(tempDir))
|
||||
}()
|
||||
|
||||
// 统一使用 fluent-bit.conf(已含 HTTP + DNS 两类 input),避免同机 Node/DNS 冲突。
|
||||
files := []struct {
|
||||
Local string
|
||||
Remote string
|
||||
}{
|
||||
{Local: "fluent-bit.conf", Remote: "fluent-bit.conf"},
|
||||
{Local: "parsers.conf", Remote: "parsers.conf"},
|
||||
{Local: "clickhouse-upstream.conf", Remote: "clickhouse-upstream.conf"},
|
||||
{Local: "logrotate.conf", Remote: "logrotate.conf"},
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
localPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", file.Local)
|
||||
if _, err := os.Stat(localPath); err != nil {
|
||||
return fmt.Errorf("fluent-bit file '%s' not found: %w", localPath, err)
|
||||
}
|
||||
remotePath := tempDir + "/" + file.Remote
|
||||
if err := this.client.Copy(localPath, remotePath, 0644); err != nil {
|
||||
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", file.Local, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := this.ensureFluentBitInstalled(tempDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, stderr, err := this.client.Exec("mkdir -p " + fluentBitConfigDir + " " + fluentBitStorageDir + " /etc/logrotate.d")
|
||||
_, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitConfigDir) + " " + shQuote(fluentBitStorageDir) + " /etc/logrotate.d")
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
|
||||
exists, err := this.remoteFileExists(fluentBitMainConfigFile)
|
||||
parserContent, err := this.readLocalParsersContent()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 若已存在配置,先做角色兼容校验,不允许覆盖。
|
||||
if exists {
|
||||
if err := this.validateExistingConfigForRole(role); err != nil {
|
||||
existingMeta, err := this.readManagedMeta()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mergedRoles, err := mergeManagedRoles(existingMeta, role)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configCopied, err := this.copyFluentBitConfigIfMissing(tempDir)
|
||||
desiredConfig, err := this.buildDesiredFluentBitConfig(mergedRoles)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configChanged, err := this.applyManagedConfig(tempDir, desiredConfig, parserContent, existingMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -99,7 +120,7 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := this.ensureFluentBitService(binPath, configCopied); err != nil {
|
||||
if err := this.ensureFluentBitService(tempDir, binPath, configChanged); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -112,102 +133,128 @@ func (this *BaseInstaller) ensureFluentBitInstalled(tempDir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := this.installFluentBitFromLocalPackage(tempDir); err != nil {
|
||||
platformKey, packageName, arch, err := this.detectRemotePlatformAndPackage()
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect fluent-bit platform failed: %w", err)
|
||||
}
|
||||
|
||||
if err := this.installFluentBitFromLocalPackage(tempDir, arch, packageName); err != nil {
|
||||
if errors.Is(err, errFluentBitLocalPackageNotFound) {
|
||||
return fmt.Errorf("install fluent-bit failed: local package not found, expected in deploy/fluent-bit/%s/linux-<arch>", fluentBitLocalPackagesRoot)
|
||||
expectedPath := filepath.Join("deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch, packageName)
|
||||
return fmt.Errorf("install fluent-bit failed: local package missing for platform '%s', expected '%s'", platformKey, expectedPath)
|
||||
}
|
||||
return fmt.Errorf("install fluent-bit from local package failed: %w", err)
|
||||
}
|
||||
|
||||
binPath, err := this.lookupFluentBitBinPath()
|
||||
binPath, err = this.lookupFluentBitBinPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if binPath == "" {
|
||||
return errors.New("fluent-bit binary not found after local package install")
|
||||
}
|
||||
|
||||
_, stderr, err := this.client.Exec(binPath + " --version")
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify fluent-bit version failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string) error {
|
||||
arch, err := this.detectRemoteLinuxArch()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string, arch string, packageName string) error {
|
||||
packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch)
|
||||
entries, err := os.ReadDir(packageDir)
|
||||
if err != nil {
|
||||
localPackagePath := filepath.Join(packageDir, packageName)
|
||||
if _, err := os.Stat(localPackagePath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return errFluentBitLocalPackageNotFound
|
||||
}
|
||||
return fmt.Errorf("read fluent-bit local package dir failed: %w", err)
|
||||
return fmt.Errorf("check local package failed: %w", err)
|
||||
}
|
||||
|
||||
packageFiles := make([]string, 0)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := strings.ToLower(entry.Name())
|
||||
if strings.HasSuffix(name, ".deb") || strings.HasSuffix(name, ".rpm") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") {
|
||||
packageFiles = append(packageFiles, filepath.Join(packageDir, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(packageFiles) == 0 {
|
||||
return errFluentBitLocalPackageNotFound
|
||||
}
|
||||
|
||||
sort.Strings(packageFiles)
|
||||
|
||||
var lastErr error
|
||||
for _, localPackagePath := range packageFiles {
|
||||
remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath)
|
||||
if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil {
|
||||
lastErr = fmt.Errorf("upload local package failed: %w", err)
|
||||
continue
|
||||
return fmt.Errorf("upload local package failed: %w", err)
|
||||
}
|
||||
|
||||
var installCmd string
|
||||
lowerName := strings.ToLower(localPackagePath)
|
||||
switch {
|
||||
case strings.HasSuffix(lowerName, ".deb"):
|
||||
installCmd = "dpkg -i " + remotePackagePath
|
||||
installCmd = "dpkg -i " + shQuote(remotePackagePath)
|
||||
case strings.HasSuffix(lowerName, ".rpm"):
|
||||
installCmd = "rpm -Uvh --force " + remotePackagePath + " || rpm -ivh --force " + remotePackagePath
|
||||
installCmd = "rpm -Uvh --force " + shQuote(remotePackagePath) + " || rpm -ivh --force " + shQuote(remotePackagePath)
|
||||
case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"):
|
||||
extractDir := tempDir + "/extract"
|
||||
installCmd = "rm -rf " + extractDir + "; mkdir -p " + extractDir + "; tar -xzf " + remotePackagePath + " -C " + extractDir + "; " +
|
||||
"bin=$(find " + extractDir + " -type f -name fluent-bit | head -n 1); " +
|
||||
installCmd = "rm -rf " + shQuote(extractDir) + "; mkdir -p " + shQuote(extractDir) + "; tar -xzf " + shQuote(remotePackagePath) + " -C " + shQuote(extractDir) + "; " +
|
||||
"bin=$(find " + shQuote(extractDir) + " -type f -name fluent-bit | head -n 1); " +
|
||||
"if [ -z \"$bin\" ]; then exit 3; fi; " +
|
||||
"mkdir -p /opt/fluent-bit/bin /usr/local/bin; " +
|
||||
"install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " +
|
||||
"ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit"
|
||||
default:
|
||||
continue
|
||||
return fmt.Errorf("unsupported local package format: %s", packageName)
|
||||
}
|
||||
|
||||
_, stderr, err := this.client.Exec(installCmd)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
|
||||
return fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) detectRemotePlatformAndPackage() (platformKey string, packageName string, arch string, err error) {
|
||||
arch, err = this.detectRemoteLinuxArch()
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
|
||||
releaseData, stderr, err := this.client.Exec("cat /etc/os-release")
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("read /etc/os-release failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
if strings.TrimSpace(releaseData) == "" {
|
||||
return "", "", "", errors.New("/etc/os-release is empty")
|
||||
}
|
||||
|
||||
releaseMap := parseOSRelease(releaseData)
|
||||
id := strings.ToLower(strings.TrimSpace(releaseMap["ID"]))
|
||||
versionID := strings.TrimSpace(releaseMap["VERSION_ID"])
|
||||
|
||||
var distro string
|
||||
switch {
|
||||
case id == "ubuntu" && strings.HasPrefix(versionID, "22.04"):
|
||||
distro = "ubuntu22.04"
|
||||
case id == "amzn" && strings.HasPrefix(versionID, "2023"):
|
||||
distro = "amzn2023"
|
||||
default:
|
||||
return "", "", "", fmt.Errorf("unsupported linux platform id='%s' version='%s'", id, versionID)
|
||||
}
|
||||
|
||||
platformKey = distro + "-" + arch
|
||||
packageName, ok := fluentBitPackageFileMapping[platformKey]
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("no local package mapping for platform '%s'", platformKey)
|
||||
}
|
||||
return platformKey, packageName, arch, nil
|
||||
}
|
||||
|
||||
func parseOSRelease(content string) map[string]string {
|
||||
result := map[string]string{}
|
||||
lines := strings.Split(content, "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
|
||||
continue
|
||||
}
|
||||
|
||||
binPath, err := this.lookupFluentBitBinPath()
|
||||
if err == nil && binPath != "" {
|
||||
return nil
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
key := strings.TrimSpace(parts[0])
|
||||
value := strings.TrimSpace(parts[1])
|
||||
value = strings.Trim(value, "\"")
|
||||
result[key] = value
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
lastErr = errors.New("fluent-bit binary not found after local package install")
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return lastErr
|
||||
}
|
||||
return errFluentBitLocalPackageNotFound
|
||||
return result
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) {
|
||||
@@ -235,74 +282,456 @@ func (this *BaseInstaller) lookupFluentBitBinPath() (string, error) {
|
||||
return strings.TrimSpace(stdout), nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) copyFluentBitConfigIfMissing(tempDir string) (bool, error) {
|
||||
targets := []struct {
|
||||
Src string
|
||||
Dest string
|
||||
}{
|
||||
{Src: tempDir + "/fluent-bit.conf", Dest: fluentBitMainConfigFile},
|
||||
{Src: tempDir + "/parsers.conf", Dest: fluentBitParsersFile},
|
||||
{Src: tempDir + "/clickhouse-upstream.conf", Dest: fluentBitUpstreamFile},
|
||||
{Src: tempDir + "/logrotate.conf", Dest: fluentBitLogrotateFile},
|
||||
func (this *BaseInstaller) readLocalParsersContent() (string, error) {
|
||||
parsersPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", "parsers.conf")
|
||||
data, err := os.ReadFile(parsersPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read local parsers config failed: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) readManagedMeta() (*fluentBitManagedMeta, error) {
|
||||
exists, err := this.remoteFileExists(fluentBitManagedMetaFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
copied := false
|
||||
for _, target := range targets {
|
||||
exists, err := this.remoteFileExists(target.Dest)
|
||||
content, stderr, err := this.client.Exec("cat " + shQuote(fluentBitManagedMetaFile))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read fluent-bit managed metadata failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
meta := &fluentBitManagedMeta{}
|
||||
if err := json.Unmarshal([]byte(content), meta); err != nil {
|
||||
return nil, fmt.Errorf("decode fluent-bit managed metadata failed: %w", err)
|
||||
}
|
||||
meta.Roles = normalizeRoles(meta.Roles)
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func mergeManagedRoles(meta *fluentBitManagedMeta, role nodeconfigs.NodeRole) ([]string, error) {
|
||||
roleName, err := mapNodeRole(role)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
roleSet := map[string]struct{}{}
|
||||
if meta != nil {
|
||||
for _, r := range normalizeRoles(meta.Roles) {
|
||||
roleSet[r] = struct{}{}
|
||||
}
|
||||
}
|
||||
roleSet[roleName] = struct{}{}
|
||||
|
||||
roles := make([]string, 0, len(roleSet))
|
||||
for r := range roleSet {
|
||||
roles = append(roles, r)
|
||||
}
|
||||
sort.Strings(roles)
|
||||
return roles, nil
|
||||
}
|
||||
|
||||
func mapNodeRole(role nodeconfigs.NodeRole) (string, error) {
|
||||
switch role {
|
||||
case nodeconfigs.NodeRoleNode:
|
||||
return fluentBitRoleNode, nil
|
||||
case nodeconfigs.NodeRoleDNS:
|
||||
return fluentBitRoleDNS, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported fluent-bit role '%s'", role)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRoles(rawRoles []string) []string {
|
||||
roleSet := map[string]struct{}{}
|
||||
for _, role := range rawRoles {
|
||||
role = strings.ToLower(strings.TrimSpace(role))
|
||||
if role != fluentBitRoleNode && role != fluentBitRoleDNS {
|
||||
continue
|
||||
}
|
||||
roleSet[role] = struct{}{}
|
||||
}
|
||||
|
||||
roles := make([]string, 0, len(roleSet))
|
||||
for role := range roleSet {
|
||||
roles = append(roles, role)
|
||||
}
|
||||
sort.Strings(roles)
|
||||
return roles
|
||||
}
|
||||
|
||||
func hasRole(roles []string, role string) bool {
|
||||
for _, one := range roles {
|
||||
if one == role {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) buildDesiredFluentBitConfig(roles []string) (*fluentBitDesiredConfig, error) {
|
||||
if len(roles) == 0 {
|
||||
return nil, errors.New("fluent-bit roles should not be empty")
|
||||
}
|
||||
|
||||
ch, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read clickhouse setting failed: %w", err)
|
||||
}
|
||||
if ch == nil {
|
||||
ch = &systemconfigs.ClickHouseSetting{}
|
||||
}
|
||||
if strings.TrimSpace(ch.Host) == "" {
|
||||
ch.Host = "127.0.0.1"
|
||||
}
|
||||
|
||||
ch.Scheme = strings.ToLower(strings.TrimSpace(ch.Scheme))
|
||||
if ch.Scheme == "" {
|
||||
ch.Scheme = "https"
|
||||
}
|
||||
if ch.Scheme != "http" && ch.Scheme != "https" {
|
||||
return nil, fmt.Errorf("unsupported clickhouse scheme '%s'", ch.Scheme)
|
||||
}
|
||||
|
||||
if ch.Port <= 0 {
|
||||
if ch.Scheme == "https" {
|
||||
ch.Port = 8443
|
||||
} else {
|
||||
ch.Port = 8443
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(ch.Database) == "" {
|
||||
ch.Database = "default"
|
||||
}
|
||||
if strings.TrimSpace(ch.User) == "" {
|
||||
ch.User = "default"
|
||||
}
|
||||
// 当前平台策略:后台固定跳过 ClickHouse TLS 证书校验,不暴露 ServerName 配置。
|
||||
ch.TLSSkipVerify = true
|
||||
ch.TLSServerName = ""
|
||||
|
||||
httpPathPattern := fluentBitHTTPPathPattern
|
||||
dnsPathPattern := fluentBitDNSPathPattern
|
||||
publicPolicyPath, err := this.readPublicAccessLogPolicyPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policyDir := dirFromPolicyPath(publicPolicyPath)
|
||||
if policyDir != "" {
|
||||
pattern := strings.TrimRight(policyDir, "/") + "/*.log"
|
||||
httpPathPattern = pattern
|
||||
dnsPathPattern = pattern
|
||||
}
|
||||
|
||||
return &fluentBitDesiredConfig{
|
||||
Roles: normalizeRoles(roles),
|
||||
ClickHouse: ch,
|
||||
HTTPPathPattern: httpPathPattern,
|
||||
DNSPathPattern: dnsPathPattern,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) readPublicAccessLogPolicyPath() (string, error) {
|
||||
policyId, err := models.SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find current public access log policy failed: %w", err)
|
||||
}
|
||||
if policyId <= 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
policy, err := models.SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(nil, policyId)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read public access log policy failed: %w", err)
|
||||
}
|
||||
if policy == nil {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return strings.TrimSpace(models.ParseHTTPAccessLogPolicyFilePath(policy)), nil
|
||||
}
|
||||
|
||||
func dirFromPolicyPath(policyPath string) string {
|
||||
pathValue := strings.TrimSpace(policyPath)
|
||||
if pathValue == "" {
|
||||
return ""
|
||||
}
|
||||
pathValue = strings.ReplaceAll(pathValue, "\\", "/")
|
||||
dir := slashpath.Dir(pathValue)
|
||||
if dir == "." {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimRight(dir, "/")
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) applyManagedConfig(tempDir string, desired *fluentBitDesiredConfig, parserContent string, existingMeta *fluentBitManagedMeta) (bool, error) {
|
||||
mainExists, err := this.remoteFileExists(fluentBitMainConfigFile)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
_, stderr, err := this.client.Exec("cp -f " + target.Src + " " + target.Dest)
|
||||
|
||||
if mainExists && existingMeta == nil {
|
||||
containsMarker, err := this.remoteFileContains(fluentBitMainConfigFile, fluentBitManagedMarker)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("copy fluent-bit file to '%s' failed: %w, stderr: %s", target.Dest, err, stderr)
|
||||
return false, err
|
||||
}
|
||||
if !containsMarker {
|
||||
// Adopt unmanaged config by backing it up and replacing with managed config below.
|
||||
}
|
||||
copied = true
|
||||
}
|
||||
|
||||
return copied, nil
|
||||
configContent, err := renderManagedConfig(desired)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
envContent := renderManagedEnv(desired.ClickHouse)
|
||||
metaContent, newMeta, err := renderManagedMeta(desired, configContent, parserContent, envContent)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
requiredFiles := []string{fluentBitMainConfigFile, fluentBitParsersFile, fluentBitManagedEnvFile, fluentBitManagedMetaFile}
|
||||
if existingMeta != nil && existingMeta.Hash == newMeta.Hash {
|
||||
allExists := true
|
||||
for _, file := range requiredFiles {
|
||||
exists, err := this.remoteFileExists(file)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !exists {
|
||||
allExists = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allExists {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
if mainExists {
|
||||
backup := fluentBitMainConfigFile + ".bak." + strconv.FormatInt(time.Now().Unix(), 10)
|
||||
_, stderr, err := this.client.Exec("cp -f " + shQuote(fluentBitMainConfigFile) + " " + shQuote(backup))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("backup existing fluent-bit config failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
if err := this.writeRemoteFileByTemp(tempDir, fluentBitMainConfigFile, configContent, 0644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := this.writeRemoteFileByTemp(tempDir, fluentBitParsersFile, parserContent, 0644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedEnvFile, envContent, 0600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedMetaFile, metaContent, 0644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
localLogrotate := filepath.Join(Tea.Root, "deploy", "fluent-bit", "logrotate.conf")
|
||||
if _, err := os.Stat(localLogrotate); err == nil {
|
||||
if err := this.copyLocalFileToRemote(tempDir, localLogrotate, fluentBitLogrotateFile, 0644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) validateExistingConfigForRole(role nodeconfigs.NodeRole) error {
|
||||
requiredPatterns := []string{}
|
||||
switch role {
|
||||
case nodeconfigs.NodeRoleNode:
|
||||
requiredPatterns = append(requiredPatterns, fluentBitHTTPPathPattern)
|
||||
case nodeconfigs.NodeRoleDNS:
|
||||
requiredPatterns = append(requiredPatterns, fluentBitDNSPathPattern)
|
||||
func renderManagedConfig(desired *fluentBitDesiredConfig) (string, error) {
|
||||
if desired == nil || desired.ClickHouse == nil {
|
||||
return "", errors.New("invalid fluent-bit desired config")
|
||||
}
|
||||
|
||||
for _, pattern := range requiredPatterns {
|
||||
ok, err := this.remoteFileContains(fluentBitMainConfigFile, pattern)
|
||||
scheme := strings.ToLower(strings.TrimSpace(desired.ClickHouse.Scheme))
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return "", fmt.Errorf("unsupported clickhouse scheme '%s'", desired.ClickHouse.Scheme)
|
||||
}
|
||||
useTLS := scheme == "https"
|
||||
|
||||
insertHTTP := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
|
||||
insertDNS := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.dns_logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
|
||||
|
||||
lines := []string{
|
||||
"# " + fluentBitManagedMarker,
|
||||
"[SERVICE]",
|
||||
" Flush 2",
|
||||
" Log_Level info",
|
||||
" Parsers_File " + fluentBitParsersFile,
|
||||
" storage.path " + fluentBitStorageDir,
|
||||
" storage.sync normal",
|
||||
" storage.checksum off",
|
||||
" storage.backlog.mem_limit 256MB",
|
||||
"",
|
||||
}
|
||||
|
||||
if hasRole(desired.Roles, fluentBitRoleNode) {
|
||||
lines = append(lines,
|
||||
"[INPUT]",
|
||||
" Name tail",
|
||||
" Path "+desired.HTTPPathPattern,
|
||||
" Tag app.http.logs",
|
||||
" Parser json",
|
||||
" Refresh_Interval 2",
|
||||
" Read_from_Head false",
|
||||
" DB /var/lib/fluent-bit/http-logs.db",
|
||||
" storage.type filesystem",
|
||||
" Mem_Buf_Limit 128MB",
|
||||
" Skip_Long_Lines On",
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
||||
if hasRole(desired.Roles, fluentBitRoleDNS) {
|
||||
lines = append(lines,
|
||||
"[INPUT]",
|
||||
" Name tail",
|
||||
" Path "+desired.DNSPathPattern,
|
||||
" Tag app.dns.logs",
|
||||
" Parser json",
|
||||
" Refresh_Interval 2",
|
||||
" Read_from_Head false",
|
||||
" DB /var/lib/fluent-bit/dns-logs.db",
|
||||
" storage.type filesystem",
|
||||
" Mem_Buf_Limit 128MB",
|
||||
" Skip_Long_Lines On",
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
||||
if hasRole(desired.Roles, fluentBitRoleNode) {
|
||||
lines = append(lines,
|
||||
"[OUTPUT]",
|
||||
" Name http",
|
||||
" Match app.http.logs",
|
||||
" Host "+desired.ClickHouse.Host,
|
||||
" Port "+strconv.Itoa(desired.ClickHouse.Port),
|
||||
" URI /?query="+insertHTTP,
|
||||
" Format json_lines",
|
||||
" http_user ${CH_USER}",
|
||||
" http_passwd ${CH_PASSWORD}",
|
||||
" json_date_key timestamp",
|
||||
" json_date_format epoch",
|
||||
" workers 1",
|
||||
" net.keepalive On",
|
||||
" Retry_Limit False",
|
||||
)
|
||||
if useTLS {
|
||||
lines = append(lines, " tls On")
|
||||
if desired.ClickHouse.TLSSkipVerify {
|
||||
lines = append(lines, " tls.verify Off")
|
||||
} else {
|
||||
lines = append(lines, " tls.verify On")
|
||||
}
|
||||
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
|
||||
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
|
||||
}
|
||||
}
|
||||
lines = append(lines, "")
|
||||
}
|
||||
|
||||
if hasRole(desired.Roles, fluentBitRoleDNS) {
|
||||
lines = append(lines,
|
||||
"[OUTPUT]",
|
||||
" Name http",
|
||||
" Match app.dns.logs",
|
||||
" Host "+desired.ClickHouse.Host,
|
||||
" Port "+strconv.Itoa(desired.ClickHouse.Port),
|
||||
" URI /?query="+insertDNS,
|
||||
" Format json_lines",
|
||||
" http_user ${CH_USER}",
|
||||
" http_passwd ${CH_PASSWORD}",
|
||||
" json_date_key timestamp",
|
||||
" json_date_format epoch",
|
||||
" workers 1",
|
||||
" net.keepalive On",
|
||||
" Retry_Limit False",
|
||||
)
|
||||
if useTLS {
|
||||
lines = append(lines, " tls On")
|
||||
if desired.ClickHouse.TLSSkipVerify {
|
||||
lines = append(lines, " tls.verify Off")
|
||||
} else {
|
||||
lines = append(lines, " tls.verify On")
|
||||
}
|
||||
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
|
||||
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
|
||||
}
|
||||
}
|
||||
lines = append(lines, "")
|
||||
}
|
||||
|
||||
return strings.Join(lines, "\n"), nil
|
||||
}
|
||||
|
||||
func renderManagedEnv(ch *systemconfigs.ClickHouseSetting) string {
|
||||
user := "default"
|
||||
password := ""
|
||||
if ch != nil {
|
||||
if strings.TrimSpace(ch.User) != "" {
|
||||
user = strings.TrimSpace(ch.User)
|
||||
}
|
||||
password = ch.Password
|
||||
}
|
||||
return "CH_USER=" + strconv.Quote(user) + "\n" +
|
||||
"CH_PASSWORD=" + strconv.Quote(password) + "\n"
|
||||
}
|
||||
|
||||
func renderManagedMeta(desired *fluentBitDesiredConfig, configContent string, parserContent string, envContent string) (string, *fluentBitManagedMeta, error) {
|
||||
hashInput := configContent + "\n---\n" + parserContent + "\n---\n" + envContent + "\n---\n" + strings.Join(desired.Roles, ",")
|
||||
hashBytes := sha256.Sum256([]byte(hashInput))
|
||||
hash := fmt.Sprintf("%x", hashBytes[:])
|
||||
|
||||
meta := &fluentBitManagedMeta{
|
||||
Roles: desired.Roles,
|
||||
Hash: hash,
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
SourceVersion: teaconst.Version,
|
||||
}
|
||||
data, err := json.MarshalIndent(meta, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
return "", nil, fmt.Errorf("encode fluent-bit managed metadata failed: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("existing fluent-bit config '%s' does not contain required path '%s'; skip overwrite by design, please update config manually", fluentBitMainConfigFile, pattern)
|
||||
return string(data) + "\n", meta, nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) copyLocalFileToRemote(tempDir string, localPath string, remotePath string, mode os.FileMode) error {
|
||||
tempFile := tempDir + "/" + filepath.Base(remotePath)
|
||||
if err := this.client.Copy(localPath, tempFile, mode); err != nil {
|
||||
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", localPath, err)
|
||||
}
|
||||
_, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("install remote fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
|
||||
stdout, stderr, err := this.client.Exec("if [ -f \"" + path + "\" ]; then echo 1; else echo 0; fi")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||
func (this *BaseInstaller) writeRemoteFileByTemp(tempDir string, remotePath string, content string, mode os.FileMode) error {
|
||||
tempFile := tempDir + "/" + filepath.Base(remotePath) + ".tmp"
|
||||
if _, err := this.client.WriteFile(tempFile, []byte(content)); err != nil {
|
||||
return fmt.Errorf("write temp fluent-bit file '%s' failed: %w", tempFile, err)
|
||||
}
|
||||
return strings.TrimSpace(stdout) == "1", nil
|
||||
|
||||
_, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("write managed fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
|
||||
stdout, stderr, err := this.client.Exec("if grep -F \"" + pattern + "\" \"" + path + "\" >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||
}
|
||||
return strings.TrimSpace(stdout) == "1", nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied bool) error {
|
||||
func (this *BaseInstaller) ensureFluentBitService(tempDir string, binPath string, configChanged bool) error {
|
||||
_, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " +
|
||||
"cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" +
|
||||
"[Unit]\n" +
|
||||
@@ -319,26 +748,90 @@ func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied b
|
||||
"EOF\n" +
|
||||
"fi")
|
||||
|
||||
stdout, stderr, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
|
||||
"if [ \"" + boolToString(configCopied) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
|
||||
"else systemctl start " + fluentBitServiceName + "; fi; else echo no-systemctl; fi")
|
||||
stdout, _, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
||||
if err != nil {
|
||||
return fmt.Errorf("check systemctl failed: %w", err)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(stdout) == "1" {
|
||||
dropInChanged, err := this.ensureServiceDropIn(tempDir, binPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
restartRequired := configChanged || dropInChanged
|
||||
_, stderr, err := this.client.Exec("systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; " +
|
||||
"if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
|
||||
"if [ \"" + boolToString(restartRequired) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
|
||||
"else systemctl start " + fluentBitServiceName + "; fi")
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if configChanged {
|
||||
_, _, _ = this.client.Exec("pkill -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1 || true")
|
||||
}
|
||||
|
||||
if strings.TrimSpace(stdout) == "no-systemctl" {
|
||||
_, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1")
|
||||
if runningErr != nil {
|
||||
_, stderr, err = this.client.Exec(binPath + " -c " + fluentBitMainConfigFile + " >/dev/null 2>&1 &")
|
||||
startCmd := "set -a; [ -f " + shQuote(fluentBitManagedEnvFile) + " ] && . " + shQuote(fluentBitManagedEnvFile) + "; set +a; " +
|
||||
shQuote(binPath) + " -c " + shQuote(fluentBitMainConfigFile) + " >/dev/null 2>&1 &"
|
||||
_, stderr, err := this.client.Exec(startCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) ensureServiceDropIn(tempDir string, binPath string) (bool, error) {
|
||||
_, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitDropInDir))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("prepare fluent-bit drop-in dir failed: %w, stderr: %s", err, stderr)
|
||||
}
|
||||
|
||||
content := "[Service]\n" +
|
||||
"EnvironmentFile=-" + fluentBitManagedEnvFile + "\n" +
|
||||
"ExecStart=\n" +
|
||||
"ExecStart=" + binPath + " -c " + fluentBitMainConfigFile + "\n"
|
||||
|
||||
existing, _, _ := this.client.Exec("if [ -f " + shQuote(fluentBitDropInFile) + " ]; then cat " + shQuote(fluentBitDropInFile) + "; fi")
|
||||
if existing == content {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err := this.writeRemoteFileByTemp(tempDir, fluentBitDropInFile, content, 0644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
|
||||
stdout, stderr, err := this.client.Exec("if [ -f " + shQuote(path) + " ]; then echo 1; else echo 0; fi")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||
}
|
||||
return strings.TrimSpace(stdout) == "1", nil
|
||||
}
|
||||
|
||||
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
|
||||
stdout, stderr, err := this.client.Exec("if grep -F " + shQuote(pattern) + " " + shQuote(path) + " >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||
}
|
||||
return strings.TrimSpace(stdout) == "1", nil
|
||||
}
|
||||
|
||||
func shQuote(value string) string {
|
||||
if value == "" {
|
||||
return "''"
|
||||
}
|
||||
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
|
||||
}
|
||||
|
||||
func boolToString(v bool) string {
|
||||
if v {
|
||||
return "1"
|
||||
|
||||
@@ -34,7 +34,7 @@ function build() {
|
||||
done
|
||||
|
||||
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
|
||||
# 生成 zip 文件名时不包含 plus 标记
|
||||
# 鐢熸垚 zip 鏂囦欢鍚嶆椂涓嶅寘鍚?plus 鏍囪
|
||||
if [ "${TAG}" = "plus" ]; then
|
||||
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
|
||||
else
|
||||
@@ -97,7 +97,7 @@ function build() {
|
||||
done
|
||||
fi
|
||||
|
||||
# 查找 EdgeAPI zip 文件时不包含 plus 标记
|
||||
# 鏌ユ壘 EdgeAPI zip 鏂囦欢鏃朵笉鍖呭惈 plus 鏍囪
|
||||
if [ "${TAG}" = "plus" ]; then
|
||||
EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip"
|
||||
else
|
||||
@@ -107,7 +107,44 @@ function build() {
|
||||
cd "$DIST"/ || exit
|
||||
unzip -q "$(basename "$EDGE_API_ZIP_FILE")"
|
||||
rm -f "$(basename "$EDGE_API_ZIP_FILE")"
|
||||
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件)
|
||||
|
||||
# ensure edge-api package always contains fluent-bit templates/packages
|
||||
FLUENT_ROOT="$ROOT/../../deploy/fluent-bit"
|
||||
FLUENT_DIST="$DIST/edge-api/deploy/fluent-bit"
|
||||
if [ -d "$FLUENT_ROOT" ]; then
|
||||
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || exit 1
|
||||
rm -rf "$FLUENT_DIST"
|
||||
mkdir -p "$FLUENT_DIST"
|
||||
|
||||
FLUENT_FILES=(
|
||||
"fluent-bit.conf"
|
||||
"fluent-bit-dns.conf"
|
||||
"fluent-bit-https.conf"
|
||||
"fluent-bit-dns-https.conf"
|
||||
"fluent-bit-windows.conf"
|
||||
"fluent-bit-windows-https.conf"
|
||||
"parsers.conf"
|
||||
"clickhouse-upstream.conf"
|
||||
"clickhouse-upstream-windows.conf"
|
||||
"logrotate.conf"
|
||||
"README.md"
|
||||
)
|
||||
for file in "${FLUENT_FILES[@]}"; do
|
||||
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -d "$FLUENT_ROOT/packages" ]; then
|
||||
cp -R "$FLUENT_ROOT/packages" "$FLUENT_DIST/"
|
||||
fi
|
||||
|
||||
rm -f "$FLUENT_DIST/.gitignore"
|
||||
rm -f "$FLUENT_DIST"/logs.db*
|
||||
rm -rf "$FLUENT_DIST/storage"
|
||||
fi
|
||||
|
||||
# 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
|
||||
find . -name "*.mmdb" -type f -delete
|
||||
find . -type d -name "iplibrary" -empty -delete
|
||||
cd - || exit
|
||||
@@ -150,7 +187,7 @@ function build() {
|
||||
#find "$DIST" -name "*.css.map" -delete
|
||||
#find "$DIST" -name "*.js.map" -delete
|
||||
|
||||
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件)
|
||||
# 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
|
||||
find "$DIST" -name "*.mmdb" -type f -delete
|
||||
find "$DIST" -type d -name "iplibrary" -empty -delete
|
||||
|
||||
@@ -167,6 +204,39 @@ function build() {
|
||||
echo "[done]"
|
||||
}
|
||||
|
||||
function verify_fluent_bit_package_matrix() {
|
||||
FLUENT_ROOT=$1
|
||||
ARCH=$2
|
||||
REQUIRED_FILES=()
|
||||
if [ "$ARCH" = "amd64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||
)
|
||||
elif [ "$ARCH" = "arm64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||
)
|
||||
else
|
||||
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||
return 1
|
||||
fi
|
||||
|
||||
MISSING=0
|
||||
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||
MISSING=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$MISSING" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function lookup-version() {
|
||||
FILE=$1
|
||||
VERSION_DATA=$(cat "$FILE")
|
||||
|
||||
@@ -3,13 +3,18 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||
"github.com/iwind/TeaGo/actions"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const clickhouseConfigCode = "clickhouseConfig"
|
||||
@@ -29,18 +34,18 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default", Scheme: "http"}
|
||||
cfg := &systemconfigs.ClickHouseSetting{Port: 8443, Database: "default", Scheme: "https"}
|
||||
if len(resp.ValueJSON) > 0 {
|
||||
_ = json.Unmarshal(resp.ValueJSON, cfg)
|
||||
}
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = 8123
|
||||
cfg.Port = 8443
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = "default"
|
||||
}
|
||||
if strings.TrimSpace(cfg.Scheme) == "" {
|
||||
cfg.Scheme = "http"
|
||||
cfg.Scheme = "https"
|
||||
}
|
||||
this.Data["config"] = map[string]interface{}{
|
||||
"host": cfg.Host,
|
||||
@@ -49,9 +54,17 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||
"password": cfg.Password,
|
||||
"database": cfg.Database,
|
||||
"scheme": cfg.Scheme,
|
||||
"tlsSkipVerify": cfg.TLSSkipVerify,
|
||||
"tlsServerName": cfg.TLSServerName,
|
||||
}
|
||||
|
||||
// 自动检测连接状态
|
||||
connStatus := "unconfigured" // unconfigured / connected / disconnected
|
||||
connError := ""
|
||||
if strings.TrimSpace(cfg.Host) != "" {
|
||||
connStatus, connError = this.probeClickHouse(cfg)
|
||||
}
|
||||
this.Data["connStatus"] = connStatus
|
||||
this.Data["connError"] = connError
|
||||
|
||||
this.Show()
|
||||
}
|
||||
|
||||
@@ -62,20 +75,18 @@ func (this *ClickHouseAction) RunPost(params struct {
|
||||
Password string
|
||||
Database string
|
||||
Scheme string
|
||||
TLSSkipVerify bool
|
||||
TLSServerName string
|
||||
|
||||
Must *actions.Must
|
||||
}) {
|
||||
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
|
||||
if params.Port <= 0 {
|
||||
params.Port = 8123
|
||||
}
|
||||
if params.Database == "" {
|
||||
params.Database = "default"
|
||||
}
|
||||
if params.Scheme != "https" {
|
||||
params.Scheme = "http"
|
||||
if params.Scheme != "http" {
|
||||
params.Scheme = "https"
|
||||
}
|
||||
if params.Port <= 0 {
|
||||
params.Port = 8443
|
||||
}
|
||||
password := params.Password
|
||||
if password == "" {
|
||||
@@ -94,8 +105,8 @@ func (this *ClickHouseAction) RunPost(params struct {
|
||||
Password: password,
|
||||
Database: params.Database,
|
||||
Scheme: params.Scheme,
|
||||
TLSSkipVerify: params.TLSSkipVerify,
|
||||
TLSServerName: strings.TrimSpace(params.TLSServerName),
|
||||
TLSSkipVerify: true,
|
||||
TLSServerName: "",
|
||||
}
|
||||
valueJSON, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
@@ -112,3 +123,51 @@ func (this *ClickHouseAction) RunPost(params struct {
|
||||
}
|
||||
this.Success()
|
||||
}
|
||||
|
||||
// probeClickHouse 快速检测 ClickHouse 连接状态(SELECT 1)
|
||||
func (this *ClickHouseAction) probeClickHouse(cfg *systemconfigs.ClickHouseSetting) (status string, errMsg string) {
|
||||
scheme := strings.ToLower(strings.TrimSpace(cfg.Scheme))
|
||||
if scheme == "" {
|
||||
scheme = "https"
|
||||
}
|
||||
port := cfg.Port
|
||||
if port <= 0 {
|
||||
port = 8443
|
||||
}
|
||||
db := cfg.Database
|
||||
if db == "" {
|
||||
db = "default"
|
||||
}
|
||||
|
||||
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s", scheme, cfg.Host, port, db)
|
||||
|
||||
transport := &http.Transport{}
|
||||
if scheme == "https" {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 3 * time.Second,
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, testURL, nil)
|
||||
if err != nil {
|
||||
return "disconnected", err.Error()
|
||||
}
|
||||
if cfg.User != "" || cfg.Password != "" {
|
||||
req.SetBasicAuth(cfg.User, cfg.Password)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "disconnected", err.Error()
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "disconnected", fmt.Sprintf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return "connected", ""
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ func (this *ClickHouseAction) Init() {
|
||||
func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||
this.Data["mainTab"] = "clickhouse"
|
||||
this.Data["config"] = map[string]interface{}{
|
||||
"host": "", "port": 8123, "user": "", "password": "", "database": "default",
|
||||
"host": "", "port": 8443, "user": "", "password": "", "database": "default", "scheme": "https",
|
||||
}
|
||||
this.Show()
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ func init() {
|
||||
Get("/logs", new(LogsAction)).
|
||||
Post("/status", new(StatusAction)).
|
||||
GetPost("/clickhouse", new(ClickHouseAction)).
|
||||
Post("/testClickhouse", new(TestClickHouseAction)).
|
||||
EndAll()
|
||||
})
|
||||
}
|
||||
|
||||
92
EdgeAdmin/internal/web/actions/default/db/testClickhouse.go
Normal file
92
EdgeAdmin/internal/web/actions/default/db/testClickhouse.go
Normal file
@@ -0,0 +1,92 @@
|
||||
//go:build plus
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||
"github.com/iwind/TeaGo/actions"
|
||||
)
|
||||
|
||||
type TestClickHouseAction struct {
|
||||
actionutils.ParentAction
|
||||
}
|
||||
|
||||
func (this *TestClickHouseAction) Init() {
|
||||
this.Nav("db", "db", "clickhouse")
|
||||
}
|
||||
|
||||
func (this *TestClickHouseAction) RunPost(params struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
Scheme string
|
||||
|
||||
Must *actions.Must
|
||||
}) {
|
||||
params.Must.
|
||||
Field("host", params.Host).
|
||||
Require("请输入 ClickHouse 连接地址")
|
||||
|
||||
if params.Database == "" {
|
||||
params.Database = "default"
|
||||
}
|
||||
scheme := "https"
|
||||
if strings.EqualFold(params.Scheme, "http") {
|
||||
scheme = "http"
|
||||
}
|
||||
if params.Port <= 0 {
|
||||
params.Port = 8443
|
||||
}
|
||||
|
||||
// 构造测试请求
|
||||
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s",
|
||||
scheme, params.Host, params.Port, params.Database)
|
||||
|
||||
transport := &http.Transport{}
|
||||
if scheme == "https" {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, testURL, nil)
|
||||
if err != nil {
|
||||
this.Fail("请求构造失败: " + err.Error())
|
||||
return
|
||||
}
|
||||
if params.User != "" || params.Password != "" {
|
||||
req.SetBasicAuth(params.User, params.Password)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
this.Fail("连接失败: " + err.Error())
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
msg := strings.TrimSpace(string(body))
|
||||
if len(msg) > 200 {
|
||||
msg = msg[:200] + "..."
|
||||
}
|
||||
this.Fail(fmt.Sprintf("ClickHouse 返回 HTTP %d: %s", resp.StatusCode, msg))
|
||||
return
|
||||
}
|
||||
|
||||
this.Success()
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//go:build !plus
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||
"github.com/iwind/TeaGo/actions"
|
||||
)
|
||||
|
||||
type TestClickHouseAction struct {
|
||||
actionutils.ParentAction
|
||||
}
|
||||
|
||||
func (this *TestClickHouseAction) Init() {
|
||||
this.Nav("db", "db", "clickhouse")
|
||||
}
|
||||
|
||||
func (this *TestClickHouseAction) RunPost(params struct {
|
||||
Host string
|
||||
Must *actions.Must
|
||||
}) {
|
||||
this.Fail("请使用商业版以测试 ClickHouse 连接")
|
||||
}
|
||||
@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
|
||||
<div>
|
||||
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
||||
|
||||
<!-- 网站 -->
|
||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
|
||||
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
|
||||
|
||||
|
||||
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
||||
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
||||
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
|
||||
</span>
|
||||
|
||||
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
|
||||
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
||||
</div>
|
||||
</div>`
|
||||
|
||||
@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
|
||||
<div>
|
||||
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
||||
|
||||
<!-- 网站 -->
|
||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
|
||||
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
|
||||
|
||||
|
||||
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
||||
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
||||
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
|
||||
</span>
|
||||
|
||||
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
|
||||
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
||||
</div>
|
||||
</div>`
|
||||
|
||||
@@ -1,70 +1,76 @@
|
||||
{$layout}
|
||||
{$template "menu"}
|
||||
|
||||
<h3>ClickHouse 配置</h3>
|
||||
<p class="comment">用于访问日志列表查询(logs_ingest 表)。配置后,访问日志列表将优先从 ClickHouse 读取;不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。</p>
|
||||
<div style="display:flex;align-items:baseline;gap:0.8em;margin-bottom:0.5em">
|
||||
<h3 style="margin:0">ClickHouse 配置</h3>
|
||||
<span v-if="connStatus === 'connected'" class="ui label green tiny" style="vertical-align:baseline">
|
||||
<i class="icon circle" style="margin-right:0.25em"></i>已连接
|
||||
</span>
|
||||
<span v-else-if="connStatus === 'disconnected'" class="ui label red tiny" style="vertical-align:baseline"
|
||||
:title="connError">
|
||||
<i class="icon circle" style="margin-right:0.25em"></i>已断开
|
||||
</span>
|
||||
<span v-else class="ui label grey tiny" style="vertical-align:baseline">
|
||||
<i class="icon circle outline" style="margin-right:0.25em"></i>未配置
|
||||
</span>
|
||||
</div>
|
||||
<p class="comment">配置后,需要在网站列表-日志策略中创建"文件+ClickHouse"的策略,才能将访问日志绕过api组件直接由边缘节点发送给ClickHouse。</p>
|
||||
|
||||
<form method="post" class="ui form" data-tea-action="$" data-tea-success="success" @submit.prevent="onSubmit">
|
||||
<form method="post" class="ui form" @submit.prevent="onSubmit">
|
||||
<csrf-token></csrf-token>
|
||||
<table class="ui table definition selectable">
|
||||
<tr>
|
||||
<td class="title">连接地址(Host)</td>
|
||||
<td>
|
||||
<input type="text" name="host" maxlength="200" ref="focus" placeholder="如 127.0.0.1 或 clickhouse.example.com" :value="config.host"/>
|
||||
<input type="text" name="host" maxlength="200" ref="focus"
|
||||
placeholder="如 127.0.0.1 或 clickhouse.example.com" v-model="form.host" />
|
||||
<p class="comment">ClickHouse 服务器地址。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>协议(Scheme)</td>
|
||||
<td>
|
||||
<select name="scheme" class="ui dropdown auto-width">
|
||||
<option value="http" :selected="config.scheme != 'https'">HTTP</option>
|
||||
<option value="https" :selected="config.scheme == 'https'">HTTPS</option>
|
||||
<select name="scheme" class="ui dropdown auto-width" v-model="form.scheme">
|
||||
<option value="http">HTTP</option>
|
||||
<option value="https">HTTPS</option>
|
||||
</select>
|
||||
<p class="comment">默认 HTTP;选择 HTTPS 时将启用 TLS 连接。</p>
|
||||
<p class="comment">默认 HTTPS;当前后台固定跳过证书校验。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>端口(Port)</td>
|
||||
<td>
|
||||
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/>
|
||||
<p class="comment">接口端口,HTTP 默认 8123,HTTPS 常用 8443(以你的 ClickHouse 实际配置为准)。</p>
|
||||
<input type="number" name="port" min="1" max="65535" style="width:6em" v-model.number="form.port" />
|
||||
<p class="comment">接口端口默认 8443(HTTPS);请与 ClickHouse 实际开放端口保持一致。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>用户名(User)</td>
|
||||
<td>
|
||||
<input type="text" name="user" maxlength="100" :value="config.user"/>
|
||||
<input type="text" name="user" maxlength="100" v-model="form.user" />
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>密码(Password)</td>
|
||||
<td>
|
||||
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" value=""/>
|
||||
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" v-model="form.password" />
|
||||
<p class="comment">留空则不修改已保存的密码。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>TLS 跳过证书校验</td>
|
||||
<td>
|
||||
<checkbox name="tlsSkipVerify" value="1" :checked="config.tlsSkipVerify"></checkbox>
|
||||
<p class="comment">仅测试环境建议开启;生产建议关闭并使用受信任证书。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>TLS Server Name</td>
|
||||
<td>
|
||||
<input type="text" name="tlsServerName" maxlength="200" placeholder="可选:证书校验域名(SNI)" :value="config.tlsServerName"/>
|
||||
<p class="comment">可选;当 ClickHouse 证书域名与连接 Host 不一致时使用。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>数据库名(Database)</td>
|
||||
<td>
|
||||
<input type="text" name="database" maxlength="100" placeholder="default" :value="config.database"/>
|
||||
<input type="text" name="database" maxlength="100" placeholder="default" v-model="form.database" />
|
||||
<p class="comment">logs_ingest 表所在库,默认 default。</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<div style="display:flex; align-items:center; gap:0.5em; margin-top:1em">
|
||||
<submit-btn></submit-btn>
|
||||
<button type="button" class="ui button basic blue" @click="testConnection" :disabled="isTesting">
|
||||
<i :class="isTesting ? 'icon spinner loading' : 'icon plug'"></i> 测试连接
|
||||
</button>
|
||||
<span v-if="testResult"
|
||||
:style="{color: testOk ? '#21ba45' : '#db2828', fontWeight:'bold'}">{{testResult}}</span>
|
||||
</div>
|
||||
</form>
|
||||
@@ -1,4 +1,22 @@
|
||||
Tea.context(function () {
|
||||
var config = this.config || {}
|
||||
this.form = {
|
||||
host: config.host || "",
|
||||
scheme: config.scheme || "https",
|
||||
port: config.port > 0 ? config.port : 8443,
|
||||
user: config.user || "",
|
||||
password: "",
|
||||
database: config.database || "default",
|
||||
}
|
||||
|
||||
this.isTesting = false
|
||||
this.testResult = ""
|
||||
this.testOk = false
|
||||
|
||||
// 页面加载时的连接状态(后端自动检测)
|
||||
this.connStatus = this.connStatus || "unconfigured"
|
||||
this.connError = this.connError || ""
|
||||
|
||||
this.success = function () {
|
||||
teaweb.success("保存成功")
|
||||
}
|
||||
@@ -9,4 +27,50 @@ Tea.context(function () {
|
||||
Tea.Vue.success()
|
||||
})
|
||||
}
|
||||
this.testConnection = function () {
|
||||
var that = Tea.Vue
|
||||
that.isTesting = true
|
||||
that.testResult = ""
|
||||
|
||||
var form = document.querySelector("form")
|
||||
var fd = new FormData(form)
|
||||
fd.set("host", that.form.host || "")
|
||||
fd.set("scheme", that.form.scheme || "https")
|
||||
fd.set("port", String(that.form.port > 0 ? that.form.port : 8443))
|
||||
fd.set("user", that.form.user || "")
|
||||
fd.set("password", that.form.password || "")
|
||||
fd.set("database", that.form.database || "default")
|
||||
|
||||
var xhr = new XMLHttpRequest()
|
||||
xhr.open("POST", Tea.url("/db/testClickhouse"), true)
|
||||
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest")
|
||||
xhr.timeout = 10000
|
||||
xhr.onload = function () {
|
||||
that.isTesting = false
|
||||
try {
|
||||
var resp = JSON.parse(xhr.responseText)
|
||||
if (resp.code === 200) {
|
||||
that.testOk = true
|
||||
that.testResult = "✅ 连接成功"
|
||||
} else {
|
||||
that.testOk = false
|
||||
that.testResult = "❌ " + (resp.message || "连接失败")
|
||||
}
|
||||
} catch (e) {
|
||||
that.testOk = false
|
||||
that.testResult = "❌ 响应解析失败"
|
||||
}
|
||||
}
|
||||
xhr.onerror = function () {
|
||||
that.isTesting = false
|
||||
that.testOk = false
|
||||
that.testResult = "❌ 网络请求失败"
|
||||
}
|
||||
xhr.ontimeout = function () {
|
||||
that.isTesting = false
|
||||
that.testOk = false
|
||||
that.testResult = "❌ 请求超时"
|
||||
}
|
||||
xhr.send(fd)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -19,6 +19,7 @@ type NSNodeConfig struct {
|
||||
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
|
||||
AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"`
|
||||
AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"`
|
||||
AccessLogRotate *serverconfigs.AccessLogRotateConfig `yaml:"accessLogRotate" json:"accessLogRotate"`
|
||||
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
|
||||
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
|
||||
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`
|
||||
|
||||
@@ -2,8 +2,65 @@
|
||||
|
||||
package serverconfigs
|
||||
|
||||
const (
|
||||
DefaultAccessLogRotateMaxSizeMB = 256
|
||||
DefaultAccessLogRotateMaxBackups = 14
|
||||
DefaultAccessLogRotateMaxAgeDays = 7
|
||||
)
|
||||
|
||||
// AccessLogRotateConfig 文件轮转配置。
|
||||
type AccessLogRotateConfig struct {
|
||||
MaxSizeMB int `yaml:"maxSizeMB" json:"maxSizeMB"` // 单文件最大大小(MB)
|
||||
MaxBackups int `yaml:"maxBackups" json:"maxBackups"` // 保留历史文件数
|
||||
MaxAgeDays int `yaml:"maxAgeDays" json:"maxAgeDays"` // 保留天数
|
||||
Compress *bool `yaml:"compress" json:"compress"` // 是否压缩历史文件
|
||||
LocalTime *bool `yaml:"localTime" json:"localTime"` // 轮转时间使用本地时区
|
||||
}
|
||||
|
||||
// AccessLogFileStorageConfig 文件存储配置
|
||||
type AccessLogFileStorageConfig struct {
|
||||
Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second}
|
||||
AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录
|
||||
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 文件轮转配置
|
||||
}
|
||||
|
||||
// NewDefaultAccessLogRotateConfig 默认轮转配置。
|
||||
func NewDefaultAccessLogRotateConfig() *AccessLogRotateConfig {
|
||||
compress := false
|
||||
localTime := true
|
||||
return &AccessLogRotateConfig{
|
||||
MaxSizeMB: DefaultAccessLogRotateMaxSizeMB,
|
||||
MaxBackups: DefaultAccessLogRotateMaxBackups,
|
||||
MaxAgeDays: DefaultAccessLogRotateMaxAgeDays,
|
||||
Compress: &compress,
|
||||
LocalTime: &localTime,
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize 归一化轮转配置,空值/非法值回退默认。
|
||||
func (c *AccessLogRotateConfig) Normalize() *AccessLogRotateConfig {
|
||||
defaultConfig := NewDefaultAccessLogRotateConfig()
|
||||
if c == nil {
|
||||
return defaultConfig
|
||||
}
|
||||
|
||||
if c.MaxSizeMB > 0 {
|
||||
defaultConfig.MaxSizeMB = c.MaxSizeMB
|
||||
}
|
||||
if c.MaxBackups > 0 {
|
||||
defaultConfig.MaxBackups = c.MaxBackups
|
||||
}
|
||||
if c.MaxAgeDays > 0 {
|
||||
defaultConfig.MaxAgeDays = c.MaxAgeDays
|
||||
}
|
||||
if c.Compress != nil {
|
||||
v := *c.Compress
|
||||
defaultConfig.Compress = &v
|
||||
}
|
||||
if c.LocalTime != nil {
|
||||
v := *c.LocalTime
|
||||
defaultConfig.LocalTime = &v
|
||||
}
|
||||
|
||||
return defaultConfig
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ func NewGlobalServerConfig() *GlobalServerConfig {
|
||||
config.HTTPAccessLog.EnableResponseHeaders = true
|
||||
config.HTTPAccessLog.EnableCookies = true
|
||||
config.HTTPAccessLog.EnableServerNotFound = true
|
||||
config.HTTPAccessLog.Rotate = NewDefaultAccessLogRotateConfig()
|
||||
|
||||
config.Log.RecordServerError = false
|
||||
|
||||
@@ -79,6 +80,7 @@ type GlobalServerConfig struct {
|
||||
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
|
||||
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse(双写/单写)
|
||||
FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用)
|
||||
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 本地日志轮转配置(lumberjack)
|
||||
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
|
||||
|
||||
Stat struct {
|
||||
|
||||
@@ -107,11 +107,12 @@ function copy_fluent_bit_assets() {
|
||||
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
||||
return 1
|
||||
fi
|
||||
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
|
||||
|
||||
rm -rf "$FLUENT_DIST"
|
||||
mkdir -p "$FLUENT_DIST"
|
||||
|
||||
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
|
||||
for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
|
||||
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||
fi
|
||||
@@ -129,6 +130,43 @@ function copy_fluent_bit_assets() {
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$FLUENT_DIST/.gitignore"
|
||||
rm -f "$FLUENT_DIST"/logs.db*
|
||||
rm -rf "$FLUENT_DIST/storage"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function verify_fluent_bit_package_matrix() {
|
||||
FLUENT_ROOT=$1
|
||||
ARCH=$2
|
||||
REQUIRED_FILES=()
|
||||
if [ "$ARCH" = "amd64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||
)
|
||||
elif [ "$ARCH" = "arm64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||
)
|
||||
else
|
||||
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||
return 1
|
||||
fi
|
||||
|
||||
MISSING=0
|
||||
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||
MISSING=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$MISSING" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
github.com/mdlayher/netlink v1.7.2
|
||||
github.com/miekg/dns v1.1.58
|
||||
github.com/shirou/gopsutil/v3 v3.24.2
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
golang.org/x/sys v0.38.0
|
||||
google.golang.org/grpc v1.78.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
|
||||
@@ -111,6 +111,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -9,7 +9,9 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
|
||||
"gopkg.in/natefinch/lumberjack.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -30,18 +32,23 @@ func SharedDNSFileWriter() *DNSFileWriter {
|
||||
return sharedDNSFileWriter
|
||||
}
|
||||
|
||||
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集.
|
||||
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集。
|
||||
// 文件轮转由 lumberjack 内建完成。
|
||||
type DNSFileWriter struct {
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
file *lumberjack.Logger
|
||||
rotateConfig *serverconfigs.AccessLogRotateConfig
|
||||
inited bool
|
||||
}
|
||||
|
||||
// NewDNSFileWriter 创建 DNS 本地日志写入器.
|
||||
func NewDNSFileWriter() *DNSFileWriter {
|
||||
logDir := resolveDefaultDNSLogDir()
|
||||
return &DNSFileWriter{dir: logDir}
|
||||
return &DNSFileWriter{
|
||||
dir: logDir,
|
||||
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
func resolveDefaultDNSLogDir() string {
|
||||
@@ -102,6 +109,25 @@ func (w *DNSFileWriter) SetDir(dir string) {
|
||||
w.dir = dir
|
||||
}
|
||||
|
||||
// SetRotateConfig 更新日志轮转配置并重建 writer。
|
||||
func (w *DNSFileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
|
||||
normalized := config.Normalize()
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
if equalDNSRotateConfig(w.rotateConfig, normalized) {
|
||||
return
|
||||
}
|
||||
|
||||
if w.file != nil {
|
||||
_ = w.file.Close()
|
||||
w.file = nil
|
||||
}
|
||||
w.inited = false
|
||||
w.rotateConfig = normalized
|
||||
}
|
||||
|
||||
// EnsureInit 在启动时预创建目录与 access.log.
|
||||
func (w *DNSFileWriter) EnsureInit() error {
|
||||
if w.dir == "" {
|
||||
@@ -127,13 +153,16 @@ func (w *DNSFileWriter) init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error())
|
||||
return err
|
||||
rotateConfig := w.rotateConfig.Normalize()
|
||||
w.file = &lumberjack.Logger{
|
||||
Filename: filepath.Join(w.dir, "access.log"),
|
||||
MaxSize: rotateConfig.MaxSizeMB,
|
||||
MaxBackups: rotateConfig.MaxBackups,
|
||||
MaxAge: rotateConfig.MaxAgeDays,
|
||||
Compress: *rotateConfig.Compress,
|
||||
LocalTime: *rotateConfig.LocalTime,
|
||||
}
|
||||
|
||||
w.file = fp
|
||||
w.inited = true
|
||||
return nil
|
||||
}
|
||||
@@ -148,14 +177,14 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
fp := w.file
|
||||
file := w.file
|
||||
w.mu.Unlock()
|
||||
if fp == nil {
|
||||
if file == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, log := range logs {
|
||||
ingestLog := FromNSAccessLog(log, clusterId)
|
||||
for _, logItem := range logs {
|
||||
ingestLog := FromNSAccessLog(logItem, clusterId)
|
||||
if ingestLog == nil {
|
||||
continue
|
||||
}
|
||||
@@ -163,11 +192,11 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, _ = fp.Write(append(line, '\n'))
|
||||
_, _ = file.Write(append(line, '\n'))
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen 关闭并重新打开日志文件(配合 logrotate).
|
||||
// Reopen 关闭并重建日志 writer(供兼容调用).
|
||||
func (w *DNSFileWriter) Reopen() error {
|
||||
w.mu.Lock()
|
||||
if w.file != nil {
|
||||
@@ -197,3 +226,14 @@ func (w *DNSFileWriter) Close() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func equalDNSRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
|
||||
if left == nil || right == nil {
|
||||
return left == right
|
||||
}
|
||||
return left.MaxSizeMB == right.MaxSizeMB &&
|
||||
left.MaxBackups == right.MaxBackups &&
|
||||
left.MaxAgeDays == right.MaxAgeDays &&
|
||||
*left.Compress == *right.Compress &&
|
||||
*left.LocalTime == *right.LocalTime
|
||||
}
|
||||
|
||||
@@ -110,6 +110,7 @@ func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) {
|
||||
teaconst.IsPlus = config.IsPlus
|
||||
|
||||
accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
|
||||
accesslogs.SharedDNSFileWriter().SetRotateConfig(config.AccessLogRotate)
|
||||
|
||||
needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
|
||||
if needWriteFile {
|
||||
|
||||
@@ -181,11 +181,12 @@ function copy_fluent_bit_assets() {
|
||||
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
||||
return 1
|
||||
fi
|
||||
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
|
||||
|
||||
rm -rf "$FLUENT_DIST"
|
||||
mkdir -p "$FLUENT_DIST"
|
||||
|
||||
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
|
||||
for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
|
||||
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||
fi
|
||||
@@ -203,6 +204,43 @@ function copy_fluent_bit_assets() {
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$FLUENT_DIST/.gitignore"
|
||||
rm -f "$FLUENT_DIST"/logs.db*
|
||||
rm -rf "$FLUENT_DIST/storage"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function verify_fluent_bit_package_matrix() {
|
||||
FLUENT_ROOT=$1
|
||||
ARCH=$2
|
||||
REQUIRED_FILES=()
|
||||
if [ "$ARCH" = "amd64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||
)
|
||||
elif [ "$ARCH" = "arm64" ]; then
|
||||
REQUIRED_FILES=(
|
||||
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||
)
|
||||
else
|
||||
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||
return 1
|
||||
fi
|
||||
|
||||
MISSING=0
|
||||
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||
MISSING=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$MISSING" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ require (
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
github.com/tdewolff/minify/v2 v2.20.20
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
|
||||
golang.org/x/image v0.15.0
|
||||
golang.org/x/net v0.47.0
|
||||
|
||||
@@ -305,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
@@ -9,7 +9,9 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"gopkg.in/natefinch/lumberjack.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -30,11 +32,13 @@ const (
|
||||
envLogDir = "EDGE_LOG_DIR"
|
||||
)
|
||||
|
||||
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 logrotate 与 Fluent Bit 采集
|
||||
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 Fluent Bit 采集。
|
||||
// 文件轮转由 lumberjack 内建完成。
|
||||
type FileWriter struct {
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
files map[string]*os.File // access.log, waf.log, error.log
|
||||
files map[string]*lumberjack.Logger // access.log, waf.log, error.log
|
||||
rotateConfig *serverconfigs.AccessLogRotateConfig
|
||||
inited bool
|
||||
}
|
||||
|
||||
@@ -43,7 +47,8 @@ func NewFileWriter() *FileWriter {
|
||||
dir := resolveDefaultLogDir()
|
||||
return &FileWriter{
|
||||
dir: dir,
|
||||
files: make(map[string]*os.File),
|
||||
files: make(map[string]*lumberjack.Logger),
|
||||
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,9 +102,9 @@ func (w *FileWriter) SetDir(dir string) {
|
||||
return
|
||||
}
|
||||
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
for name, file := range w.files {
|
||||
if file != nil {
|
||||
_ = file.Close()
|
||||
}
|
||||
w.files[name] = nil
|
||||
}
|
||||
@@ -107,6 +112,27 @@ func (w *FileWriter) SetDir(dir string) {
|
||||
w.dir = dir
|
||||
}
|
||||
|
||||
// SetRotateConfig 更新日志轮转配置并重建 writer。
|
||||
func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
|
||||
normalized := config.Normalize()
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
if equalRotateConfig(w.rotateConfig, normalized) {
|
||||
return
|
||||
}
|
||||
|
||||
for name, file := range w.files {
|
||||
if file != nil {
|
||||
_ = file.Close()
|
||||
}
|
||||
w.files[name] = nil
|
||||
}
|
||||
w.inited = false
|
||||
w.rotateConfig = normalized
|
||||
}
|
||||
|
||||
// IsEnabled 是否启用落盘(目录非空即视为启用)
|
||||
func (w *FileWriter) IsEnabled() bool {
|
||||
return w.dir != ""
|
||||
@@ -138,17 +164,24 @@ func (w *FileWriter) init() error {
|
||||
if w.files[name] != nil {
|
||||
continue
|
||||
}
|
||||
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
w.files[name] = fp
|
||||
w.files[name] = w.newLogger(name)
|
||||
}
|
||||
w.inited = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger {
|
||||
rotateConfig := w.rotateConfig.Normalize()
|
||||
return &lumberjack.Logger{
|
||||
Filename: filepath.Join(w.dir, fileName),
|
||||
MaxSize: rotateConfig.MaxSizeMB,
|
||||
MaxBackups: rotateConfig.MaxBackups,
|
||||
MaxAge: rotateConfig.MaxAgeDays,
|
||||
Compress: *rotateConfig.Compress,
|
||||
LocalTime: *rotateConfig.LocalTime,
|
||||
}
|
||||
}
|
||||
|
||||
// Write 将一条访问日志按 log_type 写入对应文件(access.log / waf.log / error.log)
|
||||
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
||||
if w.dir == "" {
|
||||
@@ -173,13 +206,12 @@ func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
||||
fileName = "access.log"
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp := w.files[fileName]
|
||||
file := w.files[fileName]
|
||||
w.mu.Unlock()
|
||||
if fp == nil {
|
||||
if file == nil {
|
||||
return
|
||||
}
|
||||
// 单行写入,末尾换行,便于 Fluent Bit / JSON 解析
|
||||
_, err = fp.Write(append(line, '\n'))
|
||||
_, err = file.Write(append(line, '\n'))
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
|
||||
}
|
||||
@@ -194,49 +226,49 @@ func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
accessFp := w.files["access.log"]
|
||||
wafFp := w.files["waf.log"]
|
||||
errorFp := w.files["error.log"]
|
||||
accessFile := w.files["access.log"]
|
||||
wafFile := w.files["waf.log"]
|
||||
errorFile := w.files["error.log"]
|
||||
w.mu.Unlock()
|
||||
if accessFp == nil && wafFp == nil && errorFp == nil {
|
||||
if accessFile == nil && wafFile == nil && errorFile == nil {
|
||||
return
|
||||
}
|
||||
for _, l := range logs {
|
||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
||||
for _, logItem := range logs {
|
||||
ingest, logType := FromHTTPAccessLog(logItem, clusterId)
|
||||
line, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
line = append(line, '\n')
|
||||
var fp *os.File
|
||||
var file *lumberjack.Logger
|
||||
switch logType {
|
||||
case LogTypeWAF:
|
||||
fp = wafFp
|
||||
file = wafFile
|
||||
case LogTypeError:
|
||||
fp = errorFp
|
||||
file = errorFile
|
||||
default:
|
||||
fp = accessFp
|
||||
file = accessFile
|
||||
}
|
||||
if fp != nil {
|
||||
_, _ = fp.Write(line)
|
||||
if file != nil {
|
||||
_, _ = file.Write(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen 关闭并重新打开所有日志文件(供 logrotate copytruncate 或 SIGHUP 后重开句柄)
|
||||
// Reopen 关闭并重建所有日志 writer(供 SIGHUP 兼容调用)。
|
||||
func (w *FileWriter) Reopen() error {
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
for name, file := range w.files {
|
||||
if file != nil {
|
||||
_ = file.Close()
|
||||
w.files[name] = nil
|
||||
}
|
||||
}
|
||||
w.inited = false
|
||||
w.mu.Unlock()
|
||||
return w.init()
|
||||
}
|
||||
|
||||
@@ -245,9 +277,9 @@ func (w *FileWriter) Close() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var lastErr error
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
if err := f.Close(); err != nil {
|
||||
for name, file := range w.files {
|
||||
if file != nil {
|
||||
if err := file.Close(); err != nil {
|
||||
lastErr = err
|
||||
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
|
||||
}
|
||||
@@ -257,3 +289,14 @@ func (w *FileWriter) Close() error {
|
||||
w.inited = false
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
|
||||
if left == nil || right == nil {
|
||||
return left == right
|
||||
}
|
||||
return left.MaxSizeMB == right.MaxSizeMB &&
|
||||
left.MaxBackups == right.MaxBackups &&
|
||||
left.MaxAgeDays == right.MaxAgeDays &&
|
||||
*left.Compress == *right.Compress &&
|
||||
*left.LocalTime == *right.LocalTime
|
||||
}
|
||||
|
||||
@@ -577,7 +577,7 @@ func (this *Node) listenSignals() {
|
||||
goman.New(func() {
|
||||
for sig := range queue {
|
||||
if sig == syscall.SIGHUP {
|
||||
// 供 logrotate 等旋转日志后重开句柄
|
||||
// 兼容 SIGHUP:重建本地日志 writer
|
||||
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
|
||||
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
|
||||
}
|
||||
@@ -890,6 +890,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) {
|
||||
var accessLogFilePath string
|
||||
if config != nil && config.GlobalServerConfig != nil {
|
||||
accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
|
||||
accesslogs.SharedFileWriter().SetRotateConfig(config.GlobalServerConfig.HTTPAccessLog.Rotate)
|
||||
}
|
||||
accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)
|
||||
|
||||
|
||||
1984
config.xml
Normal file
1984
config.xml
Normal file
File diff suppressed because it is too large
Load Diff
348
deploy.sh
Normal file
348
deploy.sh
Normal file
@@ -0,0 +1,348 @@
|
||||
#!/bin/bash
|
||||
# CloudWAF 部署脚本 - Gname DNS 提供商更新
|
||||
# 使用方法: ./deploy.sh [zip文件路径]
|
||||
|
||||
set -e # 遇到错误立即退出
|
||||
|
||||
# ========== 配置区域 ==========
|
||||
# 支持环境变量和配置文件
|
||||
# 从环境变量读取
|
||||
INSTALL_BASE_DIR="${BRAND_INSTALL_PATH:-/usr/local/goedge}"
|
||||
|
||||
# 从配置文件读取(如果存在)
|
||||
if [ -f "/etc/goedge/brand.conf" ]; then
|
||||
source /etc/goedge/brand.conf
|
||||
if [ -n "$BRAND_INSTALL_PATH" ]; then
|
||||
INSTALL_BASE_DIR="$BRAND_INSTALL_PATH"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 如果通过参数传入 zip 文件路径,则使用参数;否则使用默认路径
|
||||
ZIP_FILE_RAW="${1:-/tmp/edge-admin-linux-amd64-plus-v1.3.8.zip}"
|
||||
# 获取绝对路径,防止 cd 到临时目录后找不到文件
|
||||
if [[ "$ZIP_FILE_RAW" = /* ]]; then
|
||||
ZIP_FILE="$ZIP_FILE_RAW"
|
||||
else
|
||||
ZIP_FILE="$(pwd)/$ZIP_FILE_RAW"
|
||||
fi
|
||||
|
||||
TARGET_DIR="${INSTALL_BASE_DIR}/edge-admin"
|
||||
BACKUP_DIR="${INSTALL_BASE_DIR}/backup_$(date +%Y%m%d_%H%M%S)"
|
||||
TEMP_DIR="/tmp/edge-admin-update-$(date +%s)"
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# 日志函数
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# ========== 检查前置条件 ==========
|
||||
check_prerequisites() {
|
||||
log_info "检查前置条件..."
|
||||
|
||||
# 检查 zip 文件是否存在
|
||||
if [ ! -f "$ZIP_FILE" ]; then
|
||||
log_error "未找到更新包: $ZIP_FILE"
|
||||
log_info "使用方法: $0 [zip文件路径]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查目标目录是否存在
|
||||
if [ ! -d "$TARGET_DIR" ]; then
|
||||
log_error "目标目录不存在: $TARGET_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查是否有足够的磁盘空间(至少 500MB)
|
||||
available_space=$(df "$TARGET_DIR" | tail -1 | awk '{print $4}')
|
||||
if [ "$available_space" -lt 512000 ]; then
|
||||
log_warn "可用磁盘空间不足 500MB,可能影响部署"
|
||||
fi
|
||||
|
||||
log_info "前置条件检查通过"
|
||||
}
|
||||
|
||||
# ========== 1. 备份关键目录 ==========
|
||||
backup_files() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 1: 备份关键目录"
|
||||
log_info "=========================================="
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# 备份 EdgeAdmin 配置
|
||||
if [ -d "$TARGET_DIR/configs" ]; then
|
||||
log_info "备份 EdgeAdmin 配置文件..."
|
||||
cp -r "$TARGET_DIR/configs" "$BACKUP_DIR/edge-admin-configs"
|
||||
log_info "✅ EdgeAdmin 配置已备份"
|
||||
else
|
||||
log_warn "EdgeAdmin configs 目录不存在,跳过备份"
|
||||
fi
|
||||
|
||||
# 备份 EdgeAdmin 日志(可选)
|
||||
if [ -d "$TARGET_DIR/logs" ]; then
|
||||
log_info "备份 EdgeAdmin 日志文件..."
|
||||
cp -r "$TARGET_DIR/logs" "$BACKUP_DIR/edge-admin-logs" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 备份 EdgeAPI 配置和数据
|
||||
if [ -d "$TARGET_DIR/edge-api" ]; then
|
||||
if [ -d "$TARGET_DIR/edge-api/configs" ]; then
|
||||
log_info "备份 EdgeAPI 配置文件..."
|
||||
cp -r "$TARGET_DIR/edge-api/configs" "$BACKUP_DIR/edge-api-configs"
|
||||
log_info "✅ EdgeAPI 配置已备份"
|
||||
fi
|
||||
|
||||
if [ -d "$TARGET_DIR/edge-api/logs" ]; then
|
||||
log_info "备份 EdgeAPI 日志文件..."
|
||||
cp -r "$TARGET_DIR/edge-api/logs" "$BACKUP_DIR/edge-api-logs" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [ -d "$TARGET_DIR/edge-api/data" ]; then
|
||||
log_info "备份 EdgeAPI 数据文件..."
|
||||
cp -r "$TARGET_DIR/edge-api/data" "$BACKUP_DIR/edge-api-data" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
log_info "✅ 备份完成,备份位置: $BACKUP_DIR"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 2. 停止服务 ==========
|
||||
stop_services() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 2: 停止服务"
|
||||
log_info "=========================================="
|
||||
|
||||
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
|
||||
log_info "停止 EdgeAdmin 服务..."
|
||||
"$TARGET_DIR/bin/edge-admin" stop || true
|
||||
sleep 2
|
||||
|
||||
# 检查是否还有进程在运行 (精确匹配二进制路径,避免杀死解析 zip 路径的脚本自身)
|
||||
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
|
||||
log_warn "检测到 edge-admin 进程仍在运行,尝试强制停止..."
|
||||
pkill -9 -f "$TARGET_DIR/bin/edge-admin" || true
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
log_info "✅ 服务已停止"
|
||||
else
|
||||
log_warn "未找到 edge-admin 可执行文件,跳过停止步骤"
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 3. 解压新版本 ==========
|
||||
extract_package() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 3: 解压新版本"
|
||||
log_info "=========================================="
|
||||
|
||||
log_info "解压更新包到临时目录: $TEMP_DIR"
|
||||
mkdir -p "$TEMP_DIR"
|
||||
cd "$TEMP_DIR"
|
||||
|
||||
if ! unzip -q "$ZIP_FILE"; then
|
||||
log_error "解压失败,请检查 zip 文件是否损坏"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "✅ 解压完成"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 4. 替换文件 ==========
|
||||
replace_files() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 4: 替换文件"
|
||||
log_info "=========================================="
|
||||
|
||||
# 替换 EdgeAdmin bin
|
||||
if [ -d "$TEMP_DIR/edge-admin/bin" ]; then
|
||||
log_info "替换 EdgeAdmin 可执行文件..."
|
||||
cp -r "$TEMP_DIR/edge-admin/bin"/* "$TARGET_DIR/bin/"
|
||||
log_info "✅ EdgeAdmin bin 已更新"
|
||||
fi
|
||||
|
||||
# 替换 EdgeAdmin web(排除 tmp)
|
||||
if [ -d "$TEMP_DIR/edge-admin/web" ]; then
|
||||
log_info "替换 EdgeAdmin 前端文件..."
|
||||
if command -v rsync > /dev/null; then
|
||||
rsync -av --exclude='tmp' \
|
||||
"$TEMP_DIR/edge-admin/web/" "$TARGET_DIR/web/"
|
||||
else
|
||||
# 如果没有 rsync,使用 cp
|
||||
cp -r "$TEMP_DIR/edge-admin/web"/* "$TARGET_DIR/web/" 2>/dev/null || true
|
||||
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
|
||||
fi
|
||||
# 清空 tmp 目录
|
||||
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
|
||||
log_info "✅ EdgeAdmin web 已更新"
|
||||
fi
|
||||
|
||||
# 替换 EdgeAPI 文件
|
||||
if [ -d "$TEMP_DIR/edge-admin/edge-api" ]; then
|
||||
log_info "替换 EdgeAPI 文件..."
|
||||
|
||||
# 确保 edge-api 目录存在
|
||||
mkdir -p "$TARGET_DIR/edge-api"
|
||||
|
||||
# 替换 bin
|
||||
if [ -d "$TEMP_DIR/edge-admin/edge-api/bin" ]; then
|
||||
mkdir -p "$TARGET_DIR/edge-api/bin"
|
||||
cp -r "$TEMP_DIR/edge-admin/edge-api/bin"/* \
|
||||
"$TARGET_DIR/edge-api/bin/" 2>/dev/null || true
|
||||
log_info "✅ EdgeAPI bin 已更新"
|
||||
fi
|
||||
|
||||
# 替换 deploy(节点安装包)
|
||||
if [ -d "$TEMP_DIR/edge-admin/edge-api/deploy" ]; then
|
||||
mkdir -p "$TARGET_DIR/edge-api/deploy"
|
||||
cp -r "$TEMP_DIR/edge-admin/edge-api/deploy"/* \
|
||||
"$TARGET_DIR/edge-api/deploy/" 2>/dev/null || true
|
||||
log_info "✅ EdgeAPI deploy 已更新"
|
||||
fi
|
||||
|
||||
# 替换 installers(安装工具)
|
||||
if [ -d "$TEMP_DIR/edge-admin/edge-api/installers" ]; then
|
||||
mkdir -p "$TARGET_DIR/edge-api/installers"
|
||||
cp -r "$TEMP_DIR/edge-admin/edge-api/installers"/* \
|
||||
"$TARGET_DIR/edge-api/installers/" 2>/dev/null || true
|
||||
log_info "✅ EdgeAPI installers 已更新"
|
||||
fi
|
||||
fi
|
||||
|
||||
log_info "✅ 文件替换完成"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 5. 恢复配置文件 ==========
|
||||
restore_configs() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 5: 恢复配置文件"
|
||||
log_info "=========================================="
|
||||
|
||||
# 恢复 EdgeAdmin 配置
|
||||
if [ -d "$BACKUP_DIR/edge-admin-configs" ]; then
|
||||
log_info "恢复 EdgeAdmin 配置文件..."
|
||||
cp -r "$BACKUP_DIR/edge-admin-configs"/* "$TARGET_DIR/configs/"
|
||||
log_info "✅ EdgeAdmin 配置已恢复"
|
||||
else
|
||||
log_warn "未找到 EdgeAdmin 配置备份,请手动检查配置文件"
|
||||
fi
|
||||
|
||||
# 恢复 EdgeAPI 配置
|
||||
if [ -d "$BACKUP_DIR/edge-api-configs" ]; then
|
||||
log_info "恢复 EdgeAPI 配置文件..."
|
||||
mkdir -p "$TARGET_DIR/edge-api/configs"
|
||||
cp -r "$BACKUP_DIR/edge-api-configs"/* "$TARGET_DIR/edge-api/configs/" 2>/dev/null || true
|
||||
log_info "✅ EdgeAPI 配置已恢复"
|
||||
else
|
||||
log_warn "未找到 EdgeAPI 配置备份,如果存在 edge-api 目录,请手动检查配置文件"
|
||||
fi
|
||||
|
||||
log_info "✅ 配置文件恢复完成"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 6. 清理临时文件 ==========
|
||||
cleanup() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 6: 清理临时文件"
|
||||
log_info "=========================================="
|
||||
|
||||
rm -rf "$TEMP_DIR"
|
||||
log_info "✅ 临时文件已清理"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 7. 启动服务 ==========
|
||||
start_services() {
|
||||
log_info "=========================================="
|
||||
log_info "步骤 7: 启动服务"
|
||||
log_info "=========================================="
|
||||
|
||||
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
|
||||
log_info "启动 EdgeAdmin 服务..."
|
||||
"$TARGET_DIR/bin/edge-admin" start
|
||||
sleep 3
|
||||
|
||||
# 检查服务状态
|
||||
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
|
||||
log_info "✅ 服务启动成功"
|
||||
else
|
||||
log_warn "服务可能未正常启动,请检查日志"
|
||||
log_info "查看日志: tail -f $TARGET_DIR/logs/run.log"
|
||||
fi
|
||||
else
|
||||
log_error "未找到 edge-admin 可执行文件"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ========== 主函数 ==========
|
||||
main() {
|
||||
echo ""
|
||||
log_info "=========================================="
|
||||
log_info "CloudWAF 部署脚本 - Gname DNS 提供商更新"
|
||||
log_info "=========================================="
|
||||
echo ""
|
||||
|
||||
log_info "部署配置:"
|
||||
log_info " ZIP 文件: $ZIP_FILE"
|
||||
log_info " 目标目录: $TARGET_DIR"
|
||||
log_info " 备份目录: $BACKUP_DIR"
|
||||
echo ""
|
||||
|
||||
# 确认操作
|
||||
read -p "确认开始部署? (y/N): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log_info "部署已取消"
|
||||
exit 0
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# 执行部署步骤
|
||||
check_prerequisites
|
||||
backup_files
|
||||
stop_services
|
||||
extract_package
|
||||
replace_files
|
||||
restore_configs
|
||||
cleanup
|
||||
start_services
|
||||
|
||||
# 完成
|
||||
log_info "=========================================="
|
||||
log_info "✅ 部署完成!"
|
||||
log_info "=========================================="
|
||||
echo ""
|
||||
log_info "备份位置: $BACKUP_DIR"
|
||||
log_info "请检查服务运行状态和日志"
|
||||
echo ""
|
||||
log_info "验证步骤:"
|
||||
log_info "1. 检查服务进程: ps aux | grep \"$TARGET_DIR/bin/edge-admin\""
|
||||
log_info "2. 检查服务日志: tail -f $TARGET_DIR/logs/run.log"
|
||||
log_info "3. 访问管理后台,测试 Gname DNS 提供商功能"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# 执行主函数
|
||||
main
|
||||
|
||||
111
deploy/clickhouse/README.md
Normal file
111
deploy/clickhouse/README.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# ClickHouse + Fluent Bit 使用手册(Ubuntu 22.04 / Amazon Linux 2023)
|
||||
|
||||
## 1. 支持范围
|
||||
|
||||
- Ubuntu 22.04
|
||||
- Amazon Linux 2023(AWS)
|
||||
|
||||
安装脚本:`install_clickhouse_linux.sh`(自动识别上述系统)。
|
||||
|
||||
## 2. 安装 ClickHouse
|
||||
|
||||
```bash
|
||||
cd /path/to/waf-platform/deploy/clickhouse
|
||||
chmod +x install_clickhouse_linux.sh
|
||||
sudo ./install_clickhouse_linux.sh
|
||||
```
|
||||
|
||||
可选:安装时初始化 `default` 用户密码:
|
||||
|
||||
```bash
|
||||
sudo CLICKHOUSE_DEFAULT_PASSWORD='YourStrongPassword' ./install_clickhouse_linux.sh
|
||||
```
|
||||
|
||||
## 3. 开启 HTTPS(默认仅 crt+key)
|
||||
|
||||
脚本默认生成 `server.crt + server.key`(带 SAN)并启用 8443:
|
||||
|
||||
```bash
|
||||
cd /path/to/waf-platform/deploy/clickhouse
|
||||
chmod +x configure_clickhouse_https.sh
|
||||
sudo CH_HTTPS_PORT=8443 \
|
||||
CH_CERT_CN=clickhouse.example.com \
|
||||
CH_CERT_DNS=clickhouse.example.com \
|
||||
CH_CERT_IP=<CLICKHOUSE_IP> \
|
||||
./configure_clickhouse_https.sh
|
||||
```
|
||||
|
||||
使用已有证书:
|
||||
|
||||
```bash
|
||||
sudo SRC_CERT=/path/to/server.crt \
|
||||
SRC_KEY=/path/to/server.key \
|
||||
CH_HTTPS_PORT=8443 \
|
||||
./configure_clickhouse_https.sh
|
||||
```
|
||||
|
||||
## 4. 初始化日志表(含优化)
|
||||
|
||||
```bash
|
||||
cd /path/to/waf-platform/deploy/clickhouse
|
||||
chmod +x init_waf_logs_tables.sh
|
||||
sudo CH_HOST=127.0.0.1 \
|
||||
CH_PORT=9000 \
|
||||
CH_USER=default \
|
||||
CH_PASSWORD='YourStrongPassword' \
|
||||
CH_DATABASE=default \
|
||||
./init_waf_logs_tables.sh
|
||||
```
|
||||
|
||||
说明:
|
||||
- `init_waf_logs_tables.sql` 已内置主要优化(`CODEC`、`LowCardinality`、跳数索引)。
|
||||
- `optimize_schema.sql` 主要用于历史表补齐优化,不是首次建表必需步骤。
|
||||
|
||||
## 5. 平台侧配置(EdgeAdmin)
|
||||
|
||||
在 ClickHouse 设置页配置:
|
||||
|
||||
- Host:ClickHouse 地址
|
||||
- Port:`8443`
|
||||
- Database:`default`
|
||||
- Scheme:`https`
|
||||
|
||||
当前实现说明:
|
||||
- 前端不再提供 `TLS跳过校验` 和 `TLS Server Name` 配置项。
|
||||
- 后端固定 `TLSSkipVerify=true`(默认不校验证书)。
|
||||
|
||||
保存后点击“测试连接”。
|
||||
|
||||
## 6. Fluent Bit 配置方式
|
||||
|
||||
推荐平台托管模式(在线安装/升级 Node、DNS 时自动下发):
|
||||
|
||||
- `/etc/fluent-bit/fluent-bit.conf`
|
||||
- `/etc/fluent-bit/.edge-managed.env`
|
||||
- `/etc/fluent-bit/.edge-managed.json`
|
||||
|
||||
检查状态:
|
||||
|
||||
```bash
|
||||
sudo systemctl status fluent-bit --no-pager
|
||||
sudo cat /etc/fluent-bit/.edge-managed.json
|
||||
```
|
||||
|
||||
## 7. 验证与排障
|
||||
|
||||
查看 Fluent Bit 日志:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u fluent-bit -f
|
||||
```
|
||||
|
||||
查看写入:
|
||||
|
||||
```sql
|
||||
SELECT count() FROM default.logs_ingest;
|
||||
SELECT count() FROM default.dns_logs_ingest;
|
||||
```
|
||||
|
||||
常见错误:
|
||||
- `connection refused`:8443 未监听或网络未放行。
|
||||
- `legacy Common Name`:证书缺 SAN,需重签。
|
||||
227
deploy/clickhouse/configure_clickhouse_https.sh
Normal file
227
deploy/clickhouse/configure_clickhouse_https.sh
Normal file
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "[ERROR] please run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f /etc/os-release ]]; then
|
||||
echo "[ERROR] /etc/os-release not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /etc/os-release
|
||||
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
os_ver="${VERSION_ID:-}"
|
||||
is_ubuntu22=false
|
||||
is_amzn2023=false
|
||||
|
||||
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
|
||||
is_ubuntu22=true
|
||||
fi
|
||||
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
|
||||
is_amzn2023=true
|
||||
fi
|
||||
|
||||
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
|
||||
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v openssl >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then
|
||||
if [[ "${is_ubuntu22}" == "true" ]]; then
|
||||
apt-get update -y
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y openssl curl ca-certificates
|
||||
else
|
||||
dnf makecache -y
|
||||
dnf install -y openssl curl ca-certificates
|
||||
fi
|
||||
fi
|
||||
|
||||
CH_HTTPS_PORT="${CH_HTTPS_PORT:-8443}"
|
||||
CH_LISTEN_HOST="${CH_LISTEN_HOST:-::}"
|
||||
CH_CERT_CN="${CH_CERT_CN:-$(hostname -f 2>/dev/null || hostname)}"
|
||||
CH_CERT_DNS="${CH_CERT_DNS:-}"
|
||||
CH_CERT_IP="${CH_CERT_IP:-}"
|
||||
CH_CERT_DAYS="${CH_CERT_DAYS:-825}"
|
||||
CH_GENERATE_CA="${CH_GENERATE_CA:-false}"
|
||||
|
||||
SRC_CERT="${SRC_CERT:-}"
|
||||
SRC_KEY="${SRC_KEY:-}"
|
||||
SRC_CA="${SRC_CA:-}"
|
||||
|
||||
CH_DIR="/etc/clickhouse-server"
|
||||
CH_CONFIG_D_DIR="${CH_DIR}/config.d"
|
||||
PKI_DIR="${CH_DIR}/pki"
|
||||
SERVER_CERT="${CH_DIR}/server.crt"
|
||||
SERVER_KEY="${CH_DIR}/server.key"
|
||||
CA_CERT="${CH_DIR}/ca.crt"
|
||||
OVERRIDE_FILE="${CH_CONFIG_D_DIR}/waf-https.xml"
|
||||
|
||||
mkdir -p "${CH_CONFIG_D_DIR}" "${PKI_DIR}"
|
||||
|
||||
split_csv() {
|
||||
local raw="$1"
|
||||
if [[ -z "${raw}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
IFS=',' read -r -a arr <<<"${raw}"
|
||||
for item in "${arr[@]}"; do
|
||||
item="$(echo "${item}" | xargs)"
|
||||
if [[ -n "${item}" ]]; then
|
||||
echo "${item}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
build_san_line() {
|
||||
local san_entries=()
|
||||
while IFS= read -r dns_item; do
|
||||
san_entries+=("DNS:${dns_item}")
|
||||
done < <(split_csv "${CH_CERT_DNS}")
|
||||
while IFS= read -r ip_item; do
|
||||
san_entries+=("IP:${ip_item}")
|
||||
done < <(split_csv "${CH_CERT_IP}")
|
||||
|
||||
if [[ ${#san_entries[@]} -eq 0 ]]; then
|
||||
san_entries+=("DNS:${CH_CERT_CN}")
|
||||
fi
|
||||
|
||||
local san_line
|
||||
san_line="$(IFS=,; echo "${san_entries[*]}")"
|
||||
echo "${san_line}"
|
||||
}
|
||||
|
||||
generate_self_signed_cert() {
|
||||
echo "[INFO] generating self-signed server certificate (crt+key only) ..."
|
||||
local server_key="${PKI_DIR}/server.key"
|
||||
local server_csr="${PKI_DIR}/server.csr"
|
||||
local server_crt="${PKI_DIR}/server.crt"
|
||||
local ext_file="${PKI_DIR}/server.ext"
|
||||
local san_line
|
||||
san_line="$(build_san_line)"
|
||||
|
||||
openssl genrsa -out "${server_key}" 2048
|
||||
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
|
||||
|
||||
cat >"${ext_file}" <<EOF
|
||||
subjectAltName=${san_line}
|
||||
keyUsage=digitalSignature,keyEncipherment
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
|
||||
openssl x509 -req -in "${server_csr}" -signkey "${server_key}" \
|
||||
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
|
||||
|
||||
cp -f "${server_crt}" "${SERVER_CERT}"
|
||||
cp -f "${server_key}" "${SERVER_KEY}"
|
||||
rm -f "${CA_CERT}"
|
||||
}
|
||||
|
||||
generate_cert_with_ca() {
|
||||
echo "[INFO] generating local CA and server certificate ..."
|
||||
local ca_key="${PKI_DIR}/ca.key"
|
||||
local ca_crt="${PKI_DIR}/ca.crt"
|
||||
local server_key="${PKI_DIR}/server.key"
|
||||
local server_csr="${PKI_DIR}/server.csr"
|
||||
local server_crt="${PKI_DIR}/server.crt"
|
||||
local ext_file="${PKI_DIR}/server.ext"
|
||||
local san_line
|
||||
san_line="$(build_san_line)"
|
||||
|
||||
openssl genrsa -out "${ca_key}" 4096
|
||||
openssl req -x509 -new -nodes -key "${ca_key}" -sha256 -days 3650 \
|
||||
-out "${ca_crt}" -subj "/CN=ClickHouse Local CA"
|
||||
|
||||
openssl genrsa -out "${server_key}" 2048
|
||||
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
|
||||
|
||||
cat >"${ext_file}" <<EOF
|
||||
subjectAltName=${san_line}
|
||||
keyUsage=digitalSignature,keyEncipherment
|
||||
extendedKeyUsage=serverAuth
|
||||
EOF
|
||||
|
||||
openssl x509 -req -in "${server_csr}" -CA "${ca_crt}" -CAkey "${ca_key}" -CAcreateserial \
|
||||
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
|
||||
|
||||
cp -f "${server_crt}" "${SERVER_CERT}"
|
||||
cp -f "${server_key}" "${SERVER_KEY}"
|
||||
cp -f "${ca_crt}" "${CA_CERT}"
|
||||
}
|
||||
|
||||
if [[ -n "${SRC_CERT}" || -n "${SRC_KEY}" ]]; then
|
||||
if [[ -z "${SRC_CERT}" || -z "${SRC_KEY}" ]]; then
|
||||
echo "[ERROR] SRC_CERT and SRC_KEY must be provided together"
|
||||
exit 1
|
||||
fi
|
||||
echo "[INFO] using provided certificate files ..."
|
||||
cp -f "${SRC_CERT}" "${SERVER_CERT}"
|
||||
cp -f "${SRC_KEY}" "${SERVER_KEY}"
|
||||
if [[ -n "${SRC_CA}" ]]; then
|
||||
cp -f "${SRC_CA}" "${CA_CERT}"
|
||||
else
|
||||
rm -f "${CA_CERT}"
|
||||
fi
|
||||
else
|
||||
case "$(echo "${CH_GENERATE_CA}" | tr '[:upper:]' '[:lower:]')" in
|
||||
1|true|yes|on)
|
||||
generate_cert_with_ca
|
||||
;;
|
||||
*)
|
||||
generate_self_signed_cert
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
chown clickhouse:clickhouse "${SERVER_CERT}" "${SERVER_KEY}" || true
|
||||
chmod 0644 "${SERVER_CERT}"
|
||||
chmod 0640 "${SERVER_KEY}"
|
||||
if [[ -f "${CA_CERT}" ]]; then
|
||||
chown clickhouse:clickhouse "${CA_CERT}" || true
|
||||
chmod 0644 "${CA_CERT}"
|
||||
fi
|
||||
|
||||
echo "[INFO] writing ClickHouse HTTPS override config ..."
|
||||
cat >"${OVERRIDE_FILE}" <<EOF
|
||||
<clickhouse>
|
||||
<https_port>${CH_HTTPS_PORT}</https_port>
|
||||
<listen_host>${CH_LISTEN_HOST}</listen_host>
|
||||
<openSSL>
|
||||
<server>
|
||||
<certificateFile>${SERVER_CERT}</certificateFile>
|
||||
<privateKeyFile>${SERVER_KEY}</privateKeyFile>
|
||||
<verificationMode>none</verificationMode>
|
||||
<loadDefaultCAFile>true</loadDefaultCAFile>
|
||||
<cacheSessions>true</cacheSessions>
|
||||
<disableProtocols>sslv2,sslv3</disableProtocols>
|
||||
<preferServerCiphers>true</preferServerCiphers>
|
||||
<invalidCertificateHandler>
|
||||
<name>RejectCertificateHandler</name>
|
||||
</invalidCertificateHandler>
|
||||
</server>
|
||||
</openSSL>
|
||||
</clickhouse>
|
||||
EOF
|
||||
|
||||
echo "[INFO] restarting clickhouse-server ..."
|
||||
systemctl restart clickhouse-server
|
||||
sleep 2
|
||||
|
||||
echo "[INFO] service status ..."
|
||||
systemctl --no-pager -l status clickhouse-server | sed -n '1,15p'
|
||||
|
||||
echo "[INFO] verifying HTTPS endpoint ..."
|
||||
curl -sk "https://127.0.0.1:${CH_HTTPS_PORT}/?query=SELECT%201" || true
|
||||
echo
|
||||
|
||||
echo "[OK] ClickHouse HTTPS setup finished"
|
||||
echo " HTTPS port : ${CH_HTTPS_PORT}"
|
||||
echo " cert file : ${SERVER_CERT}"
|
||||
echo " key file : ${SERVER_KEY}"
|
||||
if [[ -f "${CA_CERT}" ]]; then
|
||||
echo " CA file : ${CA_CERT}"
|
||||
echo " import this CA file into API/Fluent Bit hosts if tls.verify=On"
|
||||
fi
|
||||
38
deploy/clickhouse/init_waf_logs_tables.sh
Normal file
38
deploy/clickhouse/init_waf_logs_tables.sh
Normal file
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SQL_FILE="${SCRIPT_DIR}/init_waf_logs_tables.sql"
|
||||
|
||||
if [[ ! -f "${SQL_FILE}" ]]; then
|
||||
echo "[ERROR] SQL file not found: ${SQL_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v clickhouse-client >/dev/null 2>&1; then
|
||||
echo "[ERROR] clickhouse-client not found. Please install ClickHouse client first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CH_HOST="${CH_HOST:-127.0.0.1}"
|
||||
CH_PORT="${CH_PORT:-9000}"
|
||||
CH_USER="${CH_USER:-default}"
|
||||
CH_PASSWORD="${CH_PASSWORD:-}"
|
||||
CH_DATABASE="${CH_DATABASE:-default}"
|
||||
|
||||
args=(--host "${CH_HOST}" --port "${CH_PORT}" --user "${CH_USER}")
|
||||
if [[ -n "${CH_PASSWORD}" ]]; then
|
||||
args+=(--password "${CH_PASSWORD}")
|
||||
fi
|
||||
|
||||
echo "[INFO] creating database if not exists: ${CH_DATABASE}"
|
||||
clickhouse-client "${args[@]}" --query "CREATE DATABASE IF NOT EXISTS ${CH_DATABASE}"
|
||||
|
||||
echo "[INFO] initializing tables in database: ${CH_DATABASE}"
|
||||
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" < "${SQL_FILE}"
|
||||
|
||||
echo "[INFO] checking table status ..."
|
||||
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" --query \
|
||||
"SELECT name, engine FROM system.tables WHERE database='${CH_DATABASE}' AND name IN ('logs_ingest','dns_logs_ingest') ORDER BY name"
|
||||
|
||||
echo "[OK] ClickHouse ingest tables are ready in database '${CH_DATABASE}'"
|
||||
69
deploy/clickhouse/init_waf_logs_tables.sql
Normal file
69
deploy/clickhouse/init_waf_logs_tables.sql
Normal file
@@ -0,0 +1,69 @@
|
||||
-- Initialize HTTP and DNS ingest tables for GoEdge access logs.
|
||||
-- Run with:
|
||||
-- clickhouse-client --database <db_name> < init_waf_logs_tables.sql
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs_ingest
|
||||
(
|
||||
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
|
||||
node_id UInt64,
|
||||
cluster_id UInt64,
|
||||
server_id UInt64,
|
||||
host LowCardinality(String),
|
||||
ip String,
|
||||
method LowCardinality(String),
|
||||
path String CODEC(ZSTD(1)),
|
||||
status UInt16,
|
||||
bytes_in UInt64 CODEC(Delta, ZSTD(1)),
|
||||
bytes_out UInt64 CODEC(Delta, ZSTD(1)),
|
||||
cost_ms UInt32 CODEC(Delta, ZSTD(1)),
|
||||
ua String CODEC(ZSTD(1)),
|
||||
referer String CODEC(ZSTD(1)),
|
||||
log_type LowCardinality(String),
|
||||
trace_id String,
|
||||
firewall_policy_id UInt64 DEFAULT 0,
|
||||
firewall_rule_group_id UInt64 DEFAULT 0,
|
||||
firewall_rule_set_id UInt64 DEFAULT 0,
|
||||
firewall_rule_id UInt64 DEFAULT 0,
|
||||
request_headers String CODEC(ZSTD(3)) DEFAULT '',
|
||||
request_body String CODEC(ZSTD(3)) DEFAULT '',
|
||||
response_headers String CODEC(ZSTD(3)) DEFAULT '',
|
||||
response_body String CODEC(ZSTD(3)) DEFAULT '',
|
||||
INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
|
||||
INDEX idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4,
|
||||
INDEX idx_status status TYPE minmax GRANULARITY 4
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(timestamp)
|
||||
ORDER BY (timestamp, node_id, server_id, trace_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dns_logs_ingest
|
||||
(
|
||||
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
|
||||
request_id String,
|
||||
node_id UInt64,
|
||||
cluster_id UInt64,
|
||||
domain_id UInt64,
|
||||
record_id UInt64,
|
||||
remote_addr String,
|
||||
question_name String,
|
||||
question_type LowCardinality(String),
|
||||
record_name String,
|
||||
record_type LowCardinality(String),
|
||||
record_value String,
|
||||
networking LowCardinality(String),
|
||||
is_recursive UInt8,
|
||||
error String CODEC(ZSTD(1)),
|
||||
ns_route_codes Array(String),
|
||||
content_json String CODEC(ZSTD(3)) DEFAULT '',
|
||||
INDEX idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||
INDEX idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
|
||||
INDEX idx_domain_id domain_id TYPE minmax GRANULARITY 4
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMMDD(timestamp)
|
||||
ORDER BY (timestamp, request_id, node_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
95
deploy/clickhouse/install_clickhouse_linux.sh
Normal file
95
deploy/clickhouse/install_clickhouse_linux.sh
Normal file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "[ERROR] please run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f /etc/os-release ]]; then
|
||||
echo "[ERROR] /etc/os-release not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source /etc/os-release
|
||||
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
os_ver="${VERSION_ID:-}"
|
||||
is_ubuntu22=false
|
||||
is_amzn2023=false
|
||||
|
||||
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
|
||||
is_ubuntu22=true
|
||||
fi
|
||||
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
|
||||
is_amzn2023=true
|
||||
fi
|
||||
|
||||
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
|
||||
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${is_ubuntu22}" == "true" ]]; then
|
||||
echo "[INFO] detected Ubuntu 22.04"
|
||||
echo "[INFO] installing prerequisites ..."
|
||||
apt-get update -y
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates gnupg apt-transport-https lsb-release
|
||||
|
||||
echo "[INFO] configuring ClickHouse apt repository ..."
|
||||
install -d -m 0755 /etc/apt/keyrings
|
||||
if [[ ! -f /etc/apt/keyrings/clickhouse.gpg ]]; then
|
||||
curl -fsSL https://packages.clickhouse.com/CLICKHOUSE-KEY.GPG | gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg
|
||||
fi
|
||||
|
||||
cat >/etc/apt/sources.list.d/clickhouse.list <<'EOF'
|
||||
deb [signed-by=/etc/apt/keyrings/clickhouse.gpg arch=amd64,arm64] https://packages.clickhouse.com/deb stable main
|
||||
EOF
|
||||
|
||||
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
|
||||
apt-get update -y
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y clickhouse-server clickhouse-client clickhouse-common-static
|
||||
fi
|
||||
|
||||
if [[ "${is_amzn2023}" == "true" ]]; then
|
||||
echo "[INFO] detected Amazon Linux 2023"
|
||||
echo "[INFO] installing prerequisites ..."
|
||||
dnf makecache -y
|
||||
dnf install -y curl ca-certificates gnupg2 dnf-plugins-core
|
||||
|
||||
echo "[INFO] configuring ClickHouse yum repository ..."
|
||||
cat >/etc/yum.repos.d/clickhouse.repo <<'EOF'
|
||||
[clickhouse-stable]
|
||||
name=ClickHouse Stable Repository
|
||||
baseurl=https://packages.clickhouse.com/rpm/stable/$basearch
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
gpgkey=https://packages.clickhouse.com/rpm/stable/repodata/repomd.xml.key
|
||||
https://packages.clickhouse.com/rpm/clickhouse-static.key
|
||||
EOF
|
||||
|
||||
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
|
||||
dnf clean all
|
||||
dnf makecache -y
|
||||
if ! dnf install -y clickhouse-server clickhouse-client clickhouse-common-static; then
|
||||
dnf install -y clickhouse-server clickhouse-client
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] enabling clickhouse-server ..."
|
||||
systemctl enable clickhouse-server >/dev/null 2>&1 || true
|
||||
systemctl restart clickhouse-server
|
||||
sleep 2
|
||||
|
||||
if [[ -n "${CLICKHOUSE_DEFAULT_PASSWORD:-}" ]]; then
|
||||
echo "[INFO] setting default user password ..."
|
||||
if [[ "${CLICKHOUSE_DEFAULT_PASSWORD}" == *"'"* ]]; then
|
||||
echo "[ERROR] CLICKHOUSE_DEFAULT_PASSWORD contains single quote, please set password manually with clickhouse-client"
|
||||
exit 1
|
||||
fi
|
||||
clickhouse-client --query "ALTER USER default IDENTIFIED WITH plaintext_password BY '${CLICKHOUSE_DEFAULT_PASSWORD}'"
|
||||
fi
|
||||
|
||||
echo "[INFO] health check ..."
|
||||
clickhouse-client --query "SELECT version()"
|
||||
echo "[OK] ClickHouse install completed: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||
123
deploy/clickhouse/optimize_schema.sql
Normal file
123
deploy/clickhouse/optimize_schema.sql
Normal file
@@ -0,0 +1,123 @@
|
||||
-- =============================================================================
|
||||
-- ClickHouse logs_ingest 表优化脚本
|
||||
--
|
||||
-- 说明:
|
||||
-- - 所有 ALTER 操作均为在线操作,无需停服
|
||||
-- - 建议按阶段顺序执行,每阶段执行后观察 system.parts 确认生效
|
||||
-- - 压缩编解码器变更仅影响新写入的 part,存量数据需等 merge 或手动 OPTIMIZE
|
||||
--
|
||||
-- 执行方式:
|
||||
-- clickhouse-client --host 127.0.0.1 --port 9000 --user default --password 'xxx' < optimize_schema.sql
|
||||
-- =============================================================================
|
||||
|
||||
-- =============================================
|
||||
-- 阶段 1:大字段压缩优化(效果最显著)
|
||||
-- =============================================
|
||||
|
||||
-- 大文本字段改用 ZSTD(3),对 JSON / HTTP 文本压缩率远优于默认 LZ4
|
||||
-- 预期效果:磁盘占用减少 40%-60%
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN request_headers String CODEC(ZSTD(3));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN request_body String CODEC(ZSTD(3));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN response_headers String CODEC(ZSTD(3));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN response_body String CODEC(ZSTD(3));
|
||||
|
||||
-- 中等长度文本字段用 ZSTD(1),平衡压缩率与 CPU 开销
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN ua String CODEC(ZSTD(1));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN path String CODEC(ZSTD(1));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN referer String CODEC(ZSTD(1));
|
||||
|
||||
-- 低基数字段改用 LowCardinality(内存+磁盘双降)
|
||||
-- method 的基数极低(GET/POST/PUT/DELETE 等),host 基数取决于站点数量
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN method LowCardinality(String);
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN log_type LowCardinality(String);
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN host LowCardinality(String);
|
||||
|
||||
-- 数值字段使用 Delta + ZSTD 编码(利用相邻行的时间/大小相关性)
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN bytes_in UInt64 CODEC(Delta, ZSTD(1));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN bytes_out UInt64 CODEC(Delta, ZSTD(1));
|
||||
ALTER TABLE logs_ingest MODIFY COLUMN cost_ms UInt32 CODEC(Delta, ZSTD(1));
|
||||
|
||||
-- =============================================
|
||||
-- 阶段 2:添加 Skipping Index(加速高频过滤查询)
|
||||
-- =============================================
|
||||
|
||||
-- trace_id 精确查找(查看日志详情 FindByTraceId)
|
||||
-- bloom_filter(0.01) = 1% 误判率,GRANULARITY 4 = 每 4 个 granule 一个 bloom block
|
||||
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||
|
||||
-- IP 精确查找
|
||||
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||
|
||||
-- host 模糊查询支持(tokenbf_v1 对 LIKE '%xxx%' 有效)
|
||||
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
|
||||
|
||||
-- firewall_policy_id 过滤(HasFirewallPolicy: > 0)
|
||||
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4;
|
||||
|
||||
-- status 范围过滤(HasError: status >= 400)
|
||||
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_status status TYPE minmax GRANULARITY 4;
|
||||
|
||||
-- =============================================
|
||||
-- 阶段 3:物化索引到现有数据(对存量数据生效)
|
||||
-- =============================================
|
||||
-- 注意:MATERIALIZE INDEX 会触发后台 mutation,大表可能需要一定时间
|
||||
-- 可通过 SELECT * FROM system.mutations WHERE is_done = 0 监控进度
|
||||
|
||||
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_trace_id;
|
||||
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_ip;
|
||||
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_host;
|
||||
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_fw_policy;
|
||||
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_status;
|
||||
|
||||
|
||||
-- =============================================================================
|
||||
-- dns_logs_ingest 表优化(DNS 日志表)
|
||||
-- =============================================================================
|
||||
|
||||
-- 大文本字段压缩
|
||||
ALTER TABLE dns_logs_ingest MODIFY COLUMN content_json String CODEC(ZSTD(3));
|
||||
ALTER TABLE dns_logs_ingest MODIFY COLUMN error String CODEC(ZSTD(1));
|
||||
|
||||
-- 低基数字段
|
||||
ALTER TABLE dns_logs_ingest MODIFY COLUMN question_type LowCardinality(String);
|
||||
ALTER TABLE dns_logs_ingest MODIFY COLUMN record_type LowCardinality(String);
|
||||
ALTER TABLE dns_logs_ingest MODIFY COLUMN networking LowCardinality(String);
|
||||
|
||||
-- request_id 精确查找
|
||||
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||
|
||||
-- remote_addr 精确查找
|
||||
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||
|
||||
-- question_name 模糊查询
|
||||
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
|
||||
|
||||
-- domain_id 过滤
|
||||
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_domain_id domain_id TYPE minmax GRANULARITY 4;
|
||||
|
||||
-- 物化索引到现有数据
|
||||
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_request_id;
|
||||
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_remote_addr;
|
||||
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_question_name;
|
||||
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_domain_id;
|
||||
|
||||
|
||||
-- =============================================================================
|
||||
-- 验证命令(执行完上述 ALTER 后运行)
|
||||
-- =============================================================================
|
||||
|
||||
-- 查看列的压缩编解码器
|
||||
-- SELECT name, type, compression_codec FROM system.columns WHERE table = 'logs_ingest' AND database = currentDatabase();
|
||||
|
||||
-- 查看表的压缩率
|
||||
-- SELECT table, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed, round(sum(data_uncompressed_bytes) / sum(data_compressed_bytes), 2) AS ratio FROM system.columns WHERE table IN ('logs_ingest', 'dns_logs_ingest') GROUP BY table;
|
||||
|
||||
-- 查看各列占用的磁盘空间(找出最大的列)
|
||||
-- SELECT name, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed FROM system.columns WHERE table = 'logs_ingest' GROUP BY name ORDER BY sum(data_compressed_bytes) DESC;
|
||||
|
||||
-- 查看 mutation 进度
|
||||
-- SELECT database, table, mutation_id, command, is_done, parts_to_do FROM system.mutations WHERE is_done = 0;
|
||||
|
||||
-- 强制触发 merge(可选,让压缩编解码器变更对存量数据生效)
|
||||
-- OPTIMIZE TABLE logs_ingest FINAL;
|
||||
-- OPTIMIZE TABLE dns_logs_ingest FINAL;
|
||||
@@ -21,7 +21,14 @@
|
||||
- **边缘节点(EdgeNode)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`(JSON Lines)。
|
||||
- **DNS 节点(EdgeDNS)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines)。
|
||||
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest` 表(见下文「五、ClickHouse 建表」)。
|
||||
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123)。
|
||||
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTPS 端口 8443)。
|
||||
- 日志轮转默认由 Node/DNS 内建 `lumberjack` 执行:
|
||||
- `maxSizeMB=256`
|
||||
- `maxBackups=14`
|
||||
- `maxAgeDays=7`
|
||||
- `compress=false`
|
||||
- `localTime=true`
|
||||
可通过公用日志策略 `file.rotate` 调整。
|
||||
|
||||
---
|
||||
|
||||
@@ -71,7 +78,7 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
|
||||
|
||||
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port:
|
||||
|
||||
- **单机**:保留一个 `[NODE]`,改 `Host`、`Port`(默认 8123)。
|
||||
- **单机**:保留一个 `[NODE]`,改 `Host`、`Port`(默认 8443)。
|
||||
- **集群**:复制多段 `[NODE]`,每段一个节点,例如:
|
||||
|
||||
```ini
|
||||
@@ -81,12 +88,12 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
|
||||
[NODE]
|
||||
Name node-01
|
||||
Host 192.168.1.10
|
||||
Port 8123
|
||||
Port 8443
|
||||
|
||||
[NODE]
|
||||
Name node-02
|
||||
Host 192.168.1.11
|
||||
Port 8123
|
||||
Port 8443
|
||||
```
|
||||
|
||||
### 3.3 ClickHouse 账号密码(有密码时必做)
|
||||
@@ -319,9 +326,9 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
|
||||
|
||||
| 组件 | 说明 |
|
||||
|------|------|
|
||||
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
|
||||
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;内建 lumberjack 轮转(默认 256MB/14份/7天,可按策略调整),仍支持 SIGHUP 重建 writer。 |
|
||||
| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines),由 Fluent Bit 采集写入 `dns_logs_ingest`。 |
|
||||
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 |
|
||||
| **logrotate** | 可选的历史兼容方案(已非必需);默认建议使用节点内建 lumberjack 轮转。 |
|
||||
| **平台(EdgeAPI)** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
|
||||
|
||||
---
|
||||
@@ -411,3 +418,54 @@ sudo systemctl restart fluent-bit
|
||||
```
|
||||
|
||||
回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。
|
||||
|
||||
---
|
||||
|
||||
## 十、平台托管模式(推荐)
|
||||
|
||||
从 `v1.4.7` 开始,Node/DNS 在线安装流程会由平台托管 Fluent Bit,默认不再要求逐台手改 `/etc/fluent-bit/fluent-bit.conf`。
|
||||
|
||||
### 10.1 托管行为
|
||||
|
||||
- 安装器优先使用发布包内置离线包(不走 `curl | sh`)。
|
||||
- 首次安装后写入:
|
||||
- `/etc/fluent-bit/fluent-bit.conf`
|
||||
- `/etc/fluent-bit/parsers.conf`
|
||||
- `/etc/fluent-bit/.edge-managed.env`
|
||||
- `/etc/fluent-bit/.edge-managed.json`
|
||||
- 配置发生变化时按 `hash` 幂等更新,仅在内容变化时重启服务。
|
||||
- Node 与 DNS 同机安装时会自动合并角色,输出单份配置。
|
||||
|
||||
### 10.2 托管元数据
|
||||
|
||||
平台会维护 `/etc/fluent-bit/.edge-managed.json`,核心字段:
|
||||
|
||||
- `roles`: 当前机器启用角色(`node`/`dns`)
|
||||
- `hash`: 当前托管配置摘要
|
||||
- `sourceVersion`: 平台版本号
|
||||
- `updatedAt`: 最近更新时间戳
|
||||
|
||||
### 10.3 支持矩阵(离线包)
|
||||
|
||||
当前固定支持以下平台键:
|
||||
|
||||
- `ubuntu22.04-amd64`
|
||||
- `ubuntu22.04-arm64`
|
||||
- `amzn2023-amd64`
|
||||
- `amzn2023-arm64`
|
||||
|
||||
构建阶段会校验矩阵包是否齐全,缺失会直接失败并打印期望文件路径。
|
||||
|
||||
### 10.4 手工配置兼容
|
||||
|
||||
- 若现有 `fluent-bit.conf` 不是平台托管文件(不含 `managed-by-edgeapi` 标记),安装器不会强制覆盖,会返回明确错误提示。
|
||||
- 需要切到托管模式时,先备份旧配置,再由平台触发一次安装/更新任务。
|
||||
|
||||
### 10.5 Resource Profile Notes (New)
|
||||
|
||||
- Managed default is now tuned for `2C4G` nodes (conservative and stable).
|
||||
- Additional sample profiles are provided for larger nodes:
|
||||
- `deploy/fluent-bit/fluent-bit-sample-4c8g.conf`
|
||||
- `deploy/fluent-bit/fluent-bit-sample-8c16g.conf`
|
||||
- These sample files are for benchmark/reference only and are not auto-applied by installer.
|
||||
- To use higher profiles in managed mode, sync those parameters into `EdgeAPI/internal/installers/fluent_bit.go` and then trigger node reinstall/upgrade.
|
||||
|
||||
@@ -8,4 +8,4 @@
|
||||
[NODE]
|
||||
Name node-01
|
||||
Host 127.0.0.1
|
||||
Port 8123
|
||||
Port 8443
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# DNS 节点专用:使用 HTTP 输出写入 ClickHouse(无需 out_clickhouse 插件)
|
||||
# DNS 节点专用:使用 HTTPS 输出写入 ClickHouse(无需 out_clickhouse 插件)
|
||||
# 启动前设置:CH_USER、CH_PASSWORD;若 ClickHouse 不在本机,请修改 Host、Port
|
||||
# Read_from_Head=true:首次启动会发送已有日志;若只采新日志建议改为 false
|
||||
|
||||
@@ -26,11 +26,15 @@
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8123
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
tls On
|
||||
tls.verify On
|
||||
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||
# tls.vhost clickhouse.example.com
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
|
||||
69
deploy/fluent-bit/fluent-bit-sample-4c8g.conf
Normal file
69
deploy/fluent-bit/fluent-bit-sample-4c8g.conf
Normal file
@@ -0,0 +1,69 @@
|
||||
# Sample profile for 4C8G nodes (Node + DNS on same host).
|
||||
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
|
||||
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 512MB
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-node/*.log
|
||||
Tag app.http.logs
|
||||
Parser json
|
||||
Refresh_Interval 2
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/http-logs.db
|
||||
storage.type filesystem
|
||||
Mem_Buf_Limit 256MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Parser json
|
||||
Refresh_Interval 2
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
storage.type filesystem
|
||||
Mem_Buf_Limit 256MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.http.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
workers 2
|
||||
net.keepalive On
|
||||
Retry_Limit False
|
||||
tls On
|
||||
tls.verify On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
workers 2
|
||||
net.keepalive On
|
||||
Retry_Limit False
|
||||
tls On
|
||||
tls.verify On
|
||||
69
deploy/fluent-bit/fluent-bit-sample-8c16g.conf
Normal file
69
deploy/fluent-bit/fluent-bit-sample-8c16g.conf
Normal file
@@ -0,0 +1,69 @@
|
||||
# Sample profile for 8C16G nodes (Node + DNS on same host).
|
||||
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
|
||||
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 1024MB
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-node/*.log
|
||||
Tag app.http.logs
|
||||
Parser json
|
||||
Refresh_Interval 1
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/http-logs.db
|
||||
storage.type filesystem
|
||||
Mem_Buf_Limit 512MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Parser json
|
||||
Refresh_Interval 1
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
storage.type filesystem
|
||||
Mem_Buf_Limit 512MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.http.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
workers 4
|
||||
net.keepalive On
|
||||
Retry_Limit False
|
||||
tls On
|
||||
tls.verify On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
workers 4
|
||||
net.keepalive On
|
||||
Retry_Limit False
|
||||
tls On
|
||||
tls.verify On
|
||||
Binary file not shown.
Binary file not shown.
@@ -47,6 +47,32 @@ flowchart TD
|
||||
- `/var/log/edge/edge-node/*.log`
|
||||
3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。
|
||||
|
||||
### 3.3 本地日志轮转(默认开启)
|
||||
从当前版本开始,EdgeNode / EdgeDNS 使用内建 `lumberjack` 轮转,不再依赖系统 `logrotate`。
|
||||
|
||||
默认值:
|
||||
- `maxSizeMB=256`
|
||||
- `maxBackups=14`
|
||||
- `maxAgeDays=7`
|
||||
- `compress=false`
|
||||
- `localTime=true`
|
||||
|
||||
可在策略 `file.rotate` 中配置,例如:
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "/var/log/web-access-${date}.log",
|
||||
"autoCreate": true,
|
||||
"rotate": {
|
||||
"maxSizeMB": 256,
|
||||
"maxBackups": 14,
|
||||
"maxAgeDays": 7,
|
||||
"compress": false,
|
||||
"localTime": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 三种目标模式怎么配
|
||||
|
||||
Reference in New Issue
Block a user