引入lumberjack和fluentbit自动分发
This commit is contained in:
@@ -18,11 +18,9 @@ const (
|
|||||||
envPassword = "CLICKHOUSE_PASSWORD"
|
envPassword = "CLICKHOUSE_PASSWORD"
|
||||||
envDatabase = "CLICKHOUSE_DATABASE"
|
envDatabase = "CLICKHOUSE_DATABASE"
|
||||||
envScheme = "CLICKHOUSE_SCHEME"
|
envScheme = "CLICKHOUSE_SCHEME"
|
||||||
envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY"
|
defaultPort = 8443
|
||||||
envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME"
|
|
||||||
defaultPort = 8123
|
|
||||||
defaultDB = "default"
|
defaultDB = "default"
|
||||||
defaultScheme = "http"
|
defaultScheme = "https"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -62,7 +60,7 @@ func ResetSharedConfig() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func loadConfig() *Config {
|
func loadConfig() *Config {
|
||||||
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme}
|
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme, TLSSkipVerify: true}
|
||||||
// 1) 优先从后台页面配置(DB)读取
|
// 1) 优先从后台页面配置(DB)读取
|
||||||
if models.SharedSysSettingDAO != nil {
|
if models.SharedSysSettingDAO != nil {
|
||||||
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
|
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
|
||||||
@@ -72,8 +70,8 @@ func loadConfig() *Config {
|
|||||||
cfg.Password = dbCfg.Password
|
cfg.Password = dbCfg.Password
|
||||||
cfg.Database = dbCfg.Database
|
cfg.Database = dbCfg.Database
|
||||||
cfg.Scheme = normalizeScheme(dbCfg.Scheme)
|
cfg.Scheme = normalizeScheme(dbCfg.Scheme)
|
||||||
cfg.TLSSkipVerify = dbCfg.TLSSkipVerify
|
cfg.TLSSkipVerify = true
|
||||||
cfg.TLSServerName = dbCfg.TLSServerName
|
cfg.TLSServerName = ""
|
||||||
if cfg.Port <= 0 {
|
if cfg.Port <= 0 {
|
||||||
cfg.Port = defaultPort
|
cfg.Port = defaultPort
|
||||||
}
|
}
|
||||||
@@ -93,8 +91,8 @@ func loadConfig() *Config {
|
|||||||
cfg.Password = ch.Password
|
cfg.Password = ch.Password
|
||||||
cfg.Database = ch.Database
|
cfg.Database = ch.Database
|
||||||
cfg.Scheme = normalizeScheme(ch.Scheme)
|
cfg.Scheme = normalizeScheme(ch.Scheme)
|
||||||
cfg.TLSSkipVerify = ch.TLSSkipVerify
|
cfg.TLSSkipVerify = true
|
||||||
cfg.TLSServerName = ch.TLSServerName
|
cfg.TLSServerName = ""
|
||||||
if cfg.Port <= 0 {
|
if cfg.Port <= 0 {
|
||||||
cfg.Port = defaultPort
|
cfg.Port = defaultPort
|
||||||
}
|
}
|
||||||
@@ -112,17 +110,13 @@ func loadConfig() *Config {
|
|||||||
cfg.Database = defaultDB
|
cfg.Database = defaultDB
|
||||||
}
|
}
|
||||||
cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
|
cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
|
||||||
cfg.TLSServerName = os.Getenv(envTLSServerName)
|
cfg.TLSServerName = ""
|
||||||
if p := os.Getenv(envPort); p != "" {
|
if p := os.Getenv(envPort); p != "" {
|
||||||
if v, err := strconv.Atoi(p); err == nil {
|
if v, err := strconv.Atoi(p); err == nil {
|
||||||
cfg.Port = v
|
cfg.Port = v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if v := os.Getenv(envTLSSkipVerify); v != "" {
|
cfg.TLSSkipVerify = true
|
||||||
if b, err := strconv.ParseBool(v); err == nil {
|
|
||||||
cfg.TLSSkipVerify = b
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -176,7 +176,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
|||||||
}
|
}
|
||||||
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
|
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
|
||||||
|
|
||||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
// 列表查询不 SELECT 大字段(request_headers / request_body / response_headers / response_body),
|
||||||
|
// 避免每次翻页读取 GB 级数据。详情查看时通过 FindByTraceId 单独获取。
|
||||||
|
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||||
table, where, orderBy, limit+1)
|
table, where, orderBy, limit+1)
|
||||||
|
|
||||||
var rawRows []map[string]interface{}
|
var rawRows []map[string]interface{}
|
||||||
|
|||||||
@@ -122,8 +122,10 @@ func (s *NSLogsIngestStore) List(ctx context.Context, f NSListFilter) (rows []*N
|
|||||||
limit = 1000
|
limit = 1000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 列表查询不 SELECT content_json 大字段,减少翻页时的数据传输量。
|
||||||
|
// 详情查看时通过 FindByRequestId 单独获取完整信息。
|
||||||
query := fmt.Sprintf(
|
query := fmt.Sprintf(
|
||||||
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes, content_json FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
|
"SELECT timestamp, request_id, node_id, cluster_id, domain_id, record_id, remote_addr, question_name, question_type, record_name, record_type, record_value, networking, is_recursive, error, ns_route_codes FROM %s WHERE %s ORDER BY timestamp %s, request_id %s LIMIT %d",
|
||||||
table,
|
table,
|
||||||
strings.Join(conditions, " AND "),
|
strings.Join(conditions, " AND "),
|
||||||
orderDir,
|
orderDir,
|
||||||
|
|||||||
@@ -20,3 +20,20 @@ func ParseHTTPAccessLogPolicyFilePath(policy *HTTPAccessLogPolicy) string {
|
|||||||
|
|
||||||
return strings.TrimSpace(config.Path)
|
return strings.TrimSpace(config.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseHTTPAccessLogPolicyRotateConfig 提取访问日志策略中的文件轮转配置(所有 file* 类型有效)。
|
||||||
|
func ParseHTTPAccessLogPolicyRotateConfig(policy *HTTPAccessLogPolicy) *serverconfigs.AccessLogRotateConfig {
|
||||||
|
if policy == nil || !serverconfigs.IsFileBasedStorageType(policy.Type) || len(policy.Options) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
config := &serverconfigs.AccessLogFileStorageConfig{}
|
||||||
|
if err := json.Unmarshal(policy.Options, config); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if config.Rotate == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.Rotate.Normalize()
|
||||||
|
}
|
||||||
|
|||||||
@@ -1176,6 +1176,7 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
|
|||||||
if publicPolicy != nil {
|
if publicPolicy != nil {
|
||||||
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||||
config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
config.GlobalServerConfig.HTTPAccessLog.FilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
||||||
|
config.GlobalServerConfig.HTTPAccessLog.Rotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -479,6 +479,7 @@ func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.
|
|||||||
if publicPolicy != nil {
|
if publicPolicy != nil {
|
||||||
config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
config.AccessLogWriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||||
config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
config.AccessLogFilePath = ParseHTTPAccessLogPolicyFilePath(publicPolicy)
|
||||||
|
config.AccessLogRotate = ParseHTTPAccessLogPolicyRotateConfig(publicPolicy)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -270,8 +270,9 @@ func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.Clic
|
|||||||
}
|
}
|
||||||
if len(valueJSON) == 0 {
|
if len(valueJSON) == 0 {
|
||||||
return &systemconfigs.ClickHouseSetting{
|
return &systemconfigs.ClickHouseSetting{
|
||||||
Port: 8123,
|
Port: 8443,
|
||||||
Database: "default",
|
Database: "default",
|
||||||
|
Scheme: "https",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
var config = &systemconfigs.ClickHouseSetting{}
|
var config = &systemconfigs.ClickHouseSetting{}
|
||||||
|
|||||||
@@ -1,14 +1,23 @@
|
|||||||
package installers
|
package installers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
slashpath "path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
|
||||||
|
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||||
|
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||||
"github.com/iwind/TeaGo/Tea"
|
"github.com/iwind/TeaGo/Tea"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -17,19 +26,45 @@ const (
|
|||||||
fluentBitStorageDir = "/var/lib/fluent-bit/storage"
|
fluentBitStorageDir = "/var/lib/fluent-bit/storage"
|
||||||
fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf"
|
fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf"
|
||||||
fluentBitParsersFile = "/etc/fluent-bit/parsers.conf"
|
fluentBitParsersFile = "/etc/fluent-bit/parsers.conf"
|
||||||
fluentBitUpstreamFile = "/etc/fluent-bit/clickhouse-upstream.conf"
|
fluentBitManagedMetaFile = "/etc/fluent-bit/.edge-managed.json"
|
||||||
|
fluentBitManagedEnvFile = "/etc/fluent-bit/.edge-managed.env"
|
||||||
fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge"
|
fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge"
|
||||||
|
fluentBitDropInDir = "/etc/systemd/system/fluent-bit.service.d"
|
||||||
|
fluentBitDropInFile = "/etc/systemd/system/fluent-bit.service.d/edge-managed.conf"
|
||||||
fluentBitServiceName = "fluent-bit"
|
fluentBitServiceName = "fluent-bit"
|
||||||
fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit"
|
fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit"
|
||||||
fluentBitLocalPackagesRoot = "packages"
|
fluentBitLocalPackagesRoot = "packages"
|
||||||
fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log"
|
fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log"
|
||||||
fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log"
|
fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log"
|
||||||
|
fluentBitManagedMarker = "managed-by-edgeapi"
|
||||||
|
fluentBitRoleNode = "node"
|
||||||
|
fluentBitRoleDNS = "dns"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found")
|
var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found")
|
||||||
|
|
||||||
// SetupFluentBit 安装 Fluent Bit(仅离线包),并同步配置文件。
|
var fluentBitPackageFileMapping = map[string]string{
|
||||||
// 升级场景下不覆盖已有配置;若已有配置与节点角色不兼容,直接报错终止安装。
|
"ubuntu22.04-amd64": "fluent-bit_4.2.2_amd64.deb",
|
||||||
|
"ubuntu22.04-arm64": "fluent-bit_4.2.2_arm64.deb",
|
||||||
|
"amzn2023-amd64": "fluent-bit-4.2.2-1.x86_64.rpm",
|
||||||
|
"amzn2023-arm64": "fluent-bit-4.2.2-1.aarch64.rpm",
|
||||||
|
}
|
||||||
|
|
||||||
|
type fluentBitManagedMeta struct {
|
||||||
|
Roles []string `json:"roles"`
|
||||||
|
Hash string `json:"hash"`
|
||||||
|
UpdatedAt int64 `json:"updatedAt"`
|
||||||
|
SourceVersion string `json:"sourceVersion"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type fluentBitDesiredConfig struct {
|
||||||
|
Roles []string
|
||||||
|
ClickHouse *systemconfigs.ClickHouseSetting
|
||||||
|
HTTPPathPattern string
|
||||||
|
DNSPathPattern string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetupFluentBit 安装并托管 Fluent Bit 配置(离线包 + 平台渲染配置)。
|
||||||
func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
||||||
if this.client == nil {
|
if this.client == nil {
|
||||||
return errors.New("ssh client is nil")
|
return errors.New("ssh client is nil")
|
||||||
@@ -41,55 +76,41 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit"
|
tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit"
|
||||||
_, _, _ = this.client.Exec("mkdir -p " + tempDir)
|
_, _, _ = this.client.Exec("mkdir -p " + shQuote(tempDir))
|
||||||
defer func() {
|
defer func() {
|
||||||
_, _, _ = this.client.Exec("rm -rf " + tempDir)
|
_, _, _ = this.client.Exec("rm -rf " + shQuote(tempDir))
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// 统一使用 fluent-bit.conf(已含 HTTP + DNS 两类 input),避免同机 Node/DNS 冲突。
|
|
||||||
files := []struct {
|
|
||||||
Local string
|
|
||||||
Remote string
|
|
||||||
}{
|
|
||||||
{Local: "fluent-bit.conf", Remote: "fluent-bit.conf"},
|
|
||||||
{Local: "parsers.conf", Remote: "parsers.conf"},
|
|
||||||
{Local: "clickhouse-upstream.conf", Remote: "clickhouse-upstream.conf"},
|
|
||||||
{Local: "logrotate.conf", Remote: "logrotate.conf"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
localPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", file.Local)
|
|
||||||
if _, err := os.Stat(localPath); err != nil {
|
|
||||||
return fmt.Errorf("fluent-bit file '%s' not found: %w", localPath, err)
|
|
||||||
}
|
|
||||||
remotePath := tempDir + "/" + file.Remote
|
|
||||||
if err := this.client.Copy(localPath, remotePath, 0644); err != nil {
|
|
||||||
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", file.Local, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := this.ensureFluentBitInstalled(tempDir); err != nil {
|
if err := this.ensureFluentBitInstalled(tempDir); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, stderr, err := this.client.Exec("mkdir -p " + fluentBitConfigDir + " " + fluentBitStorageDir + " /etc/logrotate.d")
|
_, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitConfigDir) + " " + shQuote(fluentBitStorageDir) + " /etc/logrotate.d")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr)
|
return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
exists, err := this.remoteFileExists(fluentBitMainConfigFile)
|
parserContent, err := this.readLocalParsersContent()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// 若已存在配置,先做角色兼容校验,不允许覆盖。
|
existingMeta, err := this.readManagedMeta()
|
||||||
if exists {
|
if err != nil {
|
||||||
if err := this.validateExistingConfigForRole(role); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mergedRoles, err := mergeManagedRoles(existingMeta, role)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
configCopied, err := this.copyFluentBitConfigIfMissing(tempDir)
|
desiredConfig, err := this.buildDesiredFluentBitConfig(mergedRoles)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
configChanged, err := this.applyManagedConfig(tempDir, desiredConfig, parserContent, existingMeta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -99,7 +120,7 @@ func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := this.ensureFluentBitService(binPath, configCopied); err != nil {
|
if err := this.ensureFluentBitService(tempDir, binPath, configChanged); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,102 +133,128 @@ func (this *BaseInstaller) ensureFluentBitInstalled(tempDir string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := this.installFluentBitFromLocalPackage(tempDir); err != nil {
|
platformKey, packageName, arch, err := this.detectRemotePlatformAndPackage()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("detect fluent-bit platform failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := this.installFluentBitFromLocalPackage(tempDir, arch, packageName); err != nil {
|
||||||
if errors.Is(err, errFluentBitLocalPackageNotFound) {
|
if errors.Is(err, errFluentBitLocalPackageNotFound) {
|
||||||
return fmt.Errorf("install fluent-bit failed: local package not found, expected in deploy/fluent-bit/%s/linux-<arch>", fluentBitLocalPackagesRoot)
|
expectedPath := filepath.Join("deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch, packageName)
|
||||||
|
return fmt.Errorf("install fluent-bit failed: local package missing for platform '%s', expected '%s'", platformKey, expectedPath)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("install fluent-bit from local package failed: %w", err)
|
return fmt.Errorf("install fluent-bit from local package failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
binPath, err := this.lookupFluentBitBinPath()
|
binPath, err = this.lookupFluentBitBinPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if binPath == "" {
|
if binPath == "" {
|
||||||
return errors.New("fluent-bit binary not found after local package install")
|
return errors.New("fluent-bit binary not found after local package install")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_, stderr, err := this.client.Exec(binPath + " --version")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("verify fluent-bit version failed: %w, stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string) error {
|
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string, arch string, packageName string) error {
|
||||||
arch, err := this.detectRemoteLinuxArch()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch)
|
packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch)
|
||||||
entries, err := os.ReadDir(packageDir)
|
localPackagePath := filepath.Join(packageDir, packageName)
|
||||||
if err != nil {
|
if _, err := os.Stat(localPackagePath); err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return errFluentBitLocalPackageNotFound
|
return errFluentBitLocalPackageNotFound
|
||||||
}
|
}
|
||||||
return fmt.Errorf("read fluent-bit local package dir failed: %w", err)
|
return fmt.Errorf("check local package failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
packageFiles := make([]string, 0)
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
name := strings.ToLower(entry.Name())
|
|
||||||
if strings.HasSuffix(name, ".deb") || strings.HasSuffix(name, ".rpm") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") {
|
|
||||||
packageFiles = append(packageFiles, filepath.Join(packageDir, entry.Name()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(packageFiles) == 0 {
|
|
||||||
return errFluentBitLocalPackageNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(packageFiles)
|
|
||||||
|
|
||||||
var lastErr error
|
|
||||||
for _, localPackagePath := range packageFiles {
|
|
||||||
remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath)
|
remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath)
|
||||||
if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil {
|
if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil {
|
||||||
lastErr = fmt.Errorf("upload local package failed: %w", err)
|
return fmt.Errorf("upload local package failed: %w", err)
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var installCmd string
|
var installCmd string
|
||||||
lowerName := strings.ToLower(localPackagePath)
|
lowerName := strings.ToLower(localPackagePath)
|
||||||
switch {
|
switch {
|
||||||
case strings.HasSuffix(lowerName, ".deb"):
|
case strings.HasSuffix(lowerName, ".deb"):
|
||||||
installCmd = "dpkg -i " + remotePackagePath
|
installCmd = "dpkg -i " + shQuote(remotePackagePath)
|
||||||
case strings.HasSuffix(lowerName, ".rpm"):
|
case strings.HasSuffix(lowerName, ".rpm"):
|
||||||
installCmd = "rpm -Uvh --force " + remotePackagePath + " || rpm -ivh --force " + remotePackagePath
|
installCmd = "rpm -Uvh --force " + shQuote(remotePackagePath) + " || rpm -ivh --force " + shQuote(remotePackagePath)
|
||||||
case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"):
|
case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"):
|
||||||
extractDir := tempDir + "/extract"
|
extractDir := tempDir + "/extract"
|
||||||
installCmd = "rm -rf " + extractDir + "; mkdir -p " + extractDir + "; tar -xzf " + remotePackagePath + " -C " + extractDir + "; " +
|
installCmd = "rm -rf " + shQuote(extractDir) + "; mkdir -p " + shQuote(extractDir) + "; tar -xzf " + shQuote(remotePackagePath) + " -C " + shQuote(extractDir) + "; " +
|
||||||
"bin=$(find " + extractDir + " -type f -name fluent-bit | head -n 1); " +
|
"bin=$(find " + shQuote(extractDir) + " -type f -name fluent-bit | head -n 1); " +
|
||||||
"if [ -z \"$bin\" ]; then exit 3; fi; " +
|
"if [ -z \"$bin\" ]; then exit 3; fi; " +
|
||||||
"mkdir -p /opt/fluent-bit/bin /usr/local/bin; " +
|
"mkdir -p /opt/fluent-bit/bin /usr/local/bin; " +
|
||||||
"install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " +
|
"install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " +
|
||||||
"ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit"
|
"ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit"
|
||||||
default:
|
default:
|
||||||
continue
|
return fmt.Errorf("unsupported local package format: %s", packageName)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, stderr, err := this.client.Exec(installCmd)
|
_, stderr, err := this.client.Exec(installCmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
lastErr = fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
|
return fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) detectRemotePlatformAndPackage() (platformKey string, packageName string, arch string, err error) {
|
||||||
|
arch, err = this.detectRemoteLinuxArch()
|
||||||
|
if err != nil {
|
||||||
|
return "", "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
releaseData, stderr, err := this.client.Exec("cat /etc/os-release")
|
||||||
|
if err != nil {
|
||||||
|
return "", "", "", fmt.Errorf("read /etc/os-release failed: %w, stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(releaseData) == "" {
|
||||||
|
return "", "", "", errors.New("/etc/os-release is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
releaseMap := parseOSRelease(releaseData)
|
||||||
|
id := strings.ToLower(strings.TrimSpace(releaseMap["ID"]))
|
||||||
|
versionID := strings.TrimSpace(releaseMap["VERSION_ID"])
|
||||||
|
|
||||||
|
var distro string
|
||||||
|
switch {
|
||||||
|
case id == "ubuntu" && strings.HasPrefix(versionID, "22.04"):
|
||||||
|
distro = "ubuntu22.04"
|
||||||
|
case id == "amzn" && strings.HasPrefix(versionID, "2023"):
|
||||||
|
distro = "amzn2023"
|
||||||
|
default:
|
||||||
|
return "", "", "", fmt.Errorf("unsupported linux platform id='%s' version='%s'", id, versionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
platformKey = distro + "-" + arch
|
||||||
|
packageName, ok := fluentBitPackageFileMapping[platformKey]
|
||||||
|
if !ok {
|
||||||
|
return "", "", "", fmt.Errorf("no local package mapping for platform '%s'", platformKey)
|
||||||
|
}
|
||||||
|
return platformKey, packageName, arch, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOSRelease(content string) map[string]string {
|
||||||
|
result := map[string]string{}
|
||||||
|
lines := strings.Split(content, "\n")
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
parts := strings.SplitN(line, "=", 2)
|
||||||
binPath, err := this.lookupFluentBitBinPath()
|
key := strings.TrimSpace(parts[0])
|
||||||
if err == nil && binPath != "" {
|
value := strings.TrimSpace(parts[1])
|
||||||
return nil
|
value = strings.Trim(value, "\"")
|
||||||
|
result[key] = value
|
||||||
}
|
}
|
||||||
if err != nil {
|
return result
|
||||||
lastErr = err
|
|
||||||
} else {
|
|
||||||
lastErr = errors.New("fluent-bit binary not found after local package install")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if lastErr != nil {
|
|
||||||
return lastErr
|
|
||||||
}
|
|
||||||
return errFluentBitLocalPackageNotFound
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) {
|
func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) {
|
||||||
@@ -235,74 +282,456 @@ func (this *BaseInstaller) lookupFluentBitBinPath() (string, error) {
|
|||||||
return strings.TrimSpace(stdout), nil
|
return strings.TrimSpace(stdout), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) copyFluentBitConfigIfMissing(tempDir string) (bool, error) {
|
func (this *BaseInstaller) readLocalParsersContent() (string, error) {
|
||||||
targets := []struct {
|
parsersPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", "parsers.conf")
|
||||||
Src string
|
data, err := os.ReadFile(parsersPath)
|
||||||
Dest string
|
if err != nil {
|
||||||
}{
|
return "", fmt.Errorf("read local parsers config failed: %w", err)
|
||||||
{Src: tempDir + "/fluent-bit.conf", Dest: fluentBitMainConfigFile},
|
}
|
||||||
{Src: tempDir + "/parsers.conf", Dest: fluentBitParsersFile},
|
return string(data), nil
|
||||||
{Src: tempDir + "/clickhouse-upstream.conf", Dest: fluentBitUpstreamFile},
|
}
|
||||||
{Src: tempDir + "/logrotate.conf", Dest: fluentBitLogrotateFile},
|
|
||||||
|
func (this *BaseInstaller) readManagedMeta() (*fluentBitManagedMeta, error) {
|
||||||
|
exists, err := this.remoteFileExists(fluentBitManagedMetaFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
copied := false
|
content, stderr, err := this.client.Exec("cat " + shQuote(fluentBitManagedMetaFile))
|
||||||
for _, target := range targets {
|
if err != nil {
|
||||||
exists, err := this.remoteFileExists(target.Dest)
|
return nil, fmt.Errorf("read fluent-bit managed metadata failed: %w, stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(content) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := &fluentBitManagedMeta{}
|
||||||
|
if err := json.Unmarshal([]byte(content), meta); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode fluent-bit managed metadata failed: %w", err)
|
||||||
|
}
|
||||||
|
meta.Roles = normalizeRoles(meta.Roles)
|
||||||
|
return meta, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeManagedRoles(meta *fluentBitManagedMeta, role nodeconfigs.NodeRole) ([]string, error) {
|
||||||
|
roleName, err := mapNodeRole(role)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
roleSet := map[string]struct{}{}
|
||||||
|
if meta != nil {
|
||||||
|
for _, r := range normalizeRoles(meta.Roles) {
|
||||||
|
roleSet[r] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
roleSet[roleName] = struct{}{}
|
||||||
|
|
||||||
|
roles := make([]string, 0, len(roleSet))
|
||||||
|
for r := range roleSet {
|
||||||
|
roles = append(roles, r)
|
||||||
|
}
|
||||||
|
sort.Strings(roles)
|
||||||
|
return roles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapNodeRole(role nodeconfigs.NodeRole) (string, error) {
|
||||||
|
switch role {
|
||||||
|
case nodeconfigs.NodeRoleNode:
|
||||||
|
return fluentBitRoleNode, nil
|
||||||
|
case nodeconfigs.NodeRoleDNS:
|
||||||
|
return fluentBitRoleDNS, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unsupported fluent-bit role '%s'", role)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRoles(rawRoles []string) []string {
|
||||||
|
roleSet := map[string]struct{}{}
|
||||||
|
for _, role := range rawRoles {
|
||||||
|
role = strings.ToLower(strings.TrimSpace(role))
|
||||||
|
if role != fluentBitRoleNode && role != fluentBitRoleDNS {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
roleSet[role] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
roles := make([]string, 0, len(roleSet))
|
||||||
|
for role := range roleSet {
|
||||||
|
roles = append(roles, role)
|
||||||
|
}
|
||||||
|
sort.Strings(roles)
|
||||||
|
return roles
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasRole(roles []string, role string) bool {
|
||||||
|
for _, one := range roles {
|
||||||
|
if one == role {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) buildDesiredFluentBitConfig(roles []string) (*fluentBitDesiredConfig, error) {
|
||||||
|
if len(roles) == 0 {
|
||||||
|
return nil, errors.New("fluent-bit roles should not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
ch, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read clickhouse setting failed: %w", err)
|
||||||
|
}
|
||||||
|
if ch == nil {
|
||||||
|
ch = &systemconfigs.ClickHouseSetting{}
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(ch.Host) == "" {
|
||||||
|
ch.Host = "127.0.0.1"
|
||||||
|
}
|
||||||
|
|
||||||
|
ch.Scheme = strings.ToLower(strings.TrimSpace(ch.Scheme))
|
||||||
|
if ch.Scheme == "" {
|
||||||
|
ch.Scheme = "https"
|
||||||
|
}
|
||||||
|
if ch.Scheme != "http" && ch.Scheme != "https" {
|
||||||
|
return nil, fmt.Errorf("unsupported clickhouse scheme '%s'", ch.Scheme)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ch.Port <= 0 {
|
||||||
|
if ch.Scheme == "https" {
|
||||||
|
ch.Port = 8443
|
||||||
|
} else {
|
||||||
|
ch.Port = 8443
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(ch.Database) == "" {
|
||||||
|
ch.Database = "default"
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(ch.User) == "" {
|
||||||
|
ch.User = "default"
|
||||||
|
}
|
||||||
|
// 当前平台策略:后台固定跳过 ClickHouse TLS 证书校验,不暴露 ServerName 配置。
|
||||||
|
ch.TLSSkipVerify = true
|
||||||
|
ch.TLSServerName = ""
|
||||||
|
|
||||||
|
httpPathPattern := fluentBitHTTPPathPattern
|
||||||
|
dnsPathPattern := fluentBitDNSPathPattern
|
||||||
|
publicPolicyPath, err := this.readPublicAccessLogPolicyPath()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
policyDir := dirFromPolicyPath(publicPolicyPath)
|
||||||
|
if policyDir != "" {
|
||||||
|
pattern := strings.TrimRight(policyDir, "/") + "/*.log"
|
||||||
|
httpPathPattern = pattern
|
||||||
|
dnsPathPattern = pattern
|
||||||
|
}
|
||||||
|
|
||||||
|
return &fluentBitDesiredConfig{
|
||||||
|
Roles: normalizeRoles(roles),
|
||||||
|
ClickHouse: ch,
|
||||||
|
HTTPPathPattern: httpPathPattern,
|
||||||
|
DNSPathPattern: dnsPathPattern,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) readPublicAccessLogPolicyPath() (string, error) {
|
||||||
|
policyId, err := models.SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("find current public access log policy failed: %w", err)
|
||||||
|
}
|
||||||
|
if policyId <= 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
policy, err := models.SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(nil, policyId)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read public access log policy failed: %w", err)
|
||||||
|
}
|
||||||
|
if policy == nil {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimSpace(models.ParseHTTPAccessLogPolicyFilePath(policy)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dirFromPolicyPath(policyPath string) string {
|
||||||
|
pathValue := strings.TrimSpace(policyPath)
|
||||||
|
if pathValue == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
pathValue = strings.ReplaceAll(pathValue, "\\", "/")
|
||||||
|
dir := slashpath.Dir(pathValue)
|
||||||
|
if dir == "." {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimRight(dir, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) applyManagedConfig(tempDir string, desired *fluentBitDesiredConfig, parserContent string, existingMeta *fluentBitManagedMeta) (bool, error) {
|
||||||
|
mainExists, err := this.remoteFileExists(fluentBitMainConfigFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
if exists {
|
|
||||||
continue
|
if mainExists && existingMeta == nil {
|
||||||
}
|
containsMarker, err := this.remoteFileContains(fluentBitMainConfigFile, fluentBitManagedMarker)
|
||||||
_, stderr, err := this.client.Exec("cp -f " + target.Src + " " + target.Dest)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("copy fluent-bit file to '%s' failed: %w, stderr: %s", target.Dest, err, stderr)
|
return false, err
|
||||||
|
}
|
||||||
|
if !containsMarker {
|
||||||
|
// Adopt unmanaged config by backing it up and replacing with managed config below.
|
||||||
}
|
}
|
||||||
copied = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return copied, nil
|
configContent, err := renderManagedConfig(desired)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
envContent := renderManagedEnv(desired.ClickHouse)
|
||||||
|
metaContent, newMeta, err := renderManagedMeta(desired, configContent, parserContent, envContent)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
requiredFiles := []string{fluentBitMainConfigFile, fluentBitParsersFile, fluentBitManagedEnvFile, fluentBitManagedMetaFile}
|
||||||
|
if existingMeta != nil && existingMeta.Hash == newMeta.Hash {
|
||||||
|
allExists := true
|
||||||
|
for _, file := range requiredFiles {
|
||||||
|
exists, err := this.remoteFileExists(file)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
allExists = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if allExists {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if mainExists {
|
||||||
|
backup := fluentBitMainConfigFile + ".bak." + strconv.FormatInt(time.Now().Unix(), 10)
|
||||||
|
_, stderr, err := this.client.Exec("cp -f " + shQuote(fluentBitMainConfigFile) + " " + shQuote(backup))
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("backup existing fluent-bit config failed: %w, stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := this.writeRemoteFileByTemp(tempDir, fluentBitMainConfigFile, configContent, 0644); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := this.writeRemoteFileByTemp(tempDir, fluentBitParsersFile, parserContent, 0644); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedEnvFile, envContent, 0600); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := this.writeRemoteFileByTemp(tempDir, fluentBitManagedMetaFile, metaContent, 0644); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
localLogrotate := filepath.Join(Tea.Root, "deploy", "fluent-bit", "logrotate.conf")
|
||||||
|
if _, err := os.Stat(localLogrotate); err == nil {
|
||||||
|
if err := this.copyLocalFileToRemote(tempDir, localLogrotate, fluentBitLogrotateFile, 0644); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) validateExistingConfigForRole(role nodeconfigs.NodeRole) error {
|
func renderManagedConfig(desired *fluentBitDesiredConfig) (string, error) {
|
||||||
requiredPatterns := []string{}
|
if desired == nil || desired.ClickHouse == nil {
|
||||||
switch role {
|
return "", errors.New("invalid fluent-bit desired config")
|
||||||
case nodeconfigs.NodeRoleNode:
|
|
||||||
requiredPatterns = append(requiredPatterns, fluentBitHTTPPathPattern)
|
|
||||||
case nodeconfigs.NodeRoleDNS:
|
|
||||||
requiredPatterns = append(requiredPatterns, fluentBitDNSPathPattern)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, pattern := range requiredPatterns {
|
scheme := strings.ToLower(strings.TrimSpace(desired.ClickHouse.Scheme))
|
||||||
ok, err := this.remoteFileContains(fluentBitMainConfigFile, pattern)
|
if scheme == "" {
|
||||||
|
scheme = "http"
|
||||||
|
}
|
||||||
|
if scheme != "http" && scheme != "https" {
|
||||||
|
return "", fmt.Errorf("unsupported clickhouse scheme '%s'", desired.ClickHouse.Scheme)
|
||||||
|
}
|
||||||
|
useTLS := scheme == "https"
|
||||||
|
|
||||||
|
insertHTTP := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
|
||||||
|
insertDNS := url.QueryEscape(fmt.Sprintf("INSERT INTO %s.dns_logs_ingest FORMAT JSONEachRow", desired.ClickHouse.Database))
|
||||||
|
|
||||||
|
lines := []string{
|
||||||
|
"# " + fluentBitManagedMarker,
|
||||||
|
"[SERVICE]",
|
||||||
|
" Flush 2",
|
||||||
|
" Log_Level info",
|
||||||
|
" Parsers_File " + fluentBitParsersFile,
|
||||||
|
" storage.path " + fluentBitStorageDir,
|
||||||
|
" storage.sync normal",
|
||||||
|
" storage.checksum off",
|
||||||
|
" storage.backlog.mem_limit 256MB",
|
||||||
|
"",
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasRole(desired.Roles, fluentBitRoleNode) {
|
||||||
|
lines = append(lines,
|
||||||
|
"[INPUT]",
|
||||||
|
" Name tail",
|
||||||
|
" Path "+desired.HTTPPathPattern,
|
||||||
|
" Tag app.http.logs",
|
||||||
|
" Parser json",
|
||||||
|
" Refresh_Interval 2",
|
||||||
|
" Read_from_Head false",
|
||||||
|
" DB /var/lib/fluent-bit/http-logs.db",
|
||||||
|
" storage.type filesystem",
|
||||||
|
" Mem_Buf_Limit 128MB",
|
||||||
|
" Skip_Long_Lines On",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasRole(desired.Roles, fluentBitRoleDNS) {
|
||||||
|
lines = append(lines,
|
||||||
|
"[INPUT]",
|
||||||
|
" Name tail",
|
||||||
|
" Path "+desired.DNSPathPattern,
|
||||||
|
" Tag app.dns.logs",
|
||||||
|
" Parser json",
|
||||||
|
" Refresh_Interval 2",
|
||||||
|
" Read_from_Head false",
|
||||||
|
" DB /var/lib/fluent-bit/dns-logs.db",
|
||||||
|
" storage.type filesystem",
|
||||||
|
" Mem_Buf_Limit 128MB",
|
||||||
|
" Skip_Long_Lines On",
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasRole(desired.Roles, fluentBitRoleNode) {
|
||||||
|
lines = append(lines,
|
||||||
|
"[OUTPUT]",
|
||||||
|
" Name http",
|
||||||
|
" Match app.http.logs",
|
||||||
|
" Host "+desired.ClickHouse.Host,
|
||||||
|
" Port "+strconv.Itoa(desired.ClickHouse.Port),
|
||||||
|
" URI /?query="+insertHTTP,
|
||||||
|
" Format json_lines",
|
||||||
|
" http_user ${CH_USER}",
|
||||||
|
" http_passwd ${CH_PASSWORD}",
|
||||||
|
" json_date_key timestamp",
|
||||||
|
" json_date_format epoch",
|
||||||
|
" workers 1",
|
||||||
|
" net.keepalive On",
|
||||||
|
" Retry_Limit False",
|
||||||
|
)
|
||||||
|
if useTLS {
|
||||||
|
lines = append(lines, " tls On")
|
||||||
|
if desired.ClickHouse.TLSSkipVerify {
|
||||||
|
lines = append(lines, " tls.verify Off")
|
||||||
|
} else {
|
||||||
|
lines = append(lines, " tls.verify On")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
|
||||||
|
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines = append(lines, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasRole(desired.Roles, fluentBitRoleDNS) {
|
||||||
|
lines = append(lines,
|
||||||
|
"[OUTPUT]",
|
||||||
|
" Name http",
|
||||||
|
" Match app.dns.logs",
|
||||||
|
" Host "+desired.ClickHouse.Host,
|
||||||
|
" Port "+strconv.Itoa(desired.ClickHouse.Port),
|
||||||
|
" URI /?query="+insertDNS,
|
||||||
|
" Format json_lines",
|
||||||
|
" http_user ${CH_USER}",
|
||||||
|
" http_passwd ${CH_PASSWORD}",
|
||||||
|
" json_date_key timestamp",
|
||||||
|
" json_date_format epoch",
|
||||||
|
" workers 1",
|
||||||
|
" net.keepalive On",
|
||||||
|
" Retry_Limit False",
|
||||||
|
)
|
||||||
|
if useTLS {
|
||||||
|
lines = append(lines, " tls On")
|
||||||
|
if desired.ClickHouse.TLSSkipVerify {
|
||||||
|
lines = append(lines, " tls.verify Off")
|
||||||
|
} else {
|
||||||
|
lines = append(lines, " tls.verify On")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(desired.ClickHouse.TLSServerName) != "" {
|
||||||
|
lines = append(lines, " tls.vhost "+strings.TrimSpace(desired.ClickHouse.TLSServerName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines = append(lines, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(lines, "\n"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderManagedEnv(ch *systemconfigs.ClickHouseSetting) string {
|
||||||
|
user := "default"
|
||||||
|
password := ""
|
||||||
|
if ch != nil {
|
||||||
|
if strings.TrimSpace(ch.User) != "" {
|
||||||
|
user = strings.TrimSpace(ch.User)
|
||||||
|
}
|
||||||
|
password = ch.Password
|
||||||
|
}
|
||||||
|
return "CH_USER=" + strconv.Quote(user) + "\n" +
|
||||||
|
"CH_PASSWORD=" + strconv.Quote(password) + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderManagedMeta(desired *fluentBitDesiredConfig, configContent string, parserContent string, envContent string) (string, *fluentBitManagedMeta, error) {
|
||||||
|
hashInput := configContent + "\n---\n" + parserContent + "\n---\n" + envContent + "\n---\n" + strings.Join(desired.Roles, ",")
|
||||||
|
hashBytes := sha256.Sum256([]byte(hashInput))
|
||||||
|
hash := fmt.Sprintf("%x", hashBytes[:])
|
||||||
|
|
||||||
|
meta := &fluentBitManagedMeta{
|
||||||
|
Roles: desired.Roles,
|
||||||
|
Hash: hash,
|
||||||
|
UpdatedAt: time.Now().Unix(),
|
||||||
|
SourceVersion: teaconst.Version,
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(meta, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", nil, fmt.Errorf("encode fluent-bit managed metadata failed: %w", err)
|
||||||
}
|
}
|
||||||
if !ok {
|
return string(data) + "\n", meta, nil
|
||||||
return fmt.Errorf("existing fluent-bit config '%s' does not contain required path '%s'; skip overwrite by design, please update config manually", fluentBitMainConfigFile, pattern)
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) copyLocalFileToRemote(tempDir string, localPath string, remotePath string, mode os.FileMode) error {
|
||||||
|
tempFile := tempDir + "/" + filepath.Base(remotePath)
|
||||||
|
if err := this.client.Copy(localPath, tempFile, mode); err != nil {
|
||||||
|
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", localPath, err)
|
||||||
}
|
}
|
||||||
|
_, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("install remote fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
|
func (this *BaseInstaller) writeRemoteFileByTemp(tempDir string, remotePath string, content string, mode os.FileMode) error {
|
||||||
stdout, stderr, err := this.client.Exec("if [ -f \"" + path + "\" ]; then echo 1; else echo 0; fi")
|
tempFile := tempDir + "/" + filepath.Base(remotePath) + ".tmp"
|
||||||
if err != nil {
|
if _, err := this.client.WriteFile(tempFile, []byte(content)); err != nil {
|
||||||
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
|
return fmt.Errorf("write temp fluent-bit file '%s' failed: %w", tempFile, err)
|
||||||
}
|
}
|
||||||
return strings.TrimSpace(stdout) == "1", nil
|
|
||||||
|
_, stderr, err := this.client.Exec("cp -f " + shQuote(tempFile) + " " + shQuote(remotePath) + " && chmod " + fmt.Sprintf("%04o", mode) + " " + shQuote(remotePath))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("write managed fluent-bit file '%s' failed: %w, stderr: %s", remotePath, err, stderr)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
|
func (this *BaseInstaller) ensureFluentBitService(tempDir string, binPath string, configChanged bool) error {
|
||||||
stdout, stderr, err := this.client.Exec("if grep -F \"" + pattern + "\" \"" + path + "\" >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(stdout) == "1", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied bool) error {
|
|
||||||
_, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " +
|
_, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " +
|
||||||
"cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" +
|
"cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" +
|
||||||
"[Unit]\n" +
|
"[Unit]\n" +
|
||||||
@@ -319,26 +748,90 @@ func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied b
|
|||||||
"EOF\n" +
|
"EOF\n" +
|
||||||
"fi")
|
"fi")
|
||||||
|
|
||||||
stdout, stderr, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
|
stdout, _, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
||||||
"if [ \"" + boolToString(configCopied) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
|
if err != nil {
|
||||||
"else systemctl start " + fluentBitServiceName + "; fi; else echo no-systemctl; fi")
|
return fmt.Errorf("check systemctl failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(stdout) == "1" {
|
||||||
|
dropInChanged, err := this.ensureServiceDropIn(tempDir, binPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
restartRequired := configChanged || dropInChanged
|
||||||
|
_, stderr, err := this.client.Exec("systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; " +
|
||||||
|
"if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
|
||||||
|
"if [ \"" + boolToString(restartRequired) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
|
||||||
|
"else systemctl start " + fluentBitServiceName + "; fi")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr)
|
return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr)
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if configChanged {
|
||||||
|
_, _, _ = this.client.Exec("pkill -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1 || true")
|
||||||
|
}
|
||||||
|
|
||||||
if strings.TrimSpace(stdout) == "no-systemctl" {
|
|
||||||
_, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1")
|
_, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1")
|
||||||
if runningErr != nil {
|
if runningErr != nil {
|
||||||
_, stderr, err = this.client.Exec(binPath + " -c " + fluentBitMainConfigFile + " >/dev/null 2>&1 &")
|
startCmd := "set -a; [ -f " + shQuote(fluentBitManagedEnvFile) + " ] && . " + shQuote(fluentBitManagedEnvFile) + "; set +a; " +
|
||||||
|
shQuote(binPath) + " -c " + shQuote(fluentBitMainConfigFile) + " >/dev/null 2>&1 &"
|
||||||
|
_, stderr, err := this.client.Exec(startCmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr)
|
return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) ensureServiceDropIn(tempDir string, binPath string) (bool, error) {
|
||||||
|
_, stderr, err := this.client.Exec("mkdir -p " + shQuote(fluentBitDropInDir))
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("prepare fluent-bit drop-in dir failed: %w, stderr: %s", err, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
content := "[Service]\n" +
|
||||||
|
"EnvironmentFile=-" + fluentBitManagedEnvFile + "\n" +
|
||||||
|
"ExecStart=\n" +
|
||||||
|
"ExecStart=" + binPath + " -c " + fluentBitMainConfigFile + "\n"
|
||||||
|
|
||||||
|
existing, _, _ := this.client.Exec("if [ -f " + shQuote(fluentBitDropInFile) + " ]; then cat " + shQuote(fluentBitDropInFile) + "; fi")
|
||||||
|
if existing == content {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := this.writeRemoteFileByTemp(tempDir, fluentBitDropInFile, content, 0644); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
|
||||||
|
stdout, stderr, err := this.client.Exec("if [ -f " + shQuote(path) + " ]; then echo 1; else echo 0; fi")
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(stdout) == "1", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
|
||||||
|
stdout, stderr, err := this.client.Exec("if grep -F " + shQuote(pattern) + " " + shQuote(path) + " >/dev/null 2>&1; then echo 1; else echo 0; fi")
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(stdout) == "1", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func shQuote(value string) string {
|
||||||
|
if value == "" {
|
||||||
|
return "''"
|
||||||
|
}
|
||||||
|
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
|
||||||
|
}
|
||||||
|
|
||||||
func boolToString(v bool) string {
|
func boolToString(v bool) string {
|
||||||
if v {
|
if v {
|
||||||
return "1"
|
return "1"
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ function build() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
|
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
|
||||||
# 生成 zip 文件名时不包含 plus 标记
|
# 鐢熸垚 zip 鏂囦欢鍚嶆椂涓嶅寘鍚?plus 鏍囪
|
||||||
if [ "${TAG}" = "plus" ]; then
|
if [ "${TAG}" = "plus" ]; then
|
||||||
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
|
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
|
||||||
else
|
else
|
||||||
@@ -97,7 +97,7 @@ function build() {
|
|||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 查找 EdgeAPI zip 文件时不包含 plus 标记
|
# 鏌ユ壘 EdgeAPI zip 鏂囦欢鏃朵笉鍖呭惈 plus 鏍囪
|
||||||
if [ "${TAG}" = "plus" ]; then
|
if [ "${TAG}" = "plus" ]; then
|
||||||
EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip"
|
EDGE_API_ZIP_FILE=$ROOT"/../../EdgeAPI/dist/edge-api-${OS}-${ARCH}-v${APINodeVersion}.zip"
|
||||||
else
|
else
|
||||||
@@ -107,7 +107,44 @@ function build() {
|
|||||||
cd "$DIST"/ || exit
|
cd "$DIST"/ || exit
|
||||||
unzip -q "$(basename "$EDGE_API_ZIP_FILE")"
|
unzip -q "$(basename "$EDGE_API_ZIP_FILE")"
|
||||||
rm -f "$(basename "$EDGE_API_ZIP_FILE")"
|
rm -f "$(basename "$EDGE_API_ZIP_FILE")"
|
||||||
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件)
|
|
||||||
|
# ensure edge-api package always contains fluent-bit templates/packages
|
||||||
|
FLUENT_ROOT="$ROOT/../../deploy/fluent-bit"
|
||||||
|
FLUENT_DIST="$DIST/edge-api/deploy/fluent-bit"
|
||||||
|
if [ -d "$FLUENT_ROOT" ]; then
|
||||||
|
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || exit 1
|
||||||
|
rm -rf "$FLUENT_DIST"
|
||||||
|
mkdir -p "$FLUENT_DIST"
|
||||||
|
|
||||||
|
FLUENT_FILES=(
|
||||||
|
"fluent-bit.conf"
|
||||||
|
"fluent-bit-dns.conf"
|
||||||
|
"fluent-bit-https.conf"
|
||||||
|
"fluent-bit-dns-https.conf"
|
||||||
|
"fluent-bit-windows.conf"
|
||||||
|
"fluent-bit-windows-https.conf"
|
||||||
|
"parsers.conf"
|
||||||
|
"clickhouse-upstream.conf"
|
||||||
|
"clickhouse-upstream-windows.conf"
|
||||||
|
"logrotate.conf"
|
||||||
|
"README.md"
|
||||||
|
)
|
||||||
|
for file in "${FLUENT_FILES[@]}"; do
|
||||||
|
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||||
|
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -d "$FLUENT_ROOT/packages" ]; then
|
||||||
|
cp -R "$FLUENT_ROOT/packages" "$FLUENT_DIST/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f "$FLUENT_DIST/.gitignore"
|
||||||
|
rm -f "$FLUENT_DIST"/logs.db*
|
||||||
|
rm -rf "$FLUENT_DIST/storage"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
|
||||||
find . -name "*.mmdb" -type f -delete
|
find . -name "*.mmdb" -type f -delete
|
||||||
find . -type d -name "iplibrary" -empty -delete
|
find . -type d -name "iplibrary" -empty -delete
|
||||||
cd - || exit
|
cd - || exit
|
||||||
@@ -150,7 +187,7 @@ function build() {
|
|||||||
#find "$DIST" -name "*.css.map" -delete
|
#find "$DIST" -name "*.css.map" -delete
|
||||||
#find "$DIST" -name "*.js.map" -delete
|
#find "$DIST" -name "*.js.map" -delete
|
||||||
|
|
||||||
# 删除 MaxMind 数据库文件(使用嵌入的数据库,不需要外部文件)
|
# 鍒犻櫎 MaxMind 鏁版嵁搴撴枃浠讹紙浣跨敤宓屽叆鐨勬暟鎹簱锛屼笉闇€瑕佸閮ㄦ枃浠讹級
|
||||||
find "$DIST" -name "*.mmdb" -type f -delete
|
find "$DIST" -name "*.mmdb" -type f -delete
|
||||||
find "$DIST" -type d -name "iplibrary" -empty -delete
|
find "$DIST" -type d -name "iplibrary" -empty -delete
|
||||||
|
|
||||||
@@ -167,6 +204,39 @@ function build() {
|
|||||||
echo "[done]"
|
echo "[done]"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function verify_fluent_bit_package_matrix() {
|
||||||
|
FLUENT_ROOT=$1
|
||||||
|
ARCH=$2
|
||||||
|
REQUIRED_FILES=()
|
||||||
|
if [ "$ARCH" = "amd64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||||
|
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||||
|
)
|
||||||
|
elif [ "$ARCH" = "arm64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||||
|
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
MISSING=0
|
||||||
|
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||||
|
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||||
|
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||||
|
MISSING=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$MISSING" -ne 0 ]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
function lookup-version() {
|
function lookup-version() {
|
||||||
FILE=$1
|
FILE=$1
|
||||||
VERSION_DATA=$(cat "$FILE")
|
VERSION_DATA=$(cat "$FILE")
|
||||||
|
|||||||
@@ -3,13 +3,18 @@
|
|||||||
package db
|
package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/tls"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
|
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||||
"github.com/iwind/TeaGo/actions"
|
"github.com/iwind/TeaGo/actions"
|
||||||
"strings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const clickhouseConfigCode = "clickhouseConfig"
|
const clickhouseConfigCode = "clickhouseConfig"
|
||||||
@@ -29,18 +34,18 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
|
|||||||
this.ErrorPage(err)
|
this.ErrorPage(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default", Scheme: "http"}
|
cfg := &systemconfigs.ClickHouseSetting{Port: 8443, Database: "default", Scheme: "https"}
|
||||||
if len(resp.ValueJSON) > 0 {
|
if len(resp.ValueJSON) > 0 {
|
||||||
_ = json.Unmarshal(resp.ValueJSON, cfg)
|
_ = json.Unmarshal(resp.ValueJSON, cfg)
|
||||||
}
|
}
|
||||||
if cfg.Port <= 0 {
|
if cfg.Port <= 0 {
|
||||||
cfg.Port = 8123
|
cfg.Port = 8443
|
||||||
}
|
}
|
||||||
if cfg.Database == "" {
|
if cfg.Database == "" {
|
||||||
cfg.Database = "default"
|
cfg.Database = "default"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Scheme) == "" {
|
if strings.TrimSpace(cfg.Scheme) == "" {
|
||||||
cfg.Scheme = "http"
|
cfg.Scheme = "https"
|
||||||
}
|
}
|
||||||
this.Data["config"] = map[string]interface{}{
|
this.Data["config"] = map[string]interface{}{
|
||||||
"host": cfg.Host,
|
"host": cfg.Host,
|
||||||
@@ -49,9 +54,17 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
|
|||||||
"password": cfg.Password,
|
"password": cfg.Password,
|
||||||
"database": cfg.Database,
|
"database": cfg.Database,
|
||||||
"scheme": cfg.Scheme,
|
"scheme": cfg.Scheme,
|
||||||
"tlsSkipVerify": cfg.TLSSkipVerify,
|
|
||||||
"tlsServerName": cfg.TLSServerName,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 自动检测连接状态
|
||||||
|
connStatus := "unconfigured" // unconfigured / connected / disconnected
|
||||||
|
connError := ""
|
||||||
|
if strings.TrimSpace(cfg.Host) != "" {
|
||||||
|
connStatus, connError = this.probeClickHouse(cfg)
|
||||||
|
}
|
||||||
|
this.Data["connStatus"] = connStatus
|
||||||
|
this.Data["connError"] = connError
|
||||||
|
|
||||||
this.Show()
|
this.Show()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,20 +75,18 @@ func (this *ClickHouseAction) RunPost(params struct {
|
|||||||
Password string
|
Password string
|
||||||
Database string
|
Database string
|
||||||
Scheme string
|
Scheme string
|
||||||
TLSSkipVerify bool
|
|
||||||
TLSServerName string
|
|
||||||
|
|
||||||
Must *actions.Must
|
Must *actions.Must
|
||||||
}) {
|
}) {
|
||||||
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
|
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
|
||||||
if params.Port <= 0 {
|
|
||||||
params.Port = 8123
|
|
||||||
}
|
|
||||||
if params.Database == "" {
|
if params.Database == "" {
|
||||||
params.Database = "default"
|
params.Database = "default"
|
||||||
}
|
}
|
||||||
if params.Scheme != "https" {
|
if params.Scheme != "http" {
|
||||||
params.Scheme = "http"
|
params.Scheme = "https"
|
||||||
|
}
|
||||||
|
if params.Port <= 0 {
|
||||||
|
params.Port = 8443
|
||||||
}
|
}
|
||||||
password := params.Password
|
password := params.Password
|
||||||
if password == "" {
|
if password == "" {
|
||||||
@@ -94,8 +105,8 @@ func (this *ClickHouseAction) RunPost(params struct {
|
|||||||
Password: password,
|
Password: password,
|
||||||
Database: params.Database,
|
Database: params.Database,
|
||||||
Scheme: params.Scheme,
|
Scheme: params.Scheme,
|
||||||
TLSSkipVerify: params.TLSSkipVerify,
|
TLSSkipVerify: true,
|
||||||
TLSServerName: strings.TrimSpace(params.TLSServerName),
|
TLSServerName: "",
|
||||||
}
|
}
|
||||||
valueJSON, err := json.Marshal(cfg)
|
valueJSON, err := json.Marshal(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -112,3 +123,51 @@ func (this *ClickHouseAction) RunPost(params struct {
|
|||||||
}
|
}
|
||||||
this.Success()
|
this.Success()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// probeClickHouse 快速检测 ClickHouse 连接状态(SELECT 1)
|
||||||
|
func (this *ClickHouseAction) probeClickHouse(cfg *systemconfigs.ClickHouseSetting) (status string, errMsg string) {
|
||||||
|
scheme := strings.ToLower(strings.TrimSpace(cfg.Scheme))
|
||||||
|
if scheme == "" {
|
||||||
|
scheme = "https"
|
||||||
|
}
|
||||||
|
port := cfg.Port
|
||||||
|
if port <= 0 {
|
||||||
|
port = 8443
|
||||||
|
}
|
||||||
|
db := cfg.Database
|
||||||
|
if db == "" {
|
||||||
|
db = "default"
|
||||||
|
}
|
||||||
|
|
||||||
|
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s", scheme, cfg.Host, port, db)
|
||||||
|
|
||||||
|
transport := &http.Transport{}
|
||||||
|
if scheme == "https" {
|
||||||
|
transport.TLSClientConfig = &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 3 * time.Second,
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodGet, testURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "disconnected", err.Error()
|
||||||
|
}
|
||||||
|
if cfg.User != "" || cfg.Password != "" {
|
||||||
|
req.SetBasicAuth(cfg.User, cfg.Password)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "disconnected", err.Error()
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "disconnected", fmt.Sprintf("HTTP %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
return "connected", ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ func (this *ClickHouseAction) Init() {
|
|||||||
func (this *ClickHouseAction) RunGet(params struct{}) {
|
func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||||
this.Data["mainTab"] = "clickhouse"
|
this.Data["mainTab"] = "clickhouse"
|
||||||
this.Data["config"] = map[string]interface{}{
|
this.Data["config"] = map[string]interface{}{
|
||||||
"host": "", "port": 8123, "user": "", "password": "", "database": "default",
|
"host": "", "port": 8443, "user": "", "password": "", "database": "default", "scheme": "https",
|
||||||
}
|
}
|
||||||
this.Show()
|
this.Show()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ func init() {
|
|||||||
Get("/logs", new(LogsAction)).
|
Get("/logs", new(LogsAction)).
|
||||||
Post("/status", new(StatusAction)).
|
Post("/status", new(StatusAction)).
|
||||||
GetPost("/clickhouse", new(ClickHouseAction)).
|
GetPost("/clickhouse", new(ClickHouseAction)).
|
||||||
|
Post("/testClickhouse", new(TestClickHouseAction)).
|
||||||
EndAll()
|
EndAll()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
92
EdgeAdmin/internal/web/actions/default/db/testClickhouse.go
Normal file
92
EdgeAdmin/internal/web/actions/default/db/testClickhouse.go
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
//go:build plus
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||||
|
"github.com/iwind/TeaGo/actions"
|
||||||
|
)
|
||||||
|
|
||||||
|
type TestClickHouseAction struct {
|
||||||
|
actionutils.ParentAction
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *TestClickHouseAction) Init() {
|
||||||
|
this.Nav("db", "db", "clickhouse")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *TestClickHouseAction) RunPost(params struct {
|
||||||
|
Host string
|
||||||
|
Port int
|
||||||
|
User string
|
||||||
|
Password string
|
||||||
|
Database string
|
||||||
|
Scheme string
|
||||||
|
|
||||||
|
Must *actions.Must
|
||||||
|
}) {
|
||||||
|
params.Must.
|
||||||
|
Field("host", params.Host).
|
||||||
|
Require("请输入 ClickHouse 连接地址")
|
||||||
|
|
||||||
|
if params.Database == "" {
|
||||||
|
params.Database = "default"
|
||||||
|
}
|
||||||
|
scheme := "https"
|
||||||
|
if strings.EqualFold(params.Scheme, "http") {
|
||||||
|
scheme = "http"
|
||||||
|
}
|
||||||
|
if params.Port <= 0 {
|
||||||
|
params.Port = 8443
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构造测试请求
|
||||||
|
testURL := fmt.Sprintf("%s://%s:%d/?query=SELECT+1&database=%s",
|
||||||
|
scheme, params.Host, params.Port, params.Database)
|
||||||
|
|
||||||
|
transport := &http.Transport{}
|
||||||
|
if scheme == "https" {
|
||||||
|
transport.TLSClientConfig = &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 5 * time.Second,
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodGet, testURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
this.Fail("请求构造失败: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if params.User != "" || params.Password != "" {
|
||||||
|
req.SetBasicAuth(params.User, params.Password)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
this.Fail("连接失败: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
msg := strings.TrimSpace(string(body))
|
||||||
|
if len(msg) > 200 {
|
||||||
|
msg = msg[:200] + "..."
|
||||||
|
}
|
||||||
|
this.Fail(fmt.Sprintf("ClickHouse 返回 HTTP %d: %s", resp.StatusCode, msg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
this.Success()
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
//go:build !plus
|
||||||
|
|
||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||||
|
"github.com/iwind/TeaGo/actions"
|
||||||
|
)
|
||||||
|
|
||||||
|
type TestClickHouseAction struct {
|
||||||
|
actionutils.ParentAction
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *TestClickHouseAction) Init() {
|
||||||
|
this.Nav("db", "db", "clickhouse")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *TestClickHouseAction) RunPost(params struct {
|
||||||
|
Host string
|
||||||
|
Must *actions.Must
|
||||||
|
}) {
|
||||||
|
this.Fail("请使用商业版以测试 ClickHouse 连接")
|
||||||
|
}
|
||||||
@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
|
|||||||
<div>
|
<div>
|
||||||
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
||||||
|
|
||||||
<!-- 网站 -->
|
|
||||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
|
|
||||||
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
|
|
||||||
|
|
||||||
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
||||||
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
||||||
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
|
|||||||
</span>
|
</span>
|
||||||
|
|
||||||
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
||||||
|
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
|
||||||
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
||||||
</div>
|
</div>
|
||||||
</div>`
|
</div>`
|
||||||
|
|||||||
@@ -10422,9 +10422,7 @@ Vue.component("http-access-log-box", {
|
|||||||
<div>
|
<div>
|
||||||
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
|
||||||
|
|
||||||
<!-- 网站 -->
|
|
||||||
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
|
|
||||||
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
|
|
||||||
|
|
||||||
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
|
||||||
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>"<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}" </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
|
||||||
@@ -10452,6 +10450,7 @@ Vue.component("http-access-log-box", {
|
|||||||
</span>
|
</span>
|
||||||
|
|
||||||
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small"> ({{accessLog.humanTime}})</span>
|
||||||
|
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
|
||||||
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
<a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
|
||||||
</div>
|
</div>
|
||||||
</div>`
|
</div>`
|
||||||
|
|||||||
@@ -1,70 +1,76 @@
|
|||||||
{$layout}
|
{$layout}
|
||||||
{$template "menu"}
|
{$template "menu"}
|
||||||
|
|
||||||
<h3>ClickHouse 配置</h3>
|
<div style="display:flex;align-items:baseline;gap:0.8em;margin-bottom:0.5em">
|
||||||
<p class="comment">用于访问日志列表查询(logs_ingest 表)。配置后,访问日志列表将优先从 ClickHouse 读取;不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。</p>
|
<h3 style="margin:0">ClickHouse 配置</h3>
|
||||||
|
<span v-if="connStatus === 'connected'" class="ui label green tiny" style="vertical-align:baseline">
|
||||||
|
<i class="icon circle" style="margin-right:0.25em"></i>已连接
|
||||||
|
</span>
|
||||||
|
<span v-else-if="connStatus === 'disconnected'" class="ui label red tiny" style="vertical-align:baseline"
|
||||||
|
:title="connError">
|
||||||
|
<i class="icon circle" style="margin-right:0.25em"></i>已断开
|
||||||
|
</span>
|
||||||
|
<span v-else class="ui label grey tiny" style="vertical-align:baseline">
|
||||||
|
<i class="icon circle outline" style="margin-right:0.25em"></i>未配置
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<p class="comment">配置后,需要在网站列表-日志策略中创建"文件+ClickHouse"的策略,才能将访问日志绕过api组件直接由边缘节点发送给ClickHouse。</p>
|
||||||
|
|
||||||
<form method="post" class="ui form" data-tea-action="$" data-tea-success="success" @submit.prevent="onSubmit">
|
<form method="post" class="ui form" @submit.prevent="onSubmit">
|
||||||
<csrf-token></csrf-token>
|
<csrf-token></csrf-token>
|
||||||
<table class="ui table definition selectable">
|
<table class="ui table definition selectable">
|
||||||
<tr>
|
<tr>
|
||||||
<td class="title">连接地址(Host)</td>
|
<td class="title">连接地址(Host)</td>
|
||||||
<td>
|
<td>
|
||||||
<input type="text" name="host" maxlength="200" ref="focus" placeholder="如 127.0.0.1 或 clickhouse.example.com" :value="config.host"/>
|
<input type="text" name="host" maxlength="200" ref="focus"
|
||||||
|
placeholder="如 127.0.0.1 或 clickhouse.example.com" v-model="form.host" />
|
||||||
<p class="comment">ClickHouse 服务器地址。</p>
|
<p class="comment">ClickHouse 服务器地址。</p>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>协议(Scheme)</td>
|
<td>协议(Scheme)</td>
|
||||||
<td>
|
<td>
|
||||||
<select name="scheme" class="ui dropdown auto-width">
|
<select name="scheme" class="ui dropdown auto-width" v-model="form.scheme">
|
||||||
<option value="http" :selected="config.scheme != 'https'">HTTP</option>
|
<option value="http">HTTP</option>
|
||||||
<option value="https" :selected="config.scheme == 'https'">HTTPS</option>
|
<option value="https">HTTPS</option>
|
||||||
</select>
|
</select>
|
||||||
<p class="comment">默认 HTTP;选择 HTTPS 时将启用 TLS 连接。</p>
|
<p class="comment">默认 HTTPS;当前后台固定跳过证书校验。</p>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>端口(Port)</td>
|
<td>端口(Port)</td>
|
||||||
<td>
|
<td>
|
||||||
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/>
|
<input type="number" name="port" min="1" max="65535" style="width:6em" v-model.number="form.port" />
|
||||||
<p class="comment">接口端口,HTTP 默认 8123,HTTPS 常用 8443(以你的 ClickHouse 实际配置为准)。</p>
|
<p class="comment">接口端口默认 8443(HTTPS);请与 ClickHouse 实际开放端口保持一致。</p>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>用户名(User)</td>
|
<td>用户名(User)</td>
|
||||||
<td>
|
<td>
|
||||||
<input type="text" name="user" maxlength="100" :value="config.user"/>
|
<input type="text" name="user" maxlength="100" v-model="form.user" />
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>密码(Password)</td>
|
<td>密码(Password)</td>
|
||||||
<td>
|
<td>
|
||||||
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" value=""/>
|
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" v-model="form.password" />
|
||||||
<p class="comment">留空则不修改已保存的密码。</p>
|
<p class="comment">留空则不修改已保存的密码。</p>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
|
||||||
<td>TLS 跳过证书校验</td>
|
|
||||||
<td>
|
|
||||||
<checkbox name="tlsSkipVerify" value="1" :checked="config.tlsSkipVerify"></checkbox>
|
|
||||||
<p class="comment">仅测试环境建议开启;生产建议关闭并使用受信任证书。</p>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>TLS Server Name</td>
|
|
||||||
<td>
|
|
||||||
<input type="text" name="tlsServerName" maxlength="200" placeholder="可选:证书校验域名(SNI)" :value="config.tlsServerName"/>
|
|
||||||
<p class="comment">可选;当 ClickHouse 证书域名与连接 Host 不一致时使用。</p>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
<tr>
|
||||||
<td>数据库名(Database)</td>
|
<td>数据库名(Database)</td>
|
||||||
<td>
|
<td>
|
||||||
<input type="text" name="database" maxlength="100" placeholder="default" :value="config.database"/>
|
<input type="text" name="database" maxlength="100" placeholder="default" v-model="form.database" />
|
||||||
<p class="comment">logs_ingest 表所在库,默认 default。</p>
|
<p class="comment">logs_ingest 表所在库,默认 default。</p>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
</table>
|
</table>
|
||||||
|
<div style="display:flex; align-items:center; gap:0.5em; margin-top:1em">
|
||||||
<submit-btn></submit-btn>
|
<submit-btn></submit-btn>
|
||||||
|
<button type="button" class="ui button basic blue" @click="testConnection" :disabled="isTesting">
|
||||||
|
<i :class="isTesting ? 'icon spinner loading' : 'icon plug'"></i> 测试连接
|
||||||
|
</button>
|
||||||
|
<span v-if="testResult"
|
||||||
|
:style="{color: testOk ? '#21ba45' : '#db2828', fontWeight:'bold'}">{{testResult}}</span>
|
||||||
|
</div>
|
||||||
</form>
|
</form>
|
||||||
@@ -1,4 +1,22 @@
|
|||||||
Tea.context(function () {
|
Tea.context(function () {
|
||||||
|
var config = this.config || {}
|
||||||
|
this.form = {
|
||||||
|
host: config.host || "",
|
||||||
|
scheme: config.scheme || "https",
|
||||||
|
port: config.port > 0 ? config.port : 8443,
|
||||||
|
user: config.user || "",
|
||||||
|
password: "",
|
||||||
|
database: config.database || "default",
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isTesting = false
|
||||||
|
this.testResult = ""
|
||||||
|
this.testOk = false
|
||||||
|
|
||||||
|
// 页面加载时的连接状态(后端自动检测)
|
||||||
|
this.connStatus = this.connStatus || "unconfigured"
|
||||||
|
this.connError = this.connError || ""
|
||||||
|
|
||||||
this.success = function () {
|
this.success = function () {
|
||||||
teaweb.success("保存成功")
|
teaweb.success("保存成功")
|
||||||
}
|
}
|
||||||
@@ -9,4 +27,50 @@ Tea.context(function () {
|
|||||||
Tea.Vue.success()
|
Tea.Vue.success()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
this.testConnection = function () {
|
||||||
|
var that = Tea.Vue
|
||||||
|
that.isTesting = true
|
||||||
|
that.testResult = ""
|
||||||
|
|
||||||
|
var form = document.querySelector("form")
|
||||||
|
var fd = new FormData(form)
|
||||||
|
fd.set("host", that.form.host || "")
|
||||||
|
fd.set("scheme", that.form.scheme || "https")
|
||||||
|
fd.set("port", String(that.form.port > 0 ? that.form.port : 8443))
|
||||||
|
fd.set("user", that.form.user || "")
|
||||||
|
fd.set("password", that.form.password || "")
|
||||||
|
fd.set("database", that.form.database || "default")
|
||||||
|
|
||||||
|
var xhr = new XMLHttpRequest()
|
||||||
|
xhr.open("POST", Tea.url("/db/testClickhouse"), true)
|
||||||
|
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest")
|
||||||
|
xhr.timeout = 10000
|
||||||
|
xhr.onload = function () {
|
||||||
|
that.isTesting = false
|
||||||
|
try {
|
||||||
|
var resp = JSON.parse(xhr.responseText)
|
||||||
|
if (resp.code === 200) {
|
||||||
|
that.testOk = true
|
||||||
|
that.testResult = "✅ 连接成功"
|
||||||
|
} else {
|
||||||
|
that.testOk = false
|
||||||
|
that.testResult = "❌ " + (resp.message || "连接失败")
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
that.testOk = false
|
||||||
|
that.testResult = "❌ 响应解析失败"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
xhr.onerror = function () {
|
||||||
|
that.isTesting = false
|
||||||
|
that.testOk = false
|
||||||
|
that.testResult = "❌ 网络请求失败"
|
||||||
|
}
|
||||||
|
xhr.ontimeout = function () {
|
||||||
|
that.isTesting = false
|
||||||
|
that.testOk = false
|
||||||
|
that.testResult = "❌ 请求超时"
|
||||||
|
}
|
||||||
|
xhr.send(fd)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ type NSNodeConfig struct {
|
|||||||
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
|
AccessLogRef *NSAccessLogRef `yaml:"accessLogRef" json:"accessLogRef"`
|
||||||
AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"`
|
AccessLogWriteTargets *serverconfigs.AccessLogWriteTargets `yaml:"accessLogWriteTargets" json:"accessLogWriteTargets"`
|
||||||
AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"`
|
AccessLogFilePath string `yaml:"accessLogFilePath" json:"accessLogFilePath"`
|
||||||
|
AccessLogRotate *serverconfigs.AccessLogRotateConfig `yaml:"accessLogRotate" json:"accessLogRotate"`
|
||||||
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
|
RecursionConfig *NSRecursionConfig `yaml:"recursionConfig" json:"recursionConfig"`
|
||||||
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
|
DDoSProtection *ddosconfigs.ProtectionConfig `yaml:"ddosProtection" json:"ddosProtection"`
|
||||||
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`
|
AllowedIPs []string `yaml:"allowedIPs" json:"allowedIPs"`
|
||||||
|
|||||||
@@ -2,8 +2,65 @@
|
|||||||
|
|
||||||
package serverconfigs
|
package serverconfigs
|
||||||
|
|
||||||
|
const (
|
||||||
|
DefaultAccessLogRotateMaxSizeMB = 256
|
||||||
|
DefaultAccessLogRotateMaxBackups = 14
|
||||||
|
DefaultAccessLogRotateMaxAgeDays = 7
|
||||||
|
)
|
||||||
|
|
||||||
|
// AccessLogRotateConfig 文件轮转配置。
|
||||||
|
type AccessLogRotateConfig struct {
|
||||||
|
MaxSizeMB int `yaml:"maxSizeMB" json:"maxSizeMB"` // 单文件最大大小(MB)
|
||||||
|
MaxBackups int `yaml:"maxBackups" json:"maxBackups"` // 保留历史文件数
|
||||||
|
MaxAgeDays int `yaml:"maxAgeDays" json:"maxAgeDays"` // 保留天数
|
||||||
|
Compress *bool `yaml:"compress" json:"compress"` // 是否压缩历史文件
|
||||||
|
LocalTime *bool `yaml:"localTime" json:"localTime"` // 轮转时间使用本地时区
|
||||||
|
}
|
||||||
|
|
||||||
// AccessLogFileStorageConfig 文件存储配置
|
// AccessLogFileStorageConfig 文件存储配置
|
||||||
type AccessLogFileStorageConfig struct {
|
type AccessLogFileStorageConfig struct {
|
||||||
Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second}
|
Path string `yaml:"path" json:"path"` // 文件路径,支持变量:${year|month|week|day|hour|minute|second}
|
||||||
AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录
|
AutoCreate bool `yaml:"autoCreate" json:"autoCreate"` // 是否自动创建目录
|
||||||
|
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 文件轮转配置
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDefaultAccessLogRotateConfig 默认轮转配置。
|
||||||
|
func NewDefaultAccessLogRotateConfig() *AccessLogRotateConfig {
|
||||||
|
compress := false
|
||||||
|
localTime := true
|
||||||
|
return &AccessLogRotateConfig{
|
||||||
|
MaxSizeMB: DefaultAccessLogRotateMaxSizeMB,
|
||||||
|
MaxBackups: DefaultAccessLogRotateMaxBackups,
|
||||||
|
MaxAgeDays: DefaultAccessLogRotateMaxAgeDays,
|
||||||
|
Compress: &compress,
|
||||||
|
LocalTime: &localTime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize 归一化轮转配置,空值/非法值回退默认。
|
||||||
|
func (c *AccessLogRotateConfig) Normalize() *AccessLogRotateConfig {
|
||||||
|
defaultConfig := NewDefaultAccessLogRotateConfig()
|
||||||
|
if c == nil {
|
||||||
|
return defaultConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.MaxSizeMB > 0 {
|
||||||
|
defaultConfig.MaxSizeMB = c.MaxSizeMB
|
||||||
|
}
|
||||||
|
if c.MaxBackups > 0 {
|
||||||
|
defaultConfig.MaxBackups = c.MaxBackups
|
||||||
|
}
|
||||||
|
if c.MaxAgeDays > 0 {
|
||||||
|
defaultConfig.MaxAgeDays = c.MaxAgeDays
|
||||||
|
}
|
||||||
|
if c.Compress != nil {
|
||||||
|
v := *c.Compress
|
||||||
|
defaultConfig.Compress = &v
|
||||||
|
}
|
||||||
|
if c.LocalTime != nil {
|
||||||
|
v := *c.LocalTime
|
||||||
|
defaultConfig.LocalTime = &v
|
||||||
|
}
|
||||||
|
|
||||||
|
return defaultConfig
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ func NewGlobalServerConfig() *GlobalServerConfig {
|
|||||||
config.HTTPAccessLog.EnableResponseHeaders = true
|
config.HTTPAccessLog.EnableResponseHeaders = true
|
||||||
config.HTTPAccessLog.EnableCookies = true
|
config.HTTPAccessLog.EnableCookies = true
|
||||||
config.HTTPAccessLog.EnableServerNotFound = true
|
config.HTTPAccessLog.EnableServerNotFound = true
|
||||||
|
config.HTTPAccessLog.Rotate = NewDefaultAccessLogRotateConfig()
|
||||||
|
|
||||||
config.Log.RecordServerError = false
|
config.Log.RecordServerError = false
|
||||||
|
|
||||||
@@ -79,6 +80,7 @@ type GlobalServerConfig struct {
|
|||||||
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
|
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
|
||||||
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse(双写/单写)
|
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse(双写/单写)
|
||||||
FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用)
|
FilePath string `yaml:"filePath" json:"filePath"` // 公用日志策略文件路径(用于节点侧复用)
|
||||||
|
Rotate *AccessLogRotateConfig `yaml:"rotate" json:"rotate"` // 本地日志轮转配置(lumberjack)
|
||||||
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
|
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
|
||||||
|
|
||||||
Stat struct {
|
Stat struct {
|
||||||
|
|||||||
@@ -107,11 +107,12 @@ function copy_fluent_bit_assets() {
|
|||||||
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
|
||||||
|
|
||||||
rm -rf "$FLUENT_DIST"
|
rm -rf "$FLUENT_DIST"
|
||||||
mkdir -p "$FLUENT_DIST"
|
mkdir -p "$FLUENT_DIST"
|
||||||
|
|
||||||
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
|
for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
|
||||||
if [ -f "$FLUENT_ROOT/$file" ]; then
|
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||||
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||||
fi
|
fi
|
||||||
@@ -129,6 +130,43 @@ function copy_fluent_bit_assets() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
rm -f "$FLUENT_DIST/.gitignore"
|
||||||
|
rm -f "$FLUENT_DIST"/logs.db*
|
||||||
|
rm -rf "$FLUENT_DIST/storage"
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function verify_fluent_bit_package_matrix() {
|
||||||
|
FLUENT_ROOT=$1
|
||||||
|
ARCH=$2
|
||||||
|
REQUIRED_FILES=()
|
||||||
|
if [ "$ARCH" = "amd64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||||
|
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||||
|
)
|
||||||
|
elif [ "$ARCH" = "arm64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||||
|
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
MISSING=0
|
||||||
|
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||||
|
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||||
|
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||||
|
MISSING=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$MISSING" -ne 0 ]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ require (
|
|||||||
github.com/mdlayher/netlink v1.7.2
|
github.com/mdlayher/netlink v1.7.2
|
||||||
github.com/miekg/dns v1.1.58
|
github.com/miekg/dns v1.1.58
|
||||||
github.com/shirou/gopsutil/v3 v3.24.2
|
github.com/shirou/gopsutil/v3 v3.24.2
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||||
golang.org/x/sys v0.38.0
|
golang.org/x/sys v0.38.0
|
||||||
google.golang.org/grpc v1.78.0
|
google.golang.org/grpc v1.78.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
|||||||
@@ -111,6 +111,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
|
|||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||||
|
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||||
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
|
"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
|
||||||
|
"gopkg.in/natefinch/lumberjack.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -30,18 +32,23 @@ func SharedDNSFileWriter() *DNSFileWriter {
|
|||||||
return sharedDNSFileWriter
|
return sharedDNSFileWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集.
|
// DNSFileWriter 将 DNS 访问日志以 JSON Lines 写入本地文件,供 Fluent Bit 采集。
|
||||||
|
// 文件轮转由 lumberjack 内建完成。
|
||||||
type DNSFileWriter struct {
|
type DNSFileWriter struct {
|
||||||
dir string
|
dir string
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
file *os.File
|
file *lumberjack.Logger
|
||||||
|
rotateConfig *serverconfigs.AccessLogRotateConfig
|
||||||
inited bool
|
inited bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDNSFileWriter 创建 DNS 本地日志写入器.
|
// NewDNSFileWriter 创建 DNS 本地日志写入器.
|
||||||
func NewDNSFileWriter() *DNSFileWriter {
|
func NewDNSFileWriter() *DNSFileWriter {
|
||||||
logDir := resolveDefaultDNSLogDir()
|
logDir := resolveDefaultDNSLogDir()
|
||||||
return &DNSFileWriter{dir: logDir}
|
return &DNSFileWriter{
|
||||||
|
dir: logDir,
|
||||||
|
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveDefaultDNSLogDir() string {
|
func resolveDefaultDNSLogDir() string {
|
||||||
@@ -102,6 +109,25 @@ func (w *DNSFileWriter) SetDir(dir string) {
|
|||||||
w.dir = dir
|
w.dir = dir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetRotateConfig 更新日志轮转配置并重建 writer。
|
||||||
|
func (w *DNSFileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
|
||||||
|
normalized := config.Normalize()
|
||||||
|
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
|
if equalDNSRotateConfig(w.rotateConfig, normalized) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if w.file != nil {
|
||||||
|
_ = w.file.Close()
|
||||||
|
w.file = nil
|
||||||
|
}
|
||||||
|
w.inited = false
|
||||||
|
w.rotateConfig = normalized
|
||||||
|
}
|
||||||
|
|
||||||
// EnsureInit 在启动时预创建目录与 access.log.
|
// EnsureInit 在启动时预创建目录与 access.log.
|
||||||
func (w *DNSFileWriter) EnsureInit() error {
|
func (w *DNSFileWriter) EnsureInit() error {
|
||||||
if w.dir == "" {
|
if w.dir == "" {
|
||||||
@@ -127,13 +153,16 @@ func (w *DNSFileWriter) init() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fp, err := os.OpenFile(filepath.Join(w.dir, "access.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
rotateConfig := w.rotateConfig.Normalize()
|
||||||
if err != nil {
|
w.file = &lumberjack.Logger{
|
||||||
remotelogs.Error("DNS_ACCESS_LOG_FILE", "open access.log failed: "+err.Error())
|
Filename: filepath.Join(w.dir, "access.log"),
|
||||||
return err
|
MaxSize: rotateConfig.MaxSizeMB,
|
||||||
|
MaxBackups: rotateConfig.MaxBackups,
|
||||||
|
MaxAge: rotateConfig.MaxAgeDays,
|
||||||
|
Compress: *rotateConfig.Compress,
|
||||||
|
LocalTime: *rotateConfig.LocalTime,
|
||||||
}
|
}
|
||||||
|
|
||||||
w.file = fp
|
|
||||||
w.inited = true
|
w.inited = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -148,14 +177,14 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
fp := w.file
|
file := w.file
|
||||||
w.mu.Unlock()
|
w.mu.Unlock()
|
||||||
if fp == nil {
|
if file == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, log := range logs {
|
for _, logItem := range logs {
|
||||||
ingestLog := FromNSAccessLog(log, clusterId)
|
ingestLog := FromNSAccessLog(logItem, clusterId)
|
||||||
if ingestLog == nil {
|
if ingestLog == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -163,11 +192,11 @@ func (w *DNSFileWriter) WriteBatch(logs []*pb.NSAccessLog, clusterId int64) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
_, _ = fp.Write(append(line, '\n'))
|
_, _ = file.Write(append(line, '\n'))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reopen 关闭并重新打开日志文件(配合 logrotate).
|
// Reopen 关闭并重建日志 writer(供兼容调用).
|
||||||
func (w *DNSFileWriter) Reopen() error {
|
func (w *DNSFileWriter) Reopen() error {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
if w.file != nil {
|
if w.file != nil {
|
||||||
@@ -197,3 +226,14 @@ func (w *DNSFileWriter) Close() error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func equalDNSRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
|
||||||
|
if left == nil || right == nil {
|
||||||
|
return left == right
|
||||||
|
}
|
||||||
|
return left.MaxSizeMB == right.MaxSizeMB &&
|
||||||
|
left.MaxBackups == right.MaxBackups &&
|
||||||
|
left.MaxAgeDays == right.MaxAgeDays &&
|
||||||
|
*left.Compress == *right.Compress &&
|
||||||
|
*left.LocalTime == *right.LocalTime
|
||||||
|
}
|
||||||
|
|||||||
@@ -110,6 +110,7 @@ func (this *NodeConfigManager) reload(config *dnsconfigs.NSNodeConfig) {
|
|||||||
teaconst.IsPlus = config.IsPlus
|
teaconst.IsPlus = config.IsPlus
|
||||||
|
|
||||||
accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
|
accesslogs.SharedDNSFileWriter().SetDirByPolicyPath(config.AccessLogFilePath)
|
||||||
|
accesslogs.SharedDNSFileWriter().SetRotateConfig(config.AccessLogRotate)
|
||||||
|
|
||||||
needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
|
needWriteFile := config.AccessLogWriteTargets == nil || config.AccessLogWriteTargets.File || config.AccessLogWriteTargets.ClickHouse
|
||||||
if needWriteFile {
|
if needWriteFile {
|
||||||
|
|||||||
@@ -181,11 +181,12 @@ function copy_fluent_bit_assets() {
|
|||||||
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
verify_fluent_bit_package_matrix "$FLUENT_ROOT" "$ARCH" || return 1
|
||||||
|
|
||||||
rm -rf "$FLUENT_DIST"
|
rm -rf "$FLUENT_DIST"
|
||||||
mkdir -p "$FLUENT_DIST"
|
mkdir -p "$FLUENT_DIST"
|
||||||
|
|
||||||
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
|
for file in fluent-bit.conf fluent-bit-dns.conf fluent-bit-https.conf fluent-bit-dns-https.conf fluent-bit-windows.conf fluent-bit-windows-https.conf parsers.conf clickhouse-upstream.conf clickhouse-upstream-windows.conf logrotate.conf README.md; do
|
||||||
if [ -f "$FLUENT_ROOT/$file" ]; then
|
if [ -f "$FLUENT_ROOT/$file" ]; then
|
||||||
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
|
||||||
fi
|
fi
|
||||||
@@ -203,6 +204,43 @@ function copy_fluent_bit_assets() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
rm -f "$FLUENT_DIST/.gitignore"
|
||||||
|
rm -f "$FLUENT_DIST"/logs.db*
|
||||||
|
rm -rf "$FLUENT_DIST/storage"
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function verify_fluent_bit_package_matrix() {
|
||||||
|
FLUENT_ROOT=$1
|
||||||
|
ARCH=$2
|
||||||
|
REQUIRED_FILES=()
|
||||||
|
if [ "$ARCH" = "amd64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-amd64/fluent-bit_4.2.2_amd64.deb"
|
||||||
|
"packages/linux-amd64/fluent-bit-4.2.2-1.x86_64.rpm"
|
||||||
|
)
|
||||||
|
elif [ "$ARCH" = "arm64" ]; then
|
||||||
|
REQUIRED_FILES=(
|
||||||
|
"packages/linux-arm64/fluent-bit_4.2.2_arm64.deb"
|
||||||
|
"packages/linux-arm64/fluent-bit-4.2.2-1.aarch64.rpm"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo "[error] unsupported arch for fluent-bit package validation: $ARCH"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
MISSING=0
|
||||||
|
for FILE in "${REQUIRED_FILES[@]}"; do
|
||||||
|
if [ ! -f "$FLUENT_ROOT/$FILE" ]; then
|
||||||
|
echo "[error] fluent-bit matrix package missing: $FLUENT_ROOT/$FILE"
|
||||||
|
MISSING=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$MISSING" -ne 0 ]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ require (
|
|||||||
github.com/shirou/gopsutil/v3 v3.22.2
|
github.com/shirou/gopsutil/v3 v3.22.2
|
||||||
github.com/tdewolff/minify/v2 v2.20.20
|
github.com/tdewolff/minify/v2 v2.20.20
|
||||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
|
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||||
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
|
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
|
||||||
golang.org/x/image v0.15.0
|
golang.org/x/image v0.15.0
|
||||||
golang.org/x/net v0.47.0
|
golang.org/x/net v0.47.0
|
||||||
|
|||||||
@@ -305,6 +305,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
|||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||||
|
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||||
|
"gopkg.in/natefinch/lumberjack.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -30,11 +32,13 @@ const (
|
|||||||
envLogDir = "EDGE_LOG_DIR"
|
envLogDir = "EDGE_LOG_DIR"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 logrotate 与 Fluent Bit 采集
|
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 Fluent Bit 采集。
|
||||||
|
// 文件轮转由 lumberjack 内建完成。
|
||||||
type FileWriter struct {
|
type FileWriter struct {
|
||||||
dir string
|
dir string
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
files map[string]*os.File // access.log, waf.log, error.log
|
files map[string]*lumberjack.Logger // access.log, waf.log, error.log
|
||||||
|
rotateConfig *serverconfigs.AccessLogRotateConfig
|
||||||
inited bool
|
inited bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,7 +47,8 @@ func NewFileWriter() *FileWriter {
|
|||||||
dir := resolveDefaultLogDir()
|
dir := resolveDefaultLogDir()
|
||||||
return &FileWriter{
|
return &FileWriter{
|
||||||
dir: dir,
|
dir: dir,
|
||||||
files: make(map[string]*os.File),
|
files: make(map[string]*lumberjack.Logger),
|
||||||
|
rotateConfig: serverconfigs.NewDefaultAccessLogRotateConfig(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,9 +102,9 @@ func (w *FileWriter) SetDir(dir string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, f := range w.files {
|
for name, file := range w.files {
|
||||||
if f != nil {
|
if file != nil {
|
||||||
_ = f.Close()
|
_ = file.Close()
|
||||||
}
|
}
|
||||||
w.files[name] = nil
|
w.files[name] = nil
|
||||||
}
|
}
|
||||||
@@ -107,6 +112,27 @@ func (w *FileWriter) SetDir(dir string) {
|
|||||||
w.dir = dir
|
w.dir = dir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetRotateConfig 更新日志轮转配置并重建 writer。
|
||||||
|
func (w *FileWriter) SetRotateConfig(config *serverconfigs.AccessLogRotateConfig) {
|
||||||
|
normalized := config.Normalize()
|
||||||
|
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
|
if equalRotateConfig(w.rotateConfig, normalized) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, file := range w.files {
|
||||||
|
if file != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
}
|
||||||
|
w.files[name] = nil
|
||||||
|
}
|
||||||
|
w.inited = false
|
||||||
|
w.rotateConfig = normalized
|
||||||
|
}
|
||||||
|
|
||||||
// IsEnabled 是否启用落盘(目录非空即视为启用)
|
// IsEnabled 是否启用落盘(目录非空即视为启用)
|
||||||
func (w *FileWriter) IsEnabled() bool {
|
func (w *FileWriter) IsEnabled() bool {
|
||||||
return w.dir != ""
|
return w.dir != ""
|
||||||
@@ -138,17 +164,24 @@ func (w *FileWriter) init() error {
|
|||||||
if w.files[name] != nil {
|
if w.files[name] != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
w.files[name] = w.newLogger(name)
|
||||||
if err != nil {
|
|
||||||
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
w.files[name] = fp
|
|
||||||
}
|
}
|
||||||
w.inited = true
|
w.inited = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *FileWriter) newLogger(fileName string) *lumberjack.Logger {
|
||||||
|
rotateConfig := w.rotateConfig.Normalize()
|
||||||
|
return &lumberjack.Logger{
|
||||||
|
Filename: filepath.Join(w.dir, fileName),
|
||||||
|
MaxSize: rotateConfig.MaxSizeMB,
|
||||||
|
MaxBackups: rotateConfig.MaxBackups,
|
||||||
|
MaxAge: rotateConfig.MaxAgeDays,
|
||||||
|
Compress: *rotateConfig.Compress,
|
||||||
|
LocalTime: *rotateConfig.LocalTime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Write 将一条访问日志按 log_type 写入对应文件(access.log / waf.log / error.log)
|
// Write 将一条访问日志按 log_type 写入对应文件(access.log / waf.log / error.log)
|
||||||
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
||||||
if w.dir == "" {
|
if w.dir == "" {
|
||||||
@@ -173,13 +206,12 @@ func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
|||||||
fileName = "access.log"
|
fileName = "access.log"
|
||||||
}
|
}
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
fp := w.files[fileName]
|
file := w.files[fileName]
|
||||||
w.mu.Unlock()
|
w.mu.Unlock()
|
||||||
if fp == nil {
|
if file == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// 单行写入,末尾换行,便于 Fluent Bit / JSON 解析
|
_, err = file.Write(append(line, '\n'))
|
||||||
_, err = fp.Write(append(line, '\n'))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
|
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
|
||||||
}
|
}
|
||||||
@@ -194,49 +226,49 @@ func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
accessFp := w.files["access.log"]
|
accessFile := w.files["access.log"]
|
||||||
wafFp := w.files["waf.log"]
|
wafFile := w.files["waf.log"]
|
||||||
errorFp := w.files["error.log"]
|
errorFile := w.files["error.log"]
|
||||||
w.mu.Unlock()
|
w.mu.Unlock()
|
||||||
if accessFp == nil && wafFp == nil && errorFp == nil {
|
if accessFile == nil && wafFile == nil && errorFile == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for _, l := range logs {
|
for _, logItem := range logs {
|
||||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
ingest, logType := FromHTTPAccessLog(logItem, clusterId)
|
||||||
line, err := json.Marshal(ingest)
|
line, err := json.Marshal(ingest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
line = append(line, '\n')
|
line = append(line, '\n')
|
||||||
var fp *os.File
|
var file *lumberjack.Logger
|
||||||
switch logType {
|
switch logType {
|
||||||
case LogTypeWAF:
|
case LogTypeWAF:
|
||||||
fp = wafFp
|
file = wafFile
|
||||||
case LogTypeError:
|
case LogTypeError:
|
||||||
fp = errorFp
|
file = errorFile
|
||||||
default:
|
default:
|
||||||
fp = accessFp
|
file = accessFile
|
||||||
}
|
}
|
||||||
if fp != nil {
|
if file != nil {
|
||||||
_, _ = fp.Write(line)
|
_, _ = file.Write(line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reopen 关闭并重新打开所有日志文件(供 logrotate copytruncate 或 SIGHUP 后重开句柄)
|
// Reopen 关闭并重建所有日志 writer(供 SIGHUP 兼容调用)。
|
||||||
func (w *FileWriter) Reopen() error {
|
func (w *FileWriter) Reopen() error {
|
||||||
if w.dir == "" {
|
if w.dir == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
for name, file := range w.files {
|
||||||
for name, f := range w.files {
|
if file != nil {
|
||||||
if f != nil {
|
_ = file.Close()
|
||||||
_ = f.Close()
|
|
||||||
w.files[name] = nil
|
w.files[name] = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
w.inited = false
|
w.inited = false
|
||||||
|
w.mu.Unlock()
|
||||||
return w.init()
|
return w.init()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,9 +277,9 @@ func (w *FileWriter) Close() error {
|
|||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
var lastErr error
|
var lastErr error
|
||||||
for name, f := range w.files {
|
for name, file := range w.files {
|
||||||
if f != nil {
|
if file != nil {
|
||||||
if err := f.Close(); err != nil {
|
if err := file.Close(); err != nil {
|
||||||
lastErr = err
|
lastErr = err
|
||||||
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
|
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
|
||||||
}
|
}
|
||||||
@@ -257,3 +289,14 @@ func (w *FileWriter) Close() error {
|
|||||||
w.inited = false
|
w.inited = false
|
||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func equalRotateConfig(left *serverconfigs.AccessLogRotateConfig, right *serverconfigs.AccessLogRotateConfig) bool {
|
||||||
|
if left == nil || right == nil {
|
||||||
|
return left == right
|
||||||
|
}
|
||||||
|
return left.MaxSizeMB == right.MaxSizeMB &&
|
||||||
|
left.MaxBackups == right.MaxBackups &&
|
||||||
|
left.MaxAgeDays == right.MaxAgeDays &&
|
||||||
|
*left.Compress == *right.Compress &&
|
||||||
|
*left.LocalTime == *right.LocalTime
|
||||||
|
}
|
||||||
|
|||||||
@@ -577,7 +577,7 @@ func (this *Node) listenSignals() {
|
|||||||
goman.New(func() {
|
goman.New(func() {
|
||||||
for sig := range queue {
|
for sig := range queue {
|
||||||
if sig == syscall.SIGHUP {
|
if sig == syscall.SIGHUP {
|
||||||
// 供 logrotate 等旋转日志后重开句柄
|
// 兼容 SIGHUP:重建本地日志 writer
|
||||||
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
|
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
|
||||||
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
|
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
|
||||||
}
|
}
|
||||||
@@ -890,6 +890,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig, reloadAll bool) {
|
|||||||
var accessLogFilePath string
|
var accessLogFilePath string
|
||||||
if config != nil && config.GlobalServerConfig != nil {
|
if config != nil && config.GlobalServerConfig != nil {
|
||||||
accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
|
accessLogFilePath = config.GlobalServerConfig.HTTPAccessLog.FilePath
|
||||||
|
accesslogs.SharedFileWriter().SetRotateConfig(config.GlobalServerConfig.HTTPAccessLog.Rotate)
|
||||||
}
|
}
|
||||||
accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)
|
accesslogs.SharedFileWriter().SetDirByPolicyPath(accessLogFilePath)
|
||||||
|
|
||||||
|
|||||||
1984
config.xml
Normal file
1984
config.xml
Normal file
File diff suppressed because it is too large
Load Diff
348
deploy.sh
Normal file
348
deploy.sh
Normal file
@@ -0,0 +1,348 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# CloudWAF 部署脚本 - Gname DNS 提供商更新
|
||||||
|
# 使用方法: ./deploy.sh [zip文件路径]
|
||||||
|
|
||||||
|
set -e # 遇到错误立即退出
|
||||||
|
|
||||||
|
# ========== 配置区域 ==========
|
||||||
|
# 支持环境变量和配置文件
|
||||||
|
# 从环境变量读取
|
||||||
|
INSTALL_BASE_DIR="${BRAND_INSTALL_PATH:-/usr/local/goedge}"
|
||||||
|
|
||||||
|
# 从配置文件读取(如果存在)
|
||||||
|
if [ -f "/etc/goedge/brand.conf" ]; then
|
||||||
|
source /etc/goedge/brand.conf
|
||||||
|
if [ -n "$BRAND_INSTALL_PATH" ]; then
|
||||||
|
INSTALL_BASE_DIR="$BRAND_INSTALL_PATH"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 如果通过参数传入 zip 文件路径,则使用参数;否则使用默认路径
|
||||||
|
ZIP_FILE_RAW="${1:-/tmp/edge-admin-linux-amd64-plus-v1.3.8.zip}"
|
||||||
|
# 获取绝对路径,防止 cd 到临时目录后找不到文件
|
||||||
|
if [[ "$ZIP_FILE_RAW" = /* ]]; then
|
||||||
|
ZIP_FILE="$ZIP_FILE_RAW"
|
||||||
|
else
|
||||||
|
ZIP_FILE="$(pwd)/$ZIP_FILE_RAW"
|
||||||
|
fi
|
||||||
|
|
||||||
|
TARGET_DIR="${INSTALL_BASE_DIR}/edge-admin"
|
||||||
|
BACKUP_DIR="${INSTALL_BASE_DIR}/backup_$(date +%Y%m%d_%H%M%S)"
|
||||||
|
TEMP_DIR="/tmp/edge-admin-update-$(date +%s)"
|
||||||
|
|
||||||
|
# 颜色输出
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
# 日志函数
|
||||||
|
log_info() {
|
||||||
|
echo -e "${GREEN}[INFO]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_warn() {
|
||||||
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_error() {
|
||||||
|
echo -e "${RED}[ERROR]${NC} $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 检查前置条件 ==========
|
||||||
|
check_prerequisites() {
|
||||||
|
log_info "检查前置条件..."
|
||||||
|
|
||||||
|
# 检查 zip 文件是否存在
|
||||||
|
if [ ! -f "$ZIP_FILE" ]; then
|
||||||
|
log_error "未找到更新包: $ZIP_FILE"
|
||||||
|
log_info "使用方法: $0 [zip文件路径]"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 检查目标目录是否存在
|
||||||
|
if [ ! -d "$TARGET_DIR" ]; then
|
||||||
|
log_error "目标目录不存在: $TARGET_DIR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 检查是否有足够的磁盘空间(至少 500MB)
|
||||||
|
available_space=$(df "$TARGET_DIR" | tail -1 | awk '{print $4}')
|
||||||
|
if [ "$available_space" -lt 512000 ]; then
|
||||||
|
log_warn "可用磁盘空间不足 500MB,可能影响部署"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "前置条件检查通过"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 1. 备份关键目录 ==========
|
||||||
|
backup_files() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 1: 备份关键目录"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
mkdir -p "$BACKUP_DIR"
|
||||||
|
|
||||||
|
# 备份 EdgeAdmin 配置
|
||||||
|
if [ -d "$TARGET_DIR/configs" ]; then
|
||||||
|
log_info "备份 EdgeAdmin 配置文件..."
|
||||||
|
cp -r "$TARGET_DIR/configs" "$BACKUP_DIR/edge-admin-configs"
|
||||||
|
log_info "✅ EdgeAdmin 配置已备份"
|
||||||
|
else
|
||||||
|
log_warn "EdgeAdmin configs 目录不存在,跳过备份"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 备份 EdgeAdmin 日志(可选)
|
||||||
|
if [ -d "$TARGET_DIR/logs" ]; then
|
||||||
|
log_info "备份 EdgeAdmin 日志文件..."
|
||||||
|
cp -r "$TARGET_DIR/logs" "$BACKUP_DIR/edge-admin-logs" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 备份 EdgeAPI 配置和数据
|
||||||
|
if [ -d "$TARGET_DIR/edge-api" ]; then
|
||||||
|
if [ -d "$TARGET_DIR/edge-api/configs" ]; then
|
||||||
|
log_info "备份 EdgeAPI 配置文件..."
|
||||||
|
cp -r "$TARGET_DIR/edge-api/configs" "$BACKUP_DIR/edge-api-configs"
|
||||||
|
log_info "✅ EdgeAPI 配置已备份"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -d "$TARGET_DIR/edge-api/logs" ]; then
|
||||||
|
log_info "备份 EdgeAPI 日志文件..."
|
||||||
|
cp -r "$TARGET_DIR/edge-api/logs" "$BACKUP_DIR/edge-api-logs" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -d "$TARGET_DIR/edge-api/data" ]; then
|
||||||
|
log_info "备份 EdgeAPI 数据文件..."
|
||||||
|
cp -r "$TARGET_DIR/edge-api/data" "$BACKUP_DIR/edge-api-data" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "✅ 备份完成,备份位置: $BACKUP_DIR"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 2. 停止服务 ==========
|
||||||
|
stop_services() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 2: 停止服务"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
|
||||||
|
log_info "停止 EdgeAdmin 服务..."
|
||||||
|
"$TARGET_DIR/bin/edge-admin" stop || true
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
# 检查是否还有进程在运行 (精确匹配二进制路径,避免杀死解析 zip 路径的脚本自身)
|
||||||
|
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
|
||||||
|
log_warn "检测到 edge-admin 进程仍在运行,尝试强制停止..."
|
||||||
|
pkill -9 -f "$TARGET_DIR/bin/edge-admin" || true
|
||||||
|
sleep 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "✅ 服务已停止"
|
||||||
|
else
|
||||||
|
log_warn "未找到 edge-admin 可执行文件,跳过停止步骤"
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 3. 解压新版本 ==========
|
||||||
|
extract_package() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 3: 解压新版本"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
log_info "解压更新包到临时目录: $TEMP_DIR"
|
||||||
|
mkdir -p "$TEMP_DIR"
|
||||||
|
cd "$TEMP_DIR"
|
||||||
|
|
||||||
|
if ! unzip -q "$ZIP_FILE"; then
|
||||||
|
log_error "解压失败,请检查 zip 文件是否损坏"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "✅ 解压完成"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 4. 替换文件 ==========
|
||||||
|
replace_files() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 4: 替换文件"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
# 替换 EdgeAdmin bin
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/bin" ]; then
|
||||||
|
log_info "替换 EdgeAdmin 可执行文件..."
|
||||||
|
cp -r "$TEMP_DIR/edge-admin/bin"/* "$TARGET_DIR/bin/"
|
||||||
|
log_info "✅ EdgeAdmin bin 已更新"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 替换 EdgeAdmin web(排除 tmp)
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/web" ]; then
|
||||||
|
log_info "替换 EdgeAdmin 前端文件..."
|
||||||
|
if command -v rsync > /dev/null; then
|
||||||
|
rsync -av --exclude='tmp' \
|
||||||
|
"$TEMP_DIR/edge-admin/web/" "$TARGET_DIR/web/"
|
||||||
|
else
|
||||||
|
# 如果没有 rsync,使用 cp
|
||||||
|
cp -r "$TEMP_DIR/edge-admin/web"/* "$TARGET_DIR/web/" 2>/dev/null || true
|
||||||
|
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
# 清空 tmp 目录
|
||||||
|
rm -rf "$TARGET_DIR/web/tmp"/* 2>/dev/null || true
|
||||||
|
log_info "✅ EdgeAdmin web 已更新"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 替换 EdgeAPI 文件
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/edge-api" ]; then
|
||||||
|
log_info "替换 EdgeAPI 文件..."
|
||||||
|
|
||||||
|
# 确保 edge-api 目录存在
|
||||||
|
mkdir -p "$TARGET_DIR/edge-api"
|
||||||
|
|
||||||
|
# 替换 bin
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/edge-api/bin" ]; then
|
||||||
|
mkdir -p "$TARGET_DIR/edge-api/bin"
|
||||||
|
cp -r "$TEMP_DIR/edge-admin/edge-api/bin"/* \
|
||||||
|
"$TARGET_DIR/edge-api/bin/" 2>/dev/null || true
|
||||||
|
log_info "✅ EdgeAPI bin 已更新"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 替换 deploy(节点安装包)
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/edge-api/deploy" ]; then
|
||||||
|
mkdir -p "$TARGET_DIR/edge-api/deploy"
|
||||||
|
cp -r "$TEMP_DIR/edge-admin/edge-api/deploy"/* \
|
||||||
|
"$TARGET_DIR/edge-api/deploy/" 2>/dev/null || true
|
||||||
|
log_info "✅ EdgeAPI deploy 已更新"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 替换 installers(安装工具)
|
||||||
|
if [ -d "$TEMP_DIR/edge-admin/edge-api/installers" ]; then
|
||||||
|
mkdir -p "$TARGET_DIR/edge-api/installers"
|
||||||
|
cp -r "$TEMP_DIR/edge-admin/edge-api/installers"/* \
|
||||||
|
"$TARGET_DIR/edge-api/installers/" 2>/dev/null || true
|
||||||
|
log_info "✅ EdgeAPI installers 已更新"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "✅ 文件替换完成"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 5. 恢复配置文件 ==========
|
||||||
|
restore_configs() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 5: 恢复配置文件"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
# 恢复 EdgeAdmin 配置
|
||||||
|
if [ -d "$BACKUP_DIR/edge-admin-configs" ]; then
|
||||||
|
log_info "恢复 EdgeAdmin 配置文件..."
|
||||||
|
cp -r "$BACKUP_DIR/edge-admin-configs"/* "$TARGET_DIR/configs/"
|
||||||
|
log_info "✅ EdgeAdmin 配置已恢复"
|
||||||
|
else
|
||||||
|
log_warn "未找到 EdgeAdmin 配置备份,请手动检查配置文件"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 恢复 EdgeAPI 配置
|
||||||
|
if [ -d "$BACKUP_DIR/edge-api-configs" ]; then
|
||||||
|
log_info "恢复 EdgeAPI 配置文件..."
|
||||||
|
mkdir -p "$TARGET_DIR/edge-api/configs"
|
||||||
|
cp -r "$BACKUP_DIR/edge-api-configs"/* "$TARGET_DIR/edge-api/configs/" 2>/dev/null || true
|
||||||
|
log_info "✅ EdgeAPI 配置已恢复"
|
||||||
|
else
|
||||||
|
log_warn "未找到 EdgeAPI 配置备份,如果存在 edge-api 目录,请手动检查配置文件"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "✅ 配置文件恢复完成"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 6. 清理临时文件 ==========
|
||||||
|
cleanup() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 6: 清理临时文件"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
rm -rf "$TEMP_DIR"
|
||||||
|
log_info "✅ 临时文件已清理"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 7. 启动服务 ==========
|
||||||
|
start_services() {
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "步骤 7: 启动服务"
|
||||||
|
log_info "=========================================="
|
||||||
|
|
||||||
|
if [ -f "$TARGET_DIR/bin/edge-admin" ]; then
|
||||||
|
log_info "启动 EdgeAdmin 服务..."
|
||||||
|
"$TARGET_DIR/bin/edge-admin" start
|
||||||
|
sleep 3
|
||||||
|
|
||||||
|
# 检查服务状态
|
||||||
|
if pgrep -f "$TARGET_DIR/bin/edge-admin" > /dev/null; then
|
||||||
|
log_info "✅ 服务启动成功"
|
||||||
|
else
|
||||||
|
log_warn "服务可能未正常启动,请检查日志"
|
||||||
|
log_info "查看日志: tail -f $TARGET_DIR/logs/run.log"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log_error "未找到 edge-admin 可执行文件"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========== 主函数 ==========
|
||||||
|
main() {
|
||||||
|
echo ""
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "CloudWAF 部署脚本 - Gname DNS 提供商更新"
|
||||||
|
log_info "=========================================="
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
log_info "部署配置:"
|
||||||
|
log_info " ZIP 文件: $ZIP_FILE"
|
||||||
|
log_info " 目标目录: $TARGET_DIR"
|
||||||
|
log_info " 备份目录: $BACKUP_DIR"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# 确认操作
|
||||||
|
read -p "确认开始部署? (y/N): " -n 1 -r
|
||||||
|
echo
|
||||||
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||||
|
log_info "部署已取消"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# 执行部署步骤
|
||||||
|
check_prerequisites
|
||||||
|
backup_files
|
||||||
|
stop_services
|
||||||
|
extract_package
|
||||||
|
replace_files
|
||||||
|
restore_configs
|
||||||
|
cleanup
|
||||||
|
start_services
|
||||||
|
|
||||||
|
# 完成
|
||||||
|
log_info "=========================================="
|
||||||
|
log_info "✅ 部署完成!"
|
||||||
|
log_info "=========================================="
|
||||||
|
echo ""
|
||||||
|
log_info "备份位置: $BACKUP_DIR"
|
||||||
|
log_info "请检查服务运行状态和日志"
|
||||||
|
echo ""
|
||||||
|
log_info "验证步骤:"
|
||||||
|
log_info "1. 检查服务进程: ps aux | grep \"$TARGET_DIR/bin/edge-admin\""
|
||||||
|
log_info "2. 检查服务日志: tail -f $TARGET_DIR/logs/run.log"
|
||||||
|
log_info "3. 访问管理后台,测试 Gname DNS 提供商功能"
|
||||||
|
echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
# 执行主函数
|
||||||
|
main
|
||||||
|
|
||||||
111
deploy/clickhouse/README.md
Normal file
111
deploy/clickhouse/README.md
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
# ClickHouse + Fluent Bit 使用手册(Ubuntu 22.04 / Amazon Linux 2023)
|
||||||
|
|
||||||
|
## 1. 支持范围
|
||||||
|
|
||||||
|
- Ubuntu 22.04
|
||||||
|
- Amazon Linux 2023(AWS)
|
||||||
|
|
||||||
|
安装脚本:`install_clickhouse_linux.sh`(自动识别上述系统)。
|
||||||
|
|
||||||
|
## 2. 安装 ClickHouse
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /path/to/waf-platform/deploy/clickhouse
|
||||||
|
chmod +x install_clickhouse_linux.sh
|
||||||
|
sudo ./install_clickhouse_linux.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
可选:安装时初始化 `default` 用户密码:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo CLICKHOUSE_DEFAULT_PASSWORD='YourStrongPassword' ./install_clickhouse_linux.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. 开启 HTTPS(默认仅 crt+key)
|
||||||
|
|
||||||
|
脚本默认生成 `server.crt + server.key`(带 SAN)并启用 8443:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /path/to/waf-platform/deploy/clickhouse
|
||||||
|
chmod +x configure_clickhouse_https.sh
|
||||||
|
sudo CH_HTTPS_PORT=8443 \
|
||||||
|
CH_CERT_CN=clickhouse.example.com \
|
||||||
|
CH_CERT_DNS=clickhouse.example.com \
|
||||||
|
CH_CERT_IP=<CLICKHOUSE_IP> \
|
||||||
|
./configure_clickhouse_https.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
使用已有证书:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo SRC_CERT=/path/to/server.crt \
|
||||||
|
SRC_KEY=/path/to/server.key \
|
||||||
|
CH_HTTPS_PORT=8443 \
|
||||||
|
./configure_clickhouse_https.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 初始化日志表(含优化)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /path/to/waf-platform/deploy/clickhouse
|
||||||
|
chmod +x init_waf_logs_tables.sh
|
||||||
|
sudo CH_HOST=127.0.0.1 \
|
||||||
|
CH_PORT=9000 \
|
||||||
|
CH_USER=default \
|
||||||
|
CH_PASSWORD='YourStrongPassword' \
|
||||||
|
CH_DATABASE=default \
|
||||||
|
./init_waf_logs_tables.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
说明:
|
||||||
|
- `init_waf_logs_tables.sql` 已内置主要优化(`CODEC`、`LowCardinality`、跳数索引)。
|
||||||
|
- `optimize_schema.sql` 主要用于历史表补齐优化,不是首次建表必需步骤。
|
||||||
|
|
||||||
|
## 5. 平台侧配置(EdgeAdmin)
|
||||||
|
|
||||||
|
在 ClickHouse 设置页配置:
|
||||||
|
|
||||||
|
- Host:ClickHouse 地址
|
||||||
|
- Port:`8443`
|
||||||
|
- Database:`default`
|
||||||
|
- Scheme:`https`
|
||||||
|
|
||||||
|
当前实现说明:
|
||||||
|
- 前端不再提供 `TLS跳过校验` 和 `TLS Server Name` 配置项。
|
||||||
|
- 后端固定 `TLSSkipVerify=true`(默认不校验证书)。
|
||||||
|
|
||||||
|
保存后点击“测试连接”。
|
||||||
|
|
||||||
|
## 6. Fluent Bit 配置方式
|
||||||
|
|
||||||
|
推荐平台托管模式(在线安装/升级 Node、DNS 时自动下发):
|
||||||
|
|
||||||
|
- `/etc/fluent-bit/fluent-bit.conf`
|
||||||
|
- `/etc/fluent-bit/.edge-managed.env`
|
||||||
|
- `/etc/fluent-bit/.edge-managed.json`
|
||||||
|
|
||||||
|
检查状态:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl status fluent-bit --no-pager
|
||||||
|
sudo cat /etc/fluent-bit/.edge-managed.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## 7. 验证与排障
|
||||||
|
|
||||||
|
查看 Fluent Bit 日志:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo journalctl -u fluent-bit -f
|
||||||
|
```
|
||||||
|
|
||||||
|
查看写入:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT count() FROM default.logs_ingest;
|
||||||
|
SELECT count() FROM default.dns_logs_ingest;
|
||||||
|
```
|
||||||
|
|
||||||
|
常见错误:
|
||||||
|
- `connection refused`:8443 未监听或网络未放行。
|
||||||
|
- `legacy Common Name`:证书缺 SAN,需重签。
|
||||||
227
deploy/clickhouse/configure_clickhouse_https.sh
Normal file
227
deploy/clickhouse/configure_clickhouse_https.sh
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ "${EUID}" -ne 0 ]]; then
|
||||||
|
echo "[ERROR] please run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f /etc/os-release ]]; then
|
||||||
|
echo "[ERROR] /etc/os-release not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source /etc/os-release
|
||||||
|
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
os_ver="${VERSION_ID:-}"
|
||||||
|
is_ubuntu22=false
|
||||||
|
is_amzn2023=false
|
||||||
|
|
||||||
|
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
|
||||||
|
is_ubuntu22=true
|
||||||
|
fi
|
||||||
|
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
|
||||||
|
is_amzn2023=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
|
||||||
|
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v openssl >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then
|
||||||
|
if [[ "${is_ubuntu22}" == "true" ]]; then
|
||||||
|
apt-get update -y
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y openssl curl ca-certificates
|
||||||
|
else
|
||||||
|
dnf makecache -y
|
||||||
|
dnf install -y openssl curl ca-certificates
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
CH_HTTPS_PORT="${CH_HTTPS_PORT:-8443}"
|
||||||
|
CH_LISTEN_HOST="${CH_LISTEN_HOST:-::}"
|
||||||
|
CH_CERT_CN="${CH_CERT_CN:-$(hostname -f 2>/dev/null || hostname)}"
|
||||||
|
CH_CERT_DNS="${CH_CERT_DNS:-}"
|
||||||
|
CH_CERT_IP="${CH_CERT_IP:-}"
|
||||||
|
CH_CERT_DAYS="${CH_CERT_DAYS:-825}"
|
||||||
|
CH_GENERATE_CA="${CH_GENERATE_CA:-false}"
|
||||||
|
|
||||||
|
SRC_CERT="${SRC_CERT:-}"
|
||||||
|
SRC_KEY="${SRC_KEY:-}"
|
||||||
|
SRC_CA="${SRC_CA:-}"
|
||||||
|
|
||||||
|
CH_DIR="/etc/clickhouse-server"
|
||||||
|
CH_CONFIG_D_DIR="${CH_DIR}/config.d"
|
||||||
|
PKI_DIR="${CH_DIR}/pki"
|
||||||
|
SERVER_CERT="${CH_DIR}/server.crt"
|
||||||
|
SERVER_KEY="${CH_DIR}/server.key"
|
||||||
|
CA_CERT="${CH_DIR}/ca.crt"
|
||||||
|
OVERRIDE_FILE="${CH_CONFIG_D_DIR}/waf-https.xml"
|
||||||
|
|
||||||
|
mkdir -p "${CH_CONFIG_D_DIR}" "${PKI_DIR}"
|
||||||
|
|
||||||
|
split_csv() {
|
||||||
|
local raw="$1"
|
||||||
|
if [[ -z "${raw}" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
IFS=',' read -r -a arr <<<"${raw}"
|
||||||
|
for item in "${arr[@]}"; do
|
||||||
|
item="$(echo "${item}" | xargs)"
|
||||||
|
if [[ -n "${item}" ]]; then
|
||||||
|
echo "${item}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
build_san_line() {
|
||||||
|
local san_entries=()
|
||||||
|
while IFS= read -r dns_item; do
|
||||||
|
san_entries+=("DNS:${dns_item}")
|
||||||
|
done < <(split_csv "${CH_CERT_DNS}")
|
||||||
|
while IFS= read -r ip_item; do
|
||||||
|
san_entries+=("IP:${ip_item}")
|
||||||
|
done < <(split_csv "${CH_CERT_IP}")
|
||||||
|
|
||||||
|
if [[ ${#san_entries[@]} -eq 0 ]]; then
|
||||||
|
san_entries+=("DNS:${CH_CERT_CN}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
local san_line
|
||||||
|
san_line="$(IFS=,; echo "${san_entries[*]}")"
|
||||||
|
echo "${san_line}"
|
||||||
|
}
|
||||||
|
|
||||||
|
generate_self_signed_cert() {
|
||||||
|
echo "[INFO] generating self-signed server certificate (crt+key only) ..."
|
||||||
|
local server_key="${PKI_DIR}/server.key"
|
||||||
|
local server_csr="${PKI_DIR}/server.csr"
|
||||||
|
local server_crt="${PKI_DIR}/server.crt"
|
||||||
|
local ext_file="${PKI_DIR}/server.ext"
|
||||||
|
local san_line
|
||||||
|
san_line="$(build_san_line)"
|
||||||
|
|
||||||
|
openssl genrsa -out "${server_key}" 2048
|
||||||
|
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
|
||||||
|
|
||||||
|
cat >"${ext_file}" <<EOF
|
||||||
|
subjectAltName=${san_line}
|
||||||
|
keyUsage=digitalSignature,keyEncipherment
|
||||||
|
extendedKeyUsage=serverAuth
|
||||||
|
EOF
|
||||||
|
|
||||||
|
openssl x509 -req -in "${server_csr}" -signkey "${server_key}" \
|
||||||
|
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
|
||||||
|
|
||||||
|
cp -f "${server_crt}" "${SERVER_CERT}"
|
||||||
|
cp -f "${server_key}" "${SERVER_KEY}"
|
||||||
|
rm -f "${CA_CERT}"
|
||||||
|
}
|
||||||
|
|
||||||
|
generate_cert_with_ca() {
|
||||||
|
echo "[INFO] generating local CA and server certificate ..."
|
||||||
|
local ca_key="${PKI_DIR}/ca.key"
|
||||||
|
local ca_crt="${PKI_DIR}/ca.crt"
|
||||||
|
local server_key="${PKI_DIR}/server.key"
|
||||||
|
local server_csr="${PKI_DIR}/server.csr"
|
||||||
|
local server_crt="${PKI_DIR}/server.crt"
|
||||||
|
local ext_file="${PKI_DIR}/server.ext"
|
||||||
|
local san_line
|
||||||
|
san_line="$(build_san_line)"
|
||||||
|
|
||||||
|
openssl genrsa -out "${ca_key}" 4096
|
||||||
|
openssl req -x509 -new -nodes -key "${ca_key}" -sha256 -days 3650 \
|
||||||
|
-out "${ca_crt}" -subj "/CN=ClickHouse Local CA"
|
||||||
|
|
||||||
|
openssl genrsa -out "${server_key}" 2048
|
||||||
|
openssl req -new -key "${server_key}" -out "${server_csr}" -subj "/CN=${CH_CERT_CN}"
|
||||||
|
|
||||||
|
cat >"${ext_file}" <<EOF
|
||||||
|
subjectAltName=${san_line}
|
||||||
|
keyUsage=digitalSignature,keyEncipherment
|
||||||
|
extendedKeyUsage=serverAuth
|
||||||
|
EOF
|
||||||
|
|
||||||
|
openssl x509 -req -in "${server_csr}" -CA "${ca_crt}" -CAkey "${ca_key}" -CAcreateserial \
|
||||||
|
-out "${server_crt}" -days "${CH_CERT_DAYS}" -sha256 -extfile "${ext_file}"
|
||||||
|
|
||||||
|
cp -f "${server_crt}" "${SERVER_CERT}"
|
||||||
|
cp -f "${server_key}" "${SERVER_KEY}"
|
||||||
|
cp -f "${ca_crt}" "${CA_CERT}"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ -n "${SRC_CERT}" || -n "${SRC_KEY}" ]]; then
|
||||||
|
if [[ -z "${SRC_CERT}" || -z "${SRC_KEY}" ]]; then
|
||||||
|
echo "[ERROR] SRC_CERT and SRC_KEY must be provided together"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "[INFO] using provided certificate files ..."
|
||||||
|
cp -f "${SRC_CERT}" "${SERVER_CERT}"
|
||||||
|
cp -f "${SRC_KEY}" "${SERVER_KEY}"
|
||||||
|
if [[ -n "${SRC_CA}" ]]; then
|
||||||
|
cp -f "${SRC_CA}" "${CA_CERT}"
|
||||||
|
else
|
||||||
|
rm -f "${CA_CERT}"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
case "$(echo "${CH_GENERATE_CA}" | tr '[:upper:]' '[:lower:]')" in
|
||||||
|
1|true|yes|on)
|
||||||
|
generate_cert_with_ca
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
generate_self_signed_cert
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
chown clickhouse:clickhouse "${SERVER_CERT}" "${SERVER_KEY}" || true
|
||||||
|
chmod 0644 "${SERVER_CERT}"
|
||||||
|
chmod 0640 "${SERVER_KEY}"
|
||||||
|
if [[ -f "${CA_CERT}" ]]; then
|
||||||
|
chown clickhouse:clickhouse "${CA_CERT}" || true
|
||||||
|
chmod 0644 "${CA_CERT}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[INFO] writing ClickHouse HTTPS override config ..."
|
||||||
|
cat >"${OVERRIDE_FILE}" <<EOF
|
||||||
|
<clickhouse>
|
||||||
|
<https_port>${CH_HTTPS_PORT}</https_port>
|
||||||
|
<listen_host>${CH_LISTEN_HOST}</listen_host>
|
||||||
|
<openSSL>
|
||||||
|
<server>
|
||||||
|
<certificateFile>${SERVER_CERT}</certificateFile>
|
||||||
|
<privateKeyFile>${SERVER_KEY}</privateKeyFile>
|
||||||
|
<verificationMode>none</verificationMode>
|
||||||
|
<loadDefaultCAFile>true</loadDefaultCAFile>
|
||||||
|
<cacheSessions>true</cacheSessions>
|
||||||
|
<disableProtocols>sslv2,sslv3</disableProtocols>
|
||||||
|
<preferServerCiphers>true</preferServerCiphers>
|
||||||
|
<invalidCertificateHandler>
|
||||||
|
<name>RejectCertificateHandler</name>
|
||||||
|
</invalidCertificateHandler>
|
||||||
|
</server>
|
||||||
|
</openSSL>
|
||||||
|
</clickhouse>
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "[INFO] restarting clickhouse-server ..."
|
||||||
|
systemctl restart clickhouse-server
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
echo "[INFO] service status ..."
|
||||||
|
systemctl --no-pager -l status clickhouse-server | sed -n '1,15p'
|
||||||
|
|
||||||
|
echo "[INFO] verifying HTTPS endpoint ..."
|
||||||
|
curl -sk "https://127.0.0.1:${CH_HTTPS_PORT}/?query=SELECT%201" || true
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "[OK] ClickHouse HTTPS setup finished"
|
||||||
|
echo " HTTPS port : ${CH_HTTPS_PORT}"
|
||||||
|
echo " cert file : ${SERVER_CERT}"
|
||||||
|
echo " key file : ${SERVER_KEY}"
|
||||||
|
if [[ -f "${CA_CERT}" ]]; then
|
||||||
|
echo " CA file : ${CA_CERT}"
|
||||||
|
echo " import this CA file into API/Fluent Bit hosts if tls.verify=On"
|
||||||
|
fi
|
||||||
38
deploy/clickhouse/init_waf_logs_tables.sh
Normal file
38
deploy/clickhouse/init_waf_logs_tables.sh
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
SQL_FILE="${SCRIPT_DIR}/init_waf_logs_tables.sql"
|
||||||
|
|
||||||
|
if [[ ! -f "${SQL_FILE}" ]]; then
|
||||||
|
echo "[ERROR] SQL file not found: ${SQL_FILE}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! command -v clickhouse-client >/dev/null 2>&1; then
|
||||||
|
echo "[ERROR] clickhouse-client not found. Please install ClickHouse client first."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
CH_HOST="${CH_HOST:-127.0.0.1}"
|
||||||
|
CH_PORT="${CH_PORT:-9000}"
|
||||||
|
CH_USER="${CH_USER:-default}"
|
||||||
|
CH_PASSWORD="${CH_PASSWORD:-}"
|
||||||
|
CH_DATABASE="${CH_DATABASE:-default}"
|
||||||
|
|
||||||
|
args=(--host "${CH_HOST}" --port "${CH_PORT}" --user "${CH_USER}")
|
||||||
|
if [[ -n "${CH_PASSWORD}" ]]; then
|
||||||
|
args+=(--password "${CH_PASSWORD}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[INFO] creating database if not exists: ${CH_DATABASE}"
|
||||||
|
clickhouse-client "${args[@]}" --query "CREATE DATABASE IF NOT EXISTS ${CH_DATABASE}"
|
||||||
|
|
||||||
|
echo "[INFO] initializing tables in database: ${CH_DATABASE}"
|
||||||
|
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" < "${SQL_FILE}"
|
||||||
|
|
||||||
|
echo "[INFO] checking table status ..."
|
||||||
|
clickhouse-client "${args[@]}" --database "${CH_DATABASE}" --query \
|
||||||
|
"SELECT name, engine FROM system.tables WHERE database='${CH_DATABASE}' AND name IN ('logs_ingest','dns_logs_ingest') ORDER BY name"
|
||||||
|
|
||||||
|
echo "[OK] ClickHouse ingest tables are ready in database '${CH_DATABASE}'"
|
||||||
69
deploy/clickhouse/init_waf_logs_tables.sql
Normal file
69
deploy/clickhouse/init_waf_logs_tables.sql
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
-- Initialize HTTP and DNS ingest tables for GoEdge access logs.
|
||||||
|
-- Run with:
|
||||||
|
-- clickhouse-client --database <db_name> < init_waf_logs_tables.sql
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS logs_ingest
|
||||||
|
(
|
||||||
|
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
|
||||||
|
node_id UInt64,
|
||||||
|
cluster_id UInt64,
|
||||||
|
server_id UInt64,
|
||||||
|
host LowCardinality(String),
|
||||||
|
ip String,
|
||||||
|
method LowCardinality(String),
|
||||||
|
path String CODEC(ZSTD(1)),
|
||||||
|
status UInt16,
|
||||||
|
bytes_in UInt64 CODEC(Delta, ZSTD(1)),
|
||||||
|
bytes_out UInt64 CODEC(Delta, ZSTD(1)),
|
||||||
|
cost_ms UInt32 CODEC(Delta, ZSTD(1)),
|
||||||
|
ua String CODEC(ZSTD(1)),
|
||||||
|
referer String CODEC(ZSTD(1)),
|
||||||
|
log_type LowCardinality(String),
|
||||||
|
trace_id String,
|
||||||
|
firewall_policy_id UInt64 DEFAULT 0,
|
||||||
|
firewall_rule_group_id UInt64 DEFAULT 0,
|
||||||
|
firewall_rule_set_id UInt64 DEFAULT 0,
|
||||||
|
firewall_rule_id UInt64 DEFAULT 0,
|
||||||
|
request_headers String CODEC(ZSTD(3)) DEFAULT '',
|
||||||
|
request_body String CODEC(ZSTD(3)) DEFAULT '',
|
||||||
|
response_headers String CODEC(ZSTD(3)) DEFAULT '',
|
||||||
|
response_body String CODEC(ZSTD(3)) DEFAULT '',
|
||||||
|
INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||||
|
INDEX idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||||
|
INDEX idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
|
||||||
|
INDEX idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4,
|
||||||
|
INDEX idx_status status TYPE minmax GRANULARITY 4
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
PARTITION BY toYYYYMMDD(timestamp)
|
||||||
|
ORDER BY (timestamp, node_id, server_id, trace_id)
|
||||||
|
SETTINGS index_granularity = 8192;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS dns_logs_ingest
|
||||||
|
(
|
||||||
|
timestamp DateTime CODEC(DoubleDelta, ZSTD(1)),
|
||||||
|
request_id String,
|
||||||
|
node_id UInt64,
|
||||||
|
cluster_id UInt64,
|
||||||
|
domain_id UInt64,
|
||||||
|
record_id UInt64,
|
||||||
|
remote_addr String,
|
||||||
|
question_name String,
|
||||||
|
question_type LowCardinality(String),
|
||||||
|
record_name String,
|
||||||
|
record_type LowCardinality(String),
|
||||||
|
record_value String,
|
||||||
|
networking LowCardinality(String),
|
||||||
|
is_recursive UInt8,
|
||||||
|
error String CODEC(ZSTD(1)),
|
||||||
|
ns_route_codes Array(String),
|
||||||
|
content_json String CODEC(ZSTD(3)) DEFAULT '',
|
||||||
|
INDEX idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||||
|
INDEX idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4,
|
||||||
|
INDEX idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,
|
||||||
|
INDEX idx_domain_id domain_id TYPE minmax GRANULARITY 4
|
||||||
|
)
|
||||||
|
ENGINE = MergeTree
|
||||||
|
PARTITION BY toYYYYMMDD(timestamp)
|
||||||
|
ORDER BY (timestamp, request_id, node_id)
|
||||||
|
SETTINGS index_granularity = 8192;
|
||||||
95
deploy/clickhouse/install_clickhouse_linux.sh
Normal file
95
deploy/clickhouse/install_clickhouse_linux.sh
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ "${EUID}" -ne 0 ]]; then
|
||||||
|
echo "[ERROR] please run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f /etc/os-release ]]; then
|
||||||
|
echo "[ERROR] /etc/os-release not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source /etc/os-release
|
||||||
|
os_id="$(echo "${ID:-}" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
os_ver="${VERSION_ID:-}"
|
||||||
|
is_ubuntu22=false
|
||||||
|
is_amzn2023=false
|
||||||
|
|
||||||
|
if [[ "${os_id}" == "ubuntu" && "${os_ver}" == 22.04* ]]; then
|
||||||
|
is_ubuntu22=true
|
||||||
|
fi
|
||||||
|
if [[ "${os_id}" == "amzn" && "${os_ver}" == 2023* ]]; then
|
||||||
|
is_amzn2023=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${is_ubuntu22}" != "true" && "${is_amzn2023}" != "true" ]]; then
|
||||||
|
echo "[ERROR] only Ubuntu 22.04 or Amazon Linux 2023 is supported. current: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${is_ubuntu22}" == "true" ]]; then
|
||||||
|
echo "[INFO] detected Ubuntu 22.04"
|
||||||
|
echo "[INFO] installing prerequisites ..."
|
||||||
|
apt-get update -y
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates gnupg apt-transport-https lsb-release
|
||||||
|
|
||||||
|
echo "[INFO] configuring ClickHouse apt repository ..."
|
||||||
|
install -d -m 0755 /etc/apt/keyrings
|
||||||
|
if [[ ! -f /etc/apt/keyrings/clickhouse.gpg ]]; then
|
||||||
|
curl -fsSL https://packages.clickhouse.com/CLICKHOUSE-KEY.GPG | gpg --dearmor -o /etc/apt/keyrings/clickhouse.gpg
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat >/etc/apt/sources.list.d/clickhouse.list <<'EOF'
|
||||||
|
deb [signed-by=/etc/apt/keyrings/clickhouse.gpg arch=amd64,arm64] https://packages.clickhouse.com/deb stable main
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
|
||||||
|
apt-get update -y
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y clickhouse-server clickhouse-client clickhouse-common-static
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${is_amzn2023}" == "true" ]]; then
|
||||||
|
echo "[INFO] detected Amazon Linux 2023"
|
||||||
|
echo "[INFO] installing prerequisites ..."
|
||||||
|
dnf makecache -y
|
||||||
|
dnf install -y curl ca-certificates gnupg2 dnf-plugins-core
|
||||||
|
|
||||||
|
echo "[INFO] configuring ClickHouse yum repository ..."
|
||||||
|
cat >/etc/yum.repos.d/clickhouse.repo <<'EOF'
|
||||||
|
[clickhouse-stable]
|
||||||
|
name=ClickHouse Stable Repository
|
||||||
|
baseurl=https://packages.clickhouse.com/rpm/stable/$basearch
|
||||||
|
enabled=1
|
||||||
|
gpgcheck=1
|
||||||
|
gpgkey=https://packages.clickhouse.com/rpm/stable/repodata/repomd.xml.key
|
||||||
|
https://packages.clickhouse.com/rpm/clickhouse-static.key
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "[INFO] installing clickhouse-server and clickhouse-client ..."
|
||||||
|
dnf clean all
|
||||||
|
dnf makecache -y
|
||||||
|
if ! dnf install -y clickhouse-server clickhouse-client clickhouse-common-static; then
|
||||||
|
dnf install -y clickhouse-server clickhouse-client
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[INFO] enabling clickhouse-server ..."
|
||||||
|
systemctl enable clickhouse-server >/dev/null 2>&1 || true
|
||||||
|
systemctl restart clickhouse-server
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
if [[ -n "${CLICKHOUSE_DEFAULT_PASSWORD:-}" ]]; then
|
||||||
|
echo "[INFO] setting default user password ..."
|
||||||
|
if [[ "${CLICKHOUSE_DEFAULT_PASSWORD}" == *"'"* ]]; then
|
||||||
|
echo "[ERROR] CLICKHOUSE_DEFAULT_PASSWORD contains single quote, please set password manually with clickhouse-client"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
clickhouse-client --query "ALTER USER default IDENTIFIED WITH plaintext_password BY '${CLICKHOUSE_DEFAULT_PASSWORD}'"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "[INFO] health check ..."
|
||||||
|
clickhouse-client --query "SELECT version()"
|
||||||
|
echo "[OK] ClickHouse install completed: ID=${ID:-unknown}, VERSION_ID=${VERSION_ID:-unknown}"
|
||||||
123
deploy/clickhouse/optimize_schema.sql
Normal file
123
deploy/clickhouse/optimize_schema.sql
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
-- =============================================================================
|
||||||
|
-- ClickHouse logs_ingest 表优化脚本
|
||||||
|
--
|
||||||
|
-- 说明:
|
||||||
|
-- - 所有 ALTER 操作均为在线操作,无需停服
|
||||||
|
-- - 建议按阶段顺序执行,每阶段执行后观察 system.parts 确认生效
|
||||||
|
-- - 压缩编解码器变更仅影响新写入的 part,存量数据需等 merge 或手动 OPTIMIZE
|
||||||
|
--
|
||||||
|
-- 执行方式:
|
||||||
|
-- clickhouse-client --host 127.0.0.1 --port 9000 --user default --password 'xxx' < optimize_schema.sql
|
||||||
|
-- =============================================================================
|
||||||
|
|
||||||
|
-- =============================================
|
||||||
|
-- 阶段 1:大字段压缩优化(效果最显著)
|
||||||
|
-- =============================================
|
||||||
|
|
||||||
|
-- 大文本字段改用 ZSTD(3),对 JSON / HTTP 文本压缩率远优于默认 LZ4
|
||||||
|
-- 预期效果:磁盘占用减少 40%-60%
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN request_headers String CODEC(ZSTD(3));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN request_body String CODEC(ZSTD(3));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN response_headers String CODEC(ZSTD(3));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN response_body String CODEC(ZSTD(3));
|
||||||
|
|
||||||
|
-- 中等长度文本字段用 ZSTD(1),平衡压缩率与 CPU 开销
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN ua String CODEC(ZSTD(1));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN path String CODEC(ZSTD(1));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN referer String CODEC(ZSTD(1));
|
||||||
|
|
||||||
|
-- 低基数字段改用 LowCardinality(内存+磁盘双降)
|
||||||
|
-- method 的基数极低(GET/POST/PUT/DELETE 等),host 基数取决于站点数量
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN method LowCardinality(String);
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN log_type LowCardinality(String);
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN host LowCardinality(String);
|
||||||
|
|
||||||
|
-- 数值字段使用 Delta + ZSTD 编码(利用相邻行的时间/大小相关性)
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN bytes_in UInt64 CODEC(Delta, ZSTD(1));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN bytes_out UInt64 CODEC(Delta, ZSTD(1));
|
||||||
|
ALTER TABLE logs_ingest MODIFY COLUMN cost_ms UInt32 CODEC(Delta, ZSTD(1));
|
||||||
|
|
||||||
|
-- =============================================
|
||||||
|
-- 阶段 2:添加 Skipping Index(加速高频过滤查询)
|
||||||
|
-- =============================================
|
||||||
|
|
||||||
|
-- trace_id 精确查找(查看日志详情 FindByTraceId)
|
||||||
|
-- bloom_filter(0.01) = 1% 误判率,GRANULARITY 4 = 每 4 个 granule 一个 bloom block
|
||||||
|
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- IP 精确查找
|
||||||
|
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- host 模糊查询支持(tokenbf_v1 对 LIKE '%xxx%' 有效)
|
||||||
|
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- firewall_policy_id 过滤(HasFirewallPolicy: > 0)
|
||||||
|
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4;
|
||||||
|
|
||||||
|
-- status 范围过滤(HasError: status >= 400)
|
||||||
|
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_status status TYPE minmax GRANULARITY 4;
|
||||||
|
|
||||||
|
-- =============================================
|
||||||
|
-- 阶段 3:物化索引到现有数据(对存量数据生效)
|
||||||
|
-- =============================================
|
||||||
|
-- 注意:MATERIALIZE INDEX 会触发后台 mutation,大表可能需要一定时间
|
||||||
|
-- 可通过 SELECT * FROM system.mutations WHERE is_done = 0 监控进度
|
||||||
|
|
||||||
|
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_trace_id;
|
||||||
|
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_ip;
|
||||||
|
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_host;
|
||||||
|
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_fw_policy;
|
||||||
|
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_status;
|
||||||
|
|
||||||
|
|
||||||
|
-- =============================================================================
|
||||||
|
-- dns_logs_ingest 表优化(DNS 日志表)
|
||||||
|
-- =============================================================================
|
||||||
|
|
||||||
|
-- 大文本字段压缩
|
||||||
|
ALTER TABLE dns_logs_ingest MODIFY COLUMN content_json String CODEC(ZSTD(3));
|
||||||
|
ALTER TABLE dns_logs_ingest MODIFY COLUMN error String CODEC(ZSTD(1));
|
||||||
|
|
||||||
|
-- 低基数字段
|
||||||
|
ALTER TABLE dns_logs_ingest MODIFY COLUMN question_type LowCardinality(String);
|
||||||
|
ALTER TABLE dns_logs_ingest MODIFY COLUMN record_type LowCardinality(String);
|
||||||
|
ALTER TABLE dns_logs_ingest MODIFY COLUMN networking LowCardinality(String);
|
||||||
|
|
||||||
|
-- request_id 精确查找
|
||||||
|
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- remote_addr 精确查找
|
||||||
|
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- question_name 模糊查询
|
||||||
|
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
|
||||||
|
|
||||||
|
-- domain_id 过滤
|
||||||
|
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_domain_id domain_id TYPE minmax GRANULARITY 4;
|
||||||
|
|
||||||
|
-- 物化索引到现有数据
|
||||||
|
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_request_id;
|
||||||
|
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_remote_addr;
|
||||||
|
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_question_name;
|
||||||
|
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_domain_id;
|
||||||
|
|
||||||
|
|
||||||
|
-- =============================================================================
|
||||||
|
-- 验证命令(执行完上述 ALTER 后运行)
|
||||||
|
-- =============================================================================
|
||||||
|
|
||||||
|
-- 查看列的压缩编解码器
|
||||||
|
-- SELECT name, type, compression_codec FROM system.columns WHERE table = 'logs_ingest' AND database = currentDatabase();
|
||||||
|
|
||||||
|
-- 查看表的压缩率
|
||||||
|
-- SELECT table, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed, round(sum(data_uncompressed_bytes) / sum(data_compressed_bytes), 2) AS ratio FROM system.columns WHERE table IN ('logs_ingest', 'dns_logs_ingest') GROUP BY table;
|
||||||
|
|
||||||
|
-- 查看各列占用的磁盘空间(找出最大的列)
|
||||||
|
-- SELECT name, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed FROM system.columns WHERE table = 'logs_ingest' GROUP BY name ORDER BY sum(data_compressed_bytes) DESC;
|
||||||
|
|
||||||
|
-- 查看 mutation 进度
|
||||||
|
-- SELECT database, table, mutation_id, command, is_done, parts_to_do FROM system.mutations WHERE is_done = 0;
|
||||||
|
|
||||||
|
-- 强制触发 merge(可选,让压缩编解码器变更对存量数据生效)
|
||||||
|
-- OPTIMIZE TABLE logs_ingest FINAL;
|
||||||
|
-- OPTIMIZE TABLE dns_logs_ingest FINAL;
|
||||||
@@ -21,7 +21,14 @@
|
|||||||
- **边缘节点(EdgeNode)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`(JSON Lines)。
|
- **边缘节点(EdgeNode)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`(JSON Lines)。
|
||||||
- **DNS 节点(EdgeDNS)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines)。
|
- **DNS 节点(EdgeDNS)** 已开启本地日志落盘,目录优先取“公用访问日志策略”的文件 `path`(取目录),为空时回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines)。
|
||||||
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest` 表(见下文「五、ClickHouse 建表」)。
|
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest` 表(见下文「五、ClickHouse 建表」)。
|
||||||
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123)。
|
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTPS 端口 8443)。
|
||||||
|
- 日志轮转默认由 Node/DNS 内建 `lumberjack` 执行:
|
||||||
|
- `maxSizeMB=256`
|
||||||
|
- `maxBackups=14`
|
||||||
|
- `maxAgeDays=7`
|
||||||
|
- `compress=false`
|
||||||
|
- `localTime=true`
|
||||||
|
可通过公用日志策略 `file.rotate` 调整。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -71,7 +78,7 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
|
|||||||
|
|
||||||
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port:
|
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port:
|
||||||
|
|
||||||
- **单机**:保留一个 `[NODE]`,改 `Host`、`Port`(默认 8123)。
|
- **单机**:保留一个 `[NODE]`,改 `Host`、`Port`(默认 8443)。
|
||||||
- **集群**:复制多段 `[NODE]`,每段一个节点,例如:
|
- **集群**:复制多段 `[NODE]`,每段一个节点,例如:
|
||||||
|
|
||||||
```ini
|
```ini
|
||||||
@@ -81,12 +88,12 @@ sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
|
|||||||
[NODE]
|
[NODE]
|
||||||
Name node-01
|
Name node-01
|
||||||
Host 192.168.1.10
|
Host 192.168.1.10
|
||||||
Port 8123
|
Port 8443
|
||||||
|
|
||||||
[NODE]
|
[NODE]
|
||||||
Name node-02
|
Name node-02
|
||||||
Host 192.168.1.11
|
Host 192.168.1.11
|
||||||
Port 8123
|
Port 8443
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3.3 ClickHouse 账号密码(有密码时必做)
|
### 3.3 ClickHouse 账号密码(有密码时必做)
|
||||||
@@ -319,9 +326,9 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
|
|||||||
|
|
||||||
| 组件 | 说明 |
|
| 组件 | 说明 |
|
||||||
|------|------|
|
|------|------|
|
||||||
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
|
| **EdgeNode** | 日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_LOG_DIR`,再回退默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;内建 lumberjack 轮转(默认 256MB/14份/7天,可按策略调整),仍支持 SIGHUP 重建 writer。 |
|
||||||
| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines),由 Fluent Bit 采集写入 `dns_logs_ingest`。 |
|
| **EdgeDNS** | DNS 访问日志落盘路径优先复用公用访问日志策略文件 `path`(取目录);若为空回退 `EDGE_DNS_LOG_DIR`,再回退默认 `/var/log/edge/edge-dns`;生成 `access.log`(JSON Lines),由 Fluent Bit 采集写入 `dns_logs_ingest`。 |
|
||||||
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 |
|
| **logrotate** | 可选的历史兼容方案(已非必需);默认建议使用节点内建 lumberjack 轮转。 |
|
||||||
| **平台(EdgeAPI)** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
|
| **平台(EdgeAPI)** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -411,3 +418,54 @@ sudo systemctl restart fluent-bit
|
|||||||
```
|
```
|
||||||
|
|
||||||
回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。
|
回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 十、平台托管模式(推荐)
|
||||||
|
|
||||||
|
从 `v1.4.7` 开始,Node/DNS 在线安装流程会由平台托管 Fluent Bit,默认不再要求逐台手改 `/etc/fluent-bit/fluent-bit.conf`。
|
||||||
|
|
||||||
|
### 10.1 托管行为
|
||||||
|
|
||||||
|
- 安装器优先使用发布包内置离线包(不走 `curl | sh`)。
|
||||||
|
- 首次安装后写入:
|
||||||
|
- `/etc/fluent-bit/fluent-bit.conf`
|
||||||
|
- `/etc/fluent-bit/parsers.conf`
|
||||||
|
- `/etc/fluent-bit/.edge-managed.env`
|
||||||
|
- `/etc/fluent-bit/.edge-managed.json`
|
||||||
|
- 配置发生变化时按 `hash` 幂等更新,仅在内容变化时重启服务。
|
||||||
|
- Node 与 DNS 同机安装时会自动合并角色,输出单份配置。
|
||||||
|
|
||||||
|
### 10.2 托管元数据
|
||||||
|
|
||||||
|
平台会维护 `/etc/fluent-bit/.edge-managed.json`,核心字段:
|
||||||
|
|
||||||
|
- `roles`: 当前机器启用角色(`node`/`dns`)
|
||||||
|
- `hash`: 当前托管配置摘要
|
||||||
|
- `sourceVersion`: 平台版本号
|
||||||
|
- `updatedAt`: 最近更新时间戳
|
||||||
|
|
||||||
|
### 10.3 支持矩阵(离线包)
|
||||||
|
|
||||||
|
当前固定支持以下平台键:
|
||||||
|
|
||||||
|
- `ubuntu22.04-amd64`
|
||||||
|
- `ubuntu22.04-arm64`
|
||||||
|
- `amzn2023-amd64`
|
||||||
|
- `amzn2023-arm64`
|
||||||
|
|
||||||
|
构建阶段会校验矩阵包是否齐全,缺失会直接失败并打印期望文件路径。
|
||||||
|
|
||||||
|
### 10.4 手工配置兼容
|
||||||
|
|
||||||
|
- 若现有 `fluent-bit.conf` 不是平台托管文件(不含 `managed-by-edgeapi` 标记),安装器不会强制覆盖,会返回明确错误提示。
|
||||||
|
- 需要切到托管模式时,先备份旧配置,再由平台触发一次安装/更新任务。
|
||||||
|
|
||||||
|
### 10.5 Resource Profile Notes (New)
|
||||||
|
|
||||||
|
- Managed default is now tuned for `2C4G` nodes (conservative and stable).
|
||||||
|
- Additional sample profiles are provided for larger nodes:
|
||||||
|
- `deploy/fluent-bit/fluent-bit-sample-4c8g.conf`
|
||||||
|
- `deploy/fluent-bit/fluent-bit-sample-8c16g.conf`
|
||||||
|
- These sample files are for benchmark/reference only and are not auto-applied by installer.
|
||||||
|
- To use higher profiles in managed mode, sync those parameters into `EdgeAPI/internal/installers/fluent_bit.go` and then trigger node reinstall/upgrade.
|
||||||
|
|||||||
@@ -8,4 +8,4 @@
|
|||||||
[NODE]
|
[NODE]
|
||||||
Name node-01
|
Name node-01
|
||||||
Host 127.0.0.1
|
Host 127.0.0.1
|
||||||
Port 8123
|
Port 8443
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# DNS 节点专用:使用 HTTP 输出写入 ClickHouse(无需 out_clickhouse 插件)
|
# DNS 节点专用:使用 HTTPS 输出写入 ClickHouse(无需 out_clickhouse 插件)
|
||||||
# 启动前设置:CH_USER、CH_PASSWORD;若 ClickHouse 不在本机,请修改 Host、Port
|
# 启动前设置:CH_USER、CH_PASSWORD;若 ClickHouse 不在本机,请修改 Host、Port
|
||||||
# Read_from_Head=true:首次启动会发送已有日志;若只采新日志建议改为 false
|
# Read_from_Head=true:首次启动会发送已有日志;若只采新日志建议改为 false
|
||||||
|
|
||||||
@@ -26,11 +26,15 @@
|
|||||||
Name http
|
Name http
|
||||||
Match app.dns.logs
|
Match app.dns.logs
|
||||||
Host 127.0.0.1
|
Host 127.0.0.1
|
||||||
Port 8123
|
Port 8443
|
||||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||||
Format json_lines
|
Format json_lines
|
||||||
http_user ${CH_USER}
|
http_user ${CH_USER}
|
||||||
http_passwd ${CH_PASSWORD}
|
http_passwd ${CH_PASSWORD}
|
||||||
|
tls On
|
||||||
|
tls.verify On
|
||||||
|
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||||
|
# tls.vhost clickhouse.example.com
|
||||||
json_date_key timestamp
|
json_date_key timestamp
|
||||||
json_date_format epoch
|
json_date_format epoch
|
||||||
Retry_Limit 10
|
Retry_Limit 10
|
||||||
|
|||||||
69
deploy/fluent-bit/fluent-bit-sample-4c8g.conf
Normal file
69
deploy/fluent-bit/fluent-bit-sample-4c8g.conf
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# Sample profile for 4C8G nodes (Node + DNS on same host).
|
||||||
|
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
|
||||||
|
|
||||||
|
[SERVICE]
|
||||||
|
Flush 1
|
||||||
|
Log_Level info
|
||||||
|
Parsers_File parsers.conf
|
||||||
|
storage.path /var/lib/fluent-bit/storage
|
||||||
|
storage.sync normal
|
||||||
|
storage.checksum off
|
||||||
|
storage.backlog.mem_limit 512MB
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/edge/edge-node/*.log
|
||||||
|
Tag app.http.logs
|
||||||
|
Parser json
|
||||||
|
Refresh_Interval 2
|
||||||
|
Read_from_Head false
|
||||||
|
DB /var/lib/fluent-bit/http-logs.db
|
||||||
|
storage.type filesystem
|
||||||
|
Mem_Buf_Limit 256MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/edge/edge-dns/*.log
|
||||||
|
Tag app.dns.logs
|
||||||
|
Parser json
|
||||||
|
Refresh_Interval 2
|
||||||
|
Read_from_Head false
|
||||||
|
DB /var/lib/fluent-bit/dns-logs.db
|
||||||
|
storage.type filesystem
|
||||||
|
Mem_Buf_Limit 256MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name http
|
||||||
|
Match app.http.logs
|
||||||
|
Host 127.0.0.1
|
||||||
|
Port 8443
|
||||||
|
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
|
||||||
|
Format json_lines
|
||||||
|
http_user ${CH_USER}
|
||||||
|
http_passwd ${CH_PASSWORD}
|
||||||
|
json_date_key timestamp
|
||||||
|
json_date_format epoch
|
||||||
|
workers 2
|
||||||
|
net.keepalive On
|
||||||
|
Retry_Limit False
|
||||||
|
tls On
|
||||||
|
tls.verify On
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name http
|
||||||
|
Match app.dns.logs
|
||||||
|
Host 127.0.0.1
|
||||||
|
Port 8443
|
||||||
|
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||||
|
Format json_lines
|
||||||
|
http_user ${CH_USER}
|
||||||
|
http_passwd ${CH_PASSWORD}
|
||||||
|
json_date_key timestamp
|
||||||
|
json_date_format epoch
|
||||||
|
workers 2
|
||||||
|
net.keepalive On
|
||||||
|
Retry_Limit False
|
||||||
|
tls On
|
||||||
|
tls.verify On
|
||||||
69
deploy/fluent-bit/fluent-bit-sample-8c16g.conf
Normal file
69
deploy/fluent-bit/fluent-bit-sample-8c16g.conf
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# Sample profile for 8C16G nodes (Node + DNS on same host).
|
||||||
|
# Replace Host/Port/URI and credentials according to your ClickHouse deployment.
|
||||||
|
|
||||||
|
[SERVICE]
|
||||||
|
Flush 1
|
||||||
|
Log_Level info
|
||||||
|
Parsers_File parsers.conf
|
||||||
|
storage.path /var/lib/fluent-bit/storage
|
||||||
|
storage.sync normal
|
||||||
|
storage.checksum off
|
||||||
|
storage.backlog.mem_limit 1024MB
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/edge/edge-node/*.log
|
||||||
|
Tag app.http.logs
|
||||||
|
Parser json
|
||||||
|
Refresh_Interval 1
|
||||||
|
Read_from_Head false
|
||||||
|
DB /var/lib/fluent-bit/http-logs.db
|
||||||
|
storage.type filesystem
|
||||||
|
Mem_Buf_Limit 512MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/edge/edge-dns/*.log
|
||||||
|
Tag app.dns.logs
|
||||||
|
Parser json
|
||||||
|
Refresh_Interval 1
|
||||||
|
Read_from_Head false
|
||||||
|
DB /var/lib/fluent-bit/dns-logs.db
|
||||||
|
storage.type filesystem
|
||||||
|
Mem_Buf_Limit 512MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name http
|
||||||
|
Match app.http.logs
|
||||||
|
Host 127.0.0.1
|
||||||
|
Port 8443
|
||||||
|
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
|
||||||
|
Format json_lines
|
||||||
|
http_user ${CH_USER}
|
||||||
|
http_passwd ${CH_PASSWORD}
|
||||||
|
json_date_key timestamp
|
||||||
|
json_date_format epoch
|
||||||
|
workers 4
|
||||||
|
net.keepalive On
|
||||||
|
Retry_Limit False
|
||||||
|
tls On
|
||||||
|
tls.verify On
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name http
|
||||||
|
Match app.dns.logs
|
||||||
|
Host 127.0.0.1
|
||||||
|
Port 8443
|
||||||
|
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||||
|
Format json_lines
|
||||||
|
http_user ${CH_USER}
|
||||||
|
http_passwd ${CH_PASSWORD}
|
||||||
|
json_date_key timestamp
|
||||||
|
json_date_format epoch
|
||||||
|
workers 4
|
||||||
|
net.keepalive On
|
||||||
|
Retry_Limit False
|
||||||
|
tls On
|
||||||
|
tls.verify On
|
||||||
Binary file not shown.
Binary file not shown.
@@ -47,6 +47,32 @@ flowchart TD
|
|||||||
- `/var/log/edge/edge-node/*.log`
|
- `/var/log/edge/edge-node/*.log`
|
||||||
3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。
|
3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。
|
||||||
|
|
||||||
|
### 3.3 本地日志轮转(默认开启)
|
||||||
|
从当前版本开始,EdgeNode / EdgeDNS 使用内建 `lumberjack` 轮转,不再依赖系统 `logrotate`。
|
||||||
|
|
||||||
|
默认值:
|
||||||
|
- `maxSizeMB=256`
|
||||||
|
- `maxBackups=14`
|
||||||
|
- `maxAgeDays=7`
|
||||||
|
- `compress=false`
|
||||||
|
- `localTime=true`
|
||||||
|
|
||||||
|
可在策略 `file.rotate` 中配置,例如:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/var/log/web-access-${date}.log",
|
||||||
|
"autoCreate": true,
|
||||||
|
"rotate": {
|
||||||
|
"maxSizeMB": 256,
|
||||||
|
"maxBackups": 14,
|
||||||
|
"maxAgeDays": 7,
|
||||||
|
"compress": false,
|
||||||
|
"localTime": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. 三种目标模式怎么配
|
## 4. 三种目标模式怎么配
|
||||||
|
|||||||
Reference in New Issue
Block a user