lumberjack改造前

This commit is contained in:
robin
2026-02-12 21:37:55 +08:00
parent c28317ee07
commit c6da67db79
24 changed files with 836 additions and 68 deletions

View File

@@ -2,6 +2,7 @@ package clickhouse
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
@@ -20,10 +21,18 @@ type Client struct {
// NewClient 使用共享配置创建客户端
func NewClient() *Client {
cfg := SharedConfig()
transport := &http.Transport{}
if cfg != nil && strings.EqualFold(cfg.Scheme, "https") {
transport.TLSClientConfig = &tls.Config{
InsecureSkipVerify: cfg.TLSSkipVerify,
ServerName: cfg.TLSServerName,
}
}
return &Client{
cfg: cfg,
httpCli: &http.Client{
Timeout: 30 * time.Second,
Timeout: 30 * time.Second,
Transport: transport,
},
}
}
@@ -95,12 +104,16 @@ func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) e
}
func (c *Client) buildURL(query string) string {
rawURL := fmt.Sprintf("http://%s:%d/?query=%s&database=%s",
c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
scheme := "http"
if c.cfg != nil && strings.EqualFold(c.cfg.Scheme, "https") {
scheme = "https"
}
rawURL := fmt.Sprintf("%s://%s:%d/?query=%s&database=%s",
scheme, c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
return rawURL
}
// decodeRows 将 JSONEachRow 流解析到 slice元素类型须为 *struct 或 *map[string]interface{}
// decodeRows 将 JSONEachRow 流解析到 slice元素类型须为 *struct 或 *[]map[string]interface{}
func decodeRows(dec *json.Decoder, dest interface{}) error {
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
switch d := dest.(type) {

View File

@@ -7,17 +7,22 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"os"
"strconv"
"strings"
"sync"
)
const (
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
defaultPort = 8123
defaultDB = "default"
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
envScheme = "CLICKHOUSE_SCHEME"
envTLSSkipVerify = "CLICKHOUSE_TLS_SKIP_VERIFY"
envTLSServerName = "CLICKHOUSE_TLS_SERVER_NAME"
defaultPort = 8123
defaultDB = "default"
defaultScheme = "http"
)
var (
@@ -28,11 +33,14 @@ var (
// Config ClickHouse 连接配置(仅查询,不从代码写库)
type Config struct {
Host string
Port int
User string
Password string
Database string
Host string
Port int
User string
Password string
Database string
Scheme string
TLSSkipVerify bool
TLSServerName string
}
// SharedConfig 返回全局配置(优先从后台 DB 读取,其次 api.yaml最后环境变量
@@ -54,7 +62,7 @@ func ResetSharedConfig() {
}
func loadConfig() *Config {
cfg := &Config{Port: defaultPort, Database: defaultDB}
cfg := &Config{Port: defaultPort, Database: defaultDB, Scheme: defaultScheme}
// 1) 优先从后台页面配置DB读取
if models.SharedSysSettingDAO != nil {
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
@@ -63,6 +71,9 @@ func loadConfig() *Config {
cfg.User = dbCfg.User
cfg.Password = dbCfg.Password
cfg.Database = dbCfg.Database
cfg.Scheme = normalizeScheme(dbCfg.Scheme)
cfg.TLSSkipVerify = dbCfg.TLSSkipVerify
cfg.TLSServerName = dbCfg.TLSServerName
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
@@ -81,6 +92,9 @@ func loadConfig() *Config {
cfg.User = ch.User
cfg.Password = ch.Password
cfg.Database = ch.Database
cfg.Scheme = normalizeScheme(ch.Scheme)
cfg.TLSSkipVerify = ch.TLSSkipVerify
cfg.TLSServerName = ch.TLSServerName
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
@@ -97,14 +111,29 @@ func loadConfig() *Config {
if cfg.Database == "" {
cfg.Database = defaultDB
}
cfg.Scheme = normalizeScheme(os.Getenv(envScheme))
cfg.TLSServerName = os.Getenv(envTLSServerName)
if p := os.Getenv(envPort); p != "" {
if v, err := strconv.Atoi(p); err == nil {
cfg.Port = v
}
}
if v := os.Getenv(envTLSSkipVerify); v != "" {
if b, err := strconv.ParseBool(v); err == nil {
cfg.TLSSkipVerify = b
}
}
return cfg
}
func normalizeScheme(scheme string) string {
s := strings.ToLower(strings.TrimSpace(scheme))
if s == "https" {
return "https"
}
return defaultScheme
}
// IsConfigured 是否已配置Host 非空即视为启用 ClickHouse 查询)
func (c *Config) IsConfigured() bool {
return c != nil && c.Host != ""

View File

@@ -12,11 +12,14 @@ var sharedAPIConfig *APIConfig = nil
// ClickHouseConfig 仅用于访问日志列表只读查询logs_ingest
type ClickHouseConfig struct {
Host string `yaml:"host" json:"host"`
Port int `yaml:"port" json:"port"`
User string `yaml:"user" json:"user"`
Password string `yaml:"password" json:"password"`
Database string `yaml:"database" json:"database"`
Host string `yaml:"host" json:"host"`
Port int `yaml:"port" json:"port"`
User string `yaml:"user" json:"user"`
Password string `yaml:"password" json:"password"`
Database string `yaml:"database" json:"database"`
Scheme string `yaml:"scheme" json:"scheme"`
TLSSkipVerify bool `yaml:"tlsSkipVerify" json:"tlsSkipVerify"`
TLSServerName string `yaml:"tlsServerName" json:"tlsServerName"`
}
// APIConfig API节点配置

View File

@@ -1,7 +1,7 @@
package teaconst
package teaconst
const (
Version = "1.4.6" //1.3.9
Version = "1.4.7" //1.3.9
ProductName = "Edge API"
ProcessName = "edge-api"
@@ -17,5 +17,6 @@ const (
// 其他节点版本号,用来检测是否有需要升级的节点
NodeVersion = "1.4.6" //1.3.8.2
NodeVersion = "1.4.7" //1.3.8.2
)

View File

@@ -1,12 +1,13 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build plus
package teaconst
const (
DNSNodeVersion = "1.4.6" //1.3.8.2
UserNodeVersion = "1.4.6" //1.3.8.2
DNSNodeVersion = "1.4.7" //1.3.8.2
UserNodeVersion = "1.4.7" //1.3.8.2
ReportNodeVersion = "0.1.5"
DefaultMaxNodes int32 = 50
)

View File

@@ -0,0 +1,347 @@
package installers
import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/iwind/TeaGo/Tea"
)
const (
fluentBitConfigDir = "/etc/fluent-bit"
fluentBitStorageDir = "/var/lib/fluent-bit/storage"
fluentBitMainConfigFile = "/etc/fluent-bit/fluent-bit.conf"
fluentBitParsersFile = "/etc/fluent-bit/parsers.conf"
fluentBitUpstreamFile = "/etc/fluent-bit/clickhouse-upstream.conf"
fluentBitLogrotateFile = "/etc/logrotate.d/edge-goedge"
fluentBitServiceName = "fluent-bit"
fluentBitDefaultBinPath = "/opt/fluent-bit/bin/fluent-bit"
fluentBitLocalPackagesRoot = "packages"
fluentBitHTTPPathPattern = "/var/log/edge/edge-node/*.log"
fluentBitDNSPathPattern = "/var/log/edge/edge-dns/*.log"
)
var errFluentBitLocalPackageNotFound = errors.New("fluent-bit local package not found")
// SetupFluentBit 安装 Fluent Bit仅离线包并同步配置文件。
// 升级场景下不覆盖已有配置;若已有配置与节点角色不兼容,直接报错终止安装。
func (this *BaseInstaller) SetupFluentBit(role nodeconfigs.NodeRole) error {
if this.client == nil {
return errors.New("ssh client is nil")
}
uname := this.uname()
if !strings.Contains(uname, "Linux") {
return nil
}
tempDir := strings.TrimRight(this.client.UserHome(), "/") + "/.edge-fluent-bit"
_, _, _ = this.client.Exec("mkdir -p " + tempDir)
defer func() {
_, _, _ = this.client.Exec("rm -rf " + tempDir)
}()
// 统一使用 fluent-bit.conf已含 HTTP + DNS 两类 input避免同机 Node/DNS 冲突。
files := []struct {
Local string
Remote string
}{
{Local: "fluent-bit.conf", Remote: "fluent-bit.conf"},
{Local: "parsers.conf", Remote: "parsers.conf"},
{Local: "clickhouse-upstream.conf", Remote: "clickhouse-upstream.conf"},
{Local: "logrotate.conf", Remote: "logrotate.conf"},
}
for _, file := range files {
localPath := filepath.Join(Tea.Root, "deploy", "fluent-bit", file.Local)
if _, err := os.Stat(localPath); err != nil {
return fmt.Errorf("fluent-bit file '%s' not found: %w", localPath, err)
}
remotePath := tempDir + "/" + file.Remote
if err := this.client.Copy(localPath, remotePath, 0644); err != nil {
return fmt.Errorf("upload fluent-bit file '%s' failed: %w", file.Local, err)
}
}
if err := this.ensureFluentBitInstalled(tempDir); err != nil {
return err
}
_, stderr, err := this.client.Exec("mkdir -p " + fluentBitConfigDir + " " + fluentBitStorageDir + " /etc/logrotate.d")
if err != nil {
return fmt.Errorf("prepare fluent-bit directories failed: %w, stderr: %s", err, stderr)
}
exists, err := this.remoteFileExists(fluentBitMainConfigFile)
if err != nil {
return err
}
// 若已存在配置,先做角色兼容校验,不允许覆盖。
if exists {
if err := this.validateExistingConfigForRole(role); err != nil {
return err
}
}
configCopied, err := this.copyFluentBitConfigIfMissing(tempDir)
if err != nil {
return err
}
binPath, err := this.lookupFluentBitBinPath()
if err != nil {
return err
}
if err := this.ensureFluentBitService(binPath, configCopied); err != nil {
return err
}
return nil
}
func (this *BaseInstaller) ensureFluentBitInstalled(tempDir string) error {
binPath, _ := this.lookupFluentBitBinPath()
if binPath != "" {
return nil
}
if err := this.installFluentBitFromLocalPackage(tempDir); err != nil {
if errors.Is(err, errFluentBitLocalPackageNotFound) {
return fmt.Errorf("install fluent-bit failed: local package not found, expected in deploy/fluent-bit/%s/linux-<arch>", fluentBitLocalPackagesRoot)
}
return fmt.Errorf("install fluent-bit from local package failed: %w", err)
}
binPath, err := this.lookupFluentBitBinPath()
if err != nil {
return err
}
if binPath == "" {
return errors.New("fluent-bit binary not found after local package install")
}
return nil
}
func (this *BaseInstaller) installFluentBitFromLocalPackage(tempDir string) error {
arch, err := this.detectRemoteLinuxArch()
if err != nil {
return err
}
packageDir := filepath.Join(Tea.Root, "deploy", "fluent-bit", fluentBitLocalPackagesRoot, "linux-"+arch)
entries, err := os.ReadDir(packageDir)
if err != nil {
if os.IsNotExist(err) {
return errFluentBitLocalPackageNotFound
}
return fmt.Errorf("read fluent-bit local package dir failed: %w", err)
}
packageFiles := make([]string, 0)
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := strings.ToLower(entry.Name())
if strings.HasSuffix(name, ".deb") || strings.HasSuffix(name, ".rpm") || strings.HasSuffix(name, ".tar.gz") || strings.HasSuffix(name, ".tgz") {
packageFiles = append(packageFiles, filepath.Join(packageDir, entry.Name()))
}
}
if len(packageFiles) == 0 {
return errFluentBitLocalPackageNotFound
}
sort.Strings(packageFiles)
var lastErr error
for _, localPackagePath := range packageFiles {
remotePackagePath := tempDir + "/" + filepath.Base(localPackagePath)
if err := this.client.Copy(localPackagePath, remotePackagePath, 0644); err != nil {
lastErr = fmt.Errorf("upload local package failed: %w", err)
continue
}
var installCmd string
lowerName := strings.ToLower(localPackagePath)
switch {
case strings.HasSuffix(lowerName, ".deb"):
installCmd = "dpkg -i " + remotePackagePath
case strings.HasSuffix(lowerName, ".rpm"):
installCmd = "rpm -Uvh --force " + remotePackagePath + " || rpm -ivh --force " + remotePackagePath
case strings.HasSuffix(lowerName, ".tar.gz") || strings.HasSuffix(lowerName, ".tgz"):
extractDir := tempDir + "/extract"
installCmd = "rm -rf " + extractDir + "; mkdir -p " + extractDir + "; tar -xzf " + remotePackagePath + " -C " + extractDir + "; " +
"bin=$(find " + extractDir + " -type f -name fluent-bit | head -n 1); " +
"if [ -z \"$bin\" ]; then exit 3; fi; " +
"mkdir -p /opt/fluent-bit/bin /usr/local/bin; " +
"install -m 0755 \"$bin\" /opt/fluent-bit/bin/fluent-bit; " +
"ln -sf /opt/fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit"
default:
continue
}
_, stderr, err := this.client.Exec(installCmd)
if err != nil {
lastErr = fmt.Errorf("install fluent-bit local package '%s' failed: %w, stderr: %s", filepath.Base(localPackagePath), err, stderr)
continue
}
binPath, err := this.lookupFluentBitBinPath()
if err == nil && binPath != "" {
return nil
}
if err != nil {
lastErr = err
} else {
lastErr = errors.New("fluent-bit binary not found after local package install")
}
}
if lastErr != nil {
return lastErr
}
return errFluentBitLocalPackageNotFound
}
func (this *BaseInstaller) detectRemoteLinuxArch() (string, error) {
stdout, stderr, err := this.client.Exec("uname -m")
if err != nil {
return "", fmt.Errorf("detect remote arch failed: %w, stderr: %s", err, stderr)
}
arch := strings.ToLower(strings.TrimSpace(stdout))
switch arch {
case "x86_64", "amd64":
return "amd64", nil
case "aarch64", "arm64":
return "arm64", nil
default:
return arch, nil
}
}
func (this *BaseInstaller) lookupFluentBitBinPath() (string, error) {
stdout, stderr, err := this.client.Exec("if command -v fluent-bit >/dev/null 2>&1; then command -v fluent-bit; elif [ -x " + fluentBitDefaultBinPath + " ]; then echo " + fluentBitDefaultBinPath + "; fi")
if err != nil {
return "", fmt.Errorf("lookup fluent-bit binary failed: %w, stderr: %s", err, stderr)
}
return strings.TrimSpace(stdout), nil
}
func (this *BaseInstaller) copyFluentBitConfigIfMissing(tempDir string) (bool, error) {
targets := []struct {
Src string
Dest string
}{
{Src: tempDir + "/fluent-bit.conf", Dest: fluentBitMainConfigFile},
{Src: tempDir + "/parsers.conf", Dest: fluentBitParsersFile},
{Src: tempDir + "/clickhouse-upstream.conf", Dest: fluentBitUpstreamFile},
{Src: tempDir + "/logrotate.conf", Dest: fluentBitLogrotateFile},
}
copied := false
for _, target := range targets {
exists, err := this.remoteFileExists(target.Dest)
if err != nil {
return false, err
}
if exists {
continue
}
_, stderr, err := this.client.Exec("cp -f " + target.Src + " " + target.Dest)
if err != nil {
return false, fmt.Errorf("copy fluent-bit file to '%s' failed: %w, stderr: %s", target.Dest, err, stderr)
}
copied = true
}
return copied, nil
}
func (this *BaseInstaller) validateExistingConfigForRole(role nodeconfigs.NodeRole) error {
requiredPatterns := []string{}
switch role {
case nodeconfigs.NodeRoleNode:
requiredPatterns = append(requiredPatterns, fluentBitHTTPPathPattern)
case nodeconfigs.NodeRoleDNS:
requiredPatterns = append(requiredPatterns, fluentBitDNSPathPattern)
}
for _, pattern := range requiredPatterns {
ok, err := this.remoteFileContains(fluentBitMainConfigFile, pattern)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("existing fluent-bit config '%s' does not contain required path '%s'; skip overwrite by design, please update config manually", fluentBitMainConfigFile, pattern)
}
}
return nil
}
func (this *BaseInstaller) remoteFileExists(path string) (bool, error) {
stdout, stderr, err := this.client.Exec("if [ -f \"" + path + "\" ]; then echo 1; else echo 0; fi")
if err != nil {
return false, fmt.Errorf("check remote file '%s' failed: %w, stderr: %s", path, err, stderr)
}
return strings.TrimSpace(stdout) == "1", nil
}
func (this *BaseInstaller) remoteFileContains(path string, pattern string) (bool, error) {
stdout, stderr, err := this.client.Exec("if grep -F \"" + pattern + "\" \"" + path + "\" >/dev/null 2>&1; then echo 1; else echo 0; fi")
if err != nil {
return false, fmt.Errorf("check remote file content '%s' failed: %w, stderr: %s", path, err, stderr)
}
return strings.TrimSpace(stdout) == "1", nil
}
func (this *BaseInstaller) ensureFluentBitService(binPath string, configCopied bool) error {
_, _, _ = this.client.Exec("if command -v systemctl >/dev/null 2>&1 && [ ! -f /etc/systemd/system/" + fluentBitServiceName + ".service ] && [ ! -f /lib/systemd/system/" + fluentBitServiceName + ".service ]; then " +
"cat > /etc/systemd/system/" + fluentBitServiceName + ".service <<'EOF'\n" +
"[Unit]\n" +
"Description=Fluent Bit\n" +
"After=network.target\n" +
"\n" +
"[Service]\n" +
"ExecStart=" + binPath + " -c " + fluentBitMainConfigFile + "\n" +
"Restart=always\n" +
"RestartSec=5\n" +
"\n" +
"[Install]\n" +
"WantedBy=multi-user.target\n" +
"EOF\n" +
"fi")
stdout, stderr, err := this.client.Exec("if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload; systemctl enable " + fluentBitServiceName + " >/dev/null 2>&1 || true; if systemctl is-active " + fluentBitServiceName + " >/dev/null 2>&1; then " +
"if [ \"" + boolToString(configCopied) + "\" = \"1\" ]; then systemctl restart " + fluentBitServiceName + "; fi; " +
"else systemctl start " + fluentBitServiceName + "; fi; else echo no-systemctl; fi")
if err != nil {
return fmt.Errorf("ensure fluent-bit service failed: %w, stderr: %s", err, stderr)
}
if strings.TrimSpace(stdout) == "no-systemctl" {
_, _, runningErr := this.client.Exec("pgrep -f \"fluent-bit.*fluent-bit.conf\" >/dev/null 2>&1")
if runningErr != nil {
_, stderr, err = this.client.Exec(binPath + " -c " + fluentBitMainConfigFile + " >/dev/null 2>&1 &")
if err != nil {
return fmt.Errorf("start fluent-bit without systemd failed: %w, stderr: %s", err, stderr)
}
}
}
return nil
}
func boolToString(v bool) string {
if v {
return "1"
}
return "0"
}

View File

@@ -137,6 +137,13 @@ secret: "${nodeSecret}"`)
}
}
// 在线安装/更新 Fluent Bit与边缘节点安装流程联动
err = this.SetupFluentBit(nodeconfigs.NodeRoleNode)
if err != nil {
installStatus.ErrorCode = "SETUP_FLUENT_BIT_FAILED"
return fmt.Errorf("setup fluent-bit failed: %w", err)
}
// 测试
_, stderr, err = this.client.Exec(dir + "/edge-node/bin/edge-node test")
if err != nil {

View File

@@ -139,6 +139,13 @@ secret: "${nodeSecret}"`)
}
}
// 在线安装/更新 Fluent Bit与 DNS 节点安装流程联动)
err = this.SetupFluentBit(nodeconfigs.NodeRoleDNS)
if err != nil {
installStatus.ErrorCode = "SETUP_FLUENT_BIT_FAILED"
return fmt.Errorf("setup fluent-bit failed: %w", err)
}
// 测试
_, stderr, err = this.client.Exec(dir + "/edge-dns/bin/edge-dns test")
if err != nil {

View File

@@ -1,7 +1,7 @@
FROM --platform=linux/amd64 alpine:latest
FROM --platform=linux/amd64 alpine:latest
LABEL maintainer="goedge.cdn@gmail.com"
ENV TZ "Asia/Shanghai"
ENV VERSION 1.4.6
ENV VERSION 1.4.7
ENV ROOT_DIR /usr/local/goedge
ENV TAR_FILE edge-admin-linux-amd64-plus-v${VERSION}.zip

View File

@@ -1,9 +1,9 @@
package teaconst
package teaconst
const (
Version = "1.4.6" //1.3.9
Version = "1.4.7" //1.3.9
APINodeVersion = "1.4.6" //1.3.9
APINodeVersion = "1.4.7" //1.3.9
ProductName = "Edge Admin"
ProcessName = "edge-admin"
@@ -18,3 +18,4 @@ const (
SystemdServiceName = "edge-admin"
UpdatesURL = "https://goedge.cn/api/boot/versions?os=${os}&arch=${arch}&version=${version}"
)

View File

@@ -9,6 +9,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
"github.com/iwind/TeaGo/actions"
"strings"
)
const clickhouseConfigCode = "clickhouseConfig"
@@ -28,7 +29,7 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
this.ErrorPage(err)
return
}
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default", Scheme: "http"}
if len(resp.ValueJSON) > 0 {
_ = json.Unmarshal(resp.ValueJSON, cfg)
}
@@ -38,22 +39,31 @@ func (this *ClickHouseAction) RunGet(params struct{}) {
if cfg.Database == "" {
cfg.Database = "default"
}
if strings.TrimSpace(cfg.Scheme) == "" {
cfg.Scheme = "http"
}
this.Data["config"] = map[string]interface{}{
"host": cfg.Host,
"port": cfg.Port,
"user": cfg.User,
"password": cfg.Password,
"database": cfg.Database,
"host": cfg.Host,
"port": cfg.Port,
"user": cfg.User,
"password": cfg.Password,
"database": cfg.Database,
"scheme": cfg.Scheme,
"tlsSkipVerify": cfg.TLSSkipVerify,
"tlsServerName": cfg.TLSServerName,
}
this.Show()
}
func (this *ClickHouseAction) RunPost(params struct {
Host string
Port int
User string
Password string
Database string
Host string
Port int
User string
Password string
Database string
Scheme string
TLSSkipVerify bool
TLSServerName string
Must *actions.Must
}) {
@@ -64,6 +74,9 @@ func (this *ClickHouseAction) RunPost(params struct {
if params.Database == "" {
params.Database = "default"
}
if params.Scheme != "https" {
params.Scheme = "http"
}
password := params.Password
if password == "" {
resp, _ := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: clickhouseConfigCode})
@@ -75,11 +88,14 @@ func (this *ClickHouseAction) RunPost(params struct {
}
}
cfg := &systemconfigs.ClickHouseSetting{
Host: params.Host,
Port: params.Port,
User: params.User,
Password: password,
Database: params.Database,
Host: params.Host,
Port: params.Port,
User: params.User,
Password: password,
Database: params.Database,
Scheme: params.Scheme,
TLSSkipVerify: params.TLSSkipVerify,
TLSServerName: strings.TrimSpace(params.TLSServerName),
}
valueJSON, err := json.Marshal(cfg)
if err != nil {

View File

@@ -50,8 +50,8 @@ func (this *AdvancedHelper) BeforeAction(actionPtr actions.ActionWrapper) (goNex
}
// 外层始终显示「日志数据库」与「ClickHouse 配置」两个标签,不随点击变化
path := action.Request.URL.Path
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabAccessLogDatabases), "", "/db", "", (path == "/db" || strings.HasPrefix(path, "/db/")) && path != "/db/clickhouse")
tabbar.Add(this.Lang(actionPtr, codes.DBNode_TabClickHouse), "", "/db/clickhouse", "", path == "/db/clickhouse")
tabbar.Add("日志数据库MySQL", "", "/db", "", (path == "/db" || strings.HasPrefix(path, "/db/")) && path != "/db/clickhouse")
tabbar.Add("日志数据库(ClickHouse", "", "/db/clickhouse", "", path == "/db/clickhouse")
if teaconst.IsPlus {
// 目前仅在调试模式下使用
if Tea.IsTesting() {

View File

@@ -80,9 +80,7 @@ Vue.component("http-access-log-box", {
<div>
<a v-if="accessLog.node != null && accessLog.node.nodeCluster != null" :href="'/clusters/cluster/node?nodeId=' + accessLog.node.id + '&clusterId=' + accessLog.node.nodeCluster.id" title="点击查看节点详情" target="_top"><span class="grey">[{{accessLog.node.name}}<span v-if="!accessLog.node.name.endsWith('节点')">节点</span>]</span></a>
<!-- 网站 -->
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="点击到网站" v-if="vShowServerLink && accessLog.serverId > 0"><span class="grey">[网站]</span></a>
<span v-if="vShowServerLink && (accessLog.serverId == null || accessLog.serverId == 0)" @click.prevent="mismatch()"><span class="disabled">[网站]</span></span>
<span v-if="accessLog.region != null && accessLog.region.length > 0" class="grey"><ip-box :v-ip="accessLog.remoteAddr">[{{accessLog.region}}]</ip-box></span>
<ip-box><keyword :v-word="vKeyword">{{accessLog.remoteAddr}}</keyword></ip-box> [{{accessLog.timeLocal}}] <em>&quot;<keyword :v-word="vKeyword">{{accessLog.requestMethod}}</keyword> {{accessLog.scheme}}://<keyword :v-word="vKeyword">{{accessLog.host}}</keyword><keyword :v-word="vKeyword">{{accessLog.requestURI}}</keyword> <a :href="accessLog.scheme + '://' + accessLog.host + accessLog.requestURI" target="_blank" title="新窗口打开" class="disabled"><i class="external icon tiny"></i> </a> {{accessLog.proto}}&quot; </em> <keyword :v-word="vKeyword">{{accessLog.status}}</keyword>
@@ -110,6 +108,7 @@ Vue.component("http-access-log-box", {
</span>
<span v-if="accessLog.requestTime != null"> - 耗时:{{formatCost(accessLog.requestTime)}} ms </span><span v-if="accessLog.humanTime != null && accessLog.humanTime.length > 0" class="grey small">&nbsp; ({{accessLog.humanTime}})</span>
<a :href="'/servers/server/log?serverId=' + accessLog.serverId" title="仅看此网站日志" v-if="vShowServerLink && accessLog.serverId > 0" class="ui label tiny blue basic" style="font-weight: normal; margin-left: 0.5em; padding: 2px 5px !important">网站</a>
&nbsp; <a href="" @click.prevent="showLog" title="查看详情"><i class="icon expand"></i></a>
</div>
</div>`

View File

@@ -14,11 +14,21 @@
<p class="comment">ClickHouse 服务器地址。</p>
</td>
</tr>
<tr>
<td>协议Scheme</td>
<td>
<select name="scheme" class="ui dropdown auto-width">
<option value="http" :selected="config.scheme != 'https'">HTTP</option>
<option value="https" :selected="config.scheme == 'https'">HTTPS</option>
</select>
<p class="comment">默认 HTTP选择 HTTPS 时将启用 TLS 连接。</p>
</td>
</tr>
<tr>
<td>端口Port</td>
<td>
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/>
<p class="comment">HTTP 接口端口,默认 8123</p>
<p class="comment">接口端口HTTP 默认 8123HTTPS 常用 8443以你的 ClickHouse 实际配置为准)</p>
</td>
</tr>
<tr>
@@ -34,6 +44,20 @@
<p class="comment">留空则不修改已保存的密码。</p>
</td>
</tr>
<tr>
<td>TLS 跳过证书校验</td>
<td>
<checkbox name="tlsSkipVerify" value="1" :checked="config.tlsSkipVerify"></checkbox>
<p class="comment">仅测试环境建议开启;生产建议关闭并使用受信任证书。</p>
</td>
</tr>
<tr>
<td>TLS Server Name</td>
<td>
<input type="text" name="tlsServerName" maxlength="200" placeholder="可选证书校验域名SNI" :value="config.tlsServerName"/>
<p class="comment">可选;当 ClickHouse 证书域名与连接 Host 不一致时使用。</p>
</td>
</tr>
<tr>
<td>数据库名Database</td>
<td>

View File

@@ -1,11 +1,13 @@
package systemconfigs
// ClickHouseSetting 后台页面配置的 ClickHouse 连接(访问日志 logs_ingest 查询)
type ClickHouseSetting struct {
Host string `json:"host" yaml:"host"`
Port int `json:"port" yaml:"port"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
Database string `json:"database" yaml:"database"`
Host string `json:"host" yaml:"host"`
Port int `json:"port" yaml:"port"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
Database string `json:"database" yaml:"database"`
Scheme string `json:"scheme" yaml:"scheme"`
TLSSkipVerify bool `json:"tlsSkipVerify" yaml:"tlsSkipVerify"`
TLSServerName string `json:"tlsServerName" yaml:"tlsServerName"`
}

View File

@@ -37,6 +37,7 @@ function build() {
fi
cp "$ROOT"/configs/api_dns.template.yaml "$DIST"/configs
copy_fluent_bit_assets "$ROOT" "$DIST" "$OS" "$ARCH" || exit 1
echo "building ..."
@@ -94,6 +95,43 @@ function build() {
echo "OK"
}
function copy_fluent_bit_assets() {
ROOT=$1
DIST=$2
OS=$3
ARCH=$4
FLUENT_ROOT="$ROOT/../../deploy/fluent-bit"
FLUENT_DIST="$DIST/deploy/fluent-bit"
if [ ! -d "$FLUENT_ROOT" ]; then
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
return 1
fi
rm -rf "$FLUENT_DIST"
mkdir -p "$FLUENT_DIST"
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
if [ -f "$FLUENT_ROOT/$file" ]; then
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
fi
done
if [ "$OS" = "linux" ]; then
PACKAGE_SRC="$FLUENT_ROOT/packages/linux-$ARCH"
PACKAGE_DST="$FLUENT_DIST/packages/linux-$ARCH"
if [ -d "$PACKAGE_SRC" ]; then
mkdir -p "$PACKAGE_DST"
cp -R "$PACKAGE_SRC/." "$PACKAGE_DST/"
else
echo "[error] fluent-bit package directory not found: $PACKAGE_SRC"
return 1
fi
fi
return 0
}
function lookup-version() {
FILE=$1
VERSION_DATA=$(cat "$FILE")

View File

@@ -1,7 +1,7 @@
package teaconst
package teaconst
const (
Version = "1.4.6" //1.3.8.2
Version = "1.4.7" //1.3.8.2
ProductName = "Edge DNS"
ProcessName = "edge-dns"
@@ -13,3 +13,4 @@ const (
SystemdServiceName = "edge-dns"
)

View File

@@ -61,6 +61,7 @@ function build() {
cp "$ROOT"/configs/cluster.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
copy_fluent_bit_assets "$ROOT" "$DIST" "$OS" "$ARCH" || exit 1
# we support TOA on linux only
if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]
@@ -168,6 +169,43 @@ function build() {
echo "OK"
}
function copy_fluent_bit_assets() {
ROOT=$1
DIST=$2
OS=$3
ARCH=$4
FLUENT_ROOT="$ROOT/../../deploy/fluent-bit"
FLUENT_DIST="$DIST/deploy/fluent-bit"
if [ ! -d "$FLUENT_ROOT" ]; then
echo "[error] fluent-bit source directory not found: $FLUENT_ROOT"
return 1
fi
rm -rf "$FLUENT_DIST"
mkdir -p "$FLUENT_DIST"
for file in fluent-bit.conf fluent-bit-dns.conf parsers.conf clickhouse-upstream.conf logrotate.conf README.md; do
if [ -f "$FLUENT_ROOT/$file" ]; then
cp "$FLUENT_ROOT/$file" "$FLUENT_DIST/"
fi
done
if [ "$OS" = "linux" ]; then
PACKAGE_SRC="$FLUENT_ROOT/packages/linux-$ARCH"
PACKAGE_DST="$FLUENT_DIST/packages/linux-$ARCH"
if [ -d "$PACKAGE_SRC" ]; then
mkdir -p "$PACKAGE_DST"
cp -R "$PACKAGE_SRC/." "$PACKAGE_DST/"
else
echo "[error] fluent-bit package directory not found: $PACKAGE_SRC"
return 1
fi
fi
return 0
}
function lookup-version() {
FILE=$1
VERSION_DATA=$(cat "$FILE")

View File

@@ -1,7 +1,7 @@
package teaconst
package teaconst
const (
Version = "1.4.6" //1.3.8.2
Version = "1.4.7" //1.3.8.2
ProductName = "Edge Node"
ProcessName = "edge-node"
@@ -18,3 +18,4 @@ const (
EnableKVCacheStore = true // determine store cache keys in KVStore or sqlite
)

View File

@@ -1,7 +1,7 @@
package teaconst
package teaconst
const (
Version = "1.4.6" //1.3.8.2
Version = "1.4.7" //1.3.8.2
ProductName = "Edge User"
ProcessName = "edge-user"
@@ -21,3 +21,4 @@ const (
IsPlus = true
)

View File

@@ -347,3 +347,67 @@ Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch
若在管理端设置了公用访问日志策略的文件 `path`,节点会优先使用该目录;否则才使用 `EDGE_LOG_DIR`。Fluent Bit 的 `Path` 需与实际目录一致。
以上完成即完成 Fluent Bit 的部署与验证。
---
## 九、HTTPS 模式ClickHouse
当 ClickHouse 只开放 HTTPS如 8443或链路必须加密时使用本目录新增模板
- `fluent-bit-https.conf`Node+DNS 同机采集HTTP+DNS 双输入)
- `fluent-bit-dns-https.conf`:仅 DNS 节点采集
- `fluent-bit-windows-https.conf`Windows 节点 HTTPS 采集
### 9.1 什么时候用 HTTPS 模板
- ClickHouse 仅开放 HTTPS 端口;
- 节点到 ClickHouse 跨公网或需要传输加密;
- 你希望启用证书校验和 SNI。
### 9.2 最小切换步骤Linux
1. 备份当前配置:
```bash
sudo cp /etc/fluent-bit/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf.bak
```
2. 切换为 HTTPS 模板Node+DNS 同机示例):
```bash
sudo cp /path/to/fluent-bit-https.conf /etc/fluent-bit/fluent-bit.conf
```
3. 设置账号密码(按你的服务文件方式设置):
```bash
export CH_USER=default
export CH_PASSWORD='your_password'
```
4. 修改模板中的关键项:
- `Host` / `Port`HTTPS 常见端口 `8443`
- `tls.verify``On`/`Off`
- `tls.ca_file`:自签名证书建议配置 CA 文件
- `tls.vhost`:证书 CN/SAN 对应主机名SNI
5. 重启并检查:
```bash
sudo systemctl restart fluent-bit
sudo systemctl status fluent-bit
journalctl -u fluent-bit -f
```
### 9.3 验证点
- `default.logs_ingest` 有新增数据HTTP
- `default.dns_logs_ingest` 有新增数据DNS
- Fluent Bit 日志中无 TLS 握手失败(`certificate`, `x509`, `tls`
### 9.4 回滚
TLS 配置错误导致中断时,快速回滚:
```bash
sudo cp /etc/fluent-bit/fluent-bit.conf.bak /etc/fluent-bit/fluent-bit.conf
sudo systemctl restart fluent-bit
```
回滚后恢复原 HTTP 模式,不影响平台 API/管理端配置。

View File

@@ -0,0 +1,39 @@
# DNS 节点专用 HTTPS使用 HTTP 输出写入 ClickHouse无需 out_clickhouse 插件)
# 启动前设置CH_USER、CH_PASSWORD按需修改 Host、Port默认 127.0.0.1:8443
[SERVICE]
Flush 5
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 128MB
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Parser json
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
# tls.vhost clickhouse.example.com
json_date_key timestamp
json_date_format epoch
Retry_Limit 10

View File

@@ -0,0 +1,74 @@
# Fluent Bit HTTPS 配置(边缘节点日志采集 -> ClickHouse HTTPS
# HTTP: /var/log/edge/edge-node/*.log
# DNS: /var/log/edge/edge-dns/*.log
#
# 启动前请设置环境变量:
# CH_USER=default
# CH_PASSWORD=your_password
# 如需改地址/端口,请修改 OUTPUT 中 Host/Port默认 127.0.0.1:8443
# 如证书为公网CA可省略 tls.ca_file自签名证书请配置 tls.ca_file
[SERVICE]
Flush 5
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 128MB
[INPUT]
Name tail
Path /var/log/edge/edge-node/*.log
Tag app.http.logs
Parser json
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/http-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/edge/edge-dns/*.log
Tag app.dns.logs
Parser json
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/dns-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.http.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
# tls.vhost clickhouse.example.com
json_date_key timestamp
json_date_format epoch
Retry_Limit 10
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
# tls.vhost clickhouse.example.com
json_date_key timestamp
json_date_format epoch
Retry_Limit 10

View File

@@ -0,0 +1,62 @@
[SERVICE]
Flush 1
Log_Level info
Parsers_File parsers.conf
storage.path ./storage
storage.sync normal
[INPUT]
Name tail
Path E:\var\log\edge\edge-node\*.log
Tag app.http.logs
Parser json
Refresh_Interval 1
Read_from_Head true
DB ./http-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[INPUT]
Name tail
Path E:\var\log\edge\edge-dns\*.log
Tag app.dns.logs
Parser json
Refresh_Interval 1
Read_from_Head true
DB ./dns-logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name http
Match app.http.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT+INTO+logs_ingest+FORMAT+JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file C:\\path\\to\\ca.pem
# tls.vhost clickhouse.example.com
Json_Date_Key timestamp
Json_Date_Format epoch
Retry_Limit 10
[OUTPUT]
Name http
Match app.dns.logs
Host 127.0.0.1
Port 8443
URI /?query=INSERT+INTO+dns_logs_ingest+FORMAT+JSONEachRow
Format json_lines
http_user ${CH_USER}
http_passwd ${CH_PASSWORD}
tls On
tls.verify On
# tls.ca_file C:\\path\\to\\ca.pem
# tls.vhost clickhouse.example.com
Json_Date_Key timestamp
Json_Date_Format epoch
Retry_Limit 10