主分支代码

This commit is contained in:
robin
2026-02-07 20:30:31 +08:00
parent 3b042d1dad
commit bc223fd1aa
65 changed files with 1969 additions and 188 deletions

View File

@@ -24,6 +24,7 @@ type StorageManager struct {
publicPolicyId int64
disableDefaultDB bool
writeTargets *serverconfigs.AccessLogWriteTargets // 公用策略的写入目标
locker sync.Mutex
}
@@ -79,12 +80,14 @@ func (this *StorageManager) Loop() error {
if int64(policy.Id) == publicPolicyId {
this.disableDefaultDB = policy.DisableDefaultDB
this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB)
foundPolicy = true
}
}
}
if !foundPolicy {
this.disableDefaultDB = false
this.writeTargets = nil
}
this.locker.Lock()
@@ -160,6 +163,27 @@ func (this *StorageManager) DisableDefaultDB() bool {
return this.disableDefaultDB
}
// WriteMySQL 公用策略是否写入 MySQL以 writeTargets 为准,无则用 disableDefaultDB
func (this *StorageManager) WriteMySQL() bool {
if this.writeTargets != nil {
return this.writeTargets.MySQL
}
return !this.disableDefaultDB
}
// WriteClickHouse 公用策略是否写入 ClickHouse文件+Fluent Bit 或后续 API 直写)
func (this *StorageManager) WriteClickHouse() bool {
if this.writeTargets != nil {
return this.writeTargets.ClickHouse
}
return false
}
// WriteTargets 返回公用策略的写入目标(供节点配置注入等)
func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets {
return this.writeTargets
}
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
switch storageType {
case serverconfigs.AccessLogStorageTypeFile:

View File

@@ -0,0 +1,134 @@
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// Client 通过 HTTP 接口执行只读查询SELECT返回 JSONEachRow 解析为 map 或结构体
type Client struct {
cfg *Config
httpCli *http.Client
}
// NewClient 使用共享配置创建客户端
func NewClient() *Client {
cfg := SharedConfig()
return &Client{
cfg: cfg,
httpCli: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// IsConfigured 是否已配置
func (c *Client) IsConfigured() bool {
return c.cfg != nil && c.cfg.IsConfigured()
}
// Query 执行 SELECT将每行 JSON 解析到 dest 切片dest 元素类型需为 *struct 或 map
func (c *Client) Query(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
// 强制 JSONEachRow 便于解析
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeRows(dec, dest)
}
// QueryRow 执行仅返回一行的查询,将结果解析到 dest*struct 或 *map
func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeOneRow(dec, dest)
}
func (c *Client) buildURL(query string) string {
rawURL := fmt.Sprintf("http://%s:%d/?query=%s&database=%s",
c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
return rawURL
}
// decodeRows 将 JSONEachRow 流解析到 slice元素类型须为 *struct 或 *map[string]interface{}
func decodeRows(dec *json.Decoder, dest interface{}) error {
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
switch d := dest.(type) {
case *[]map[string]interface{}:
*d = (*d)[:0]
for {
var row map[string]interface{}
if err := dec.Decode(&row); err != nil {
if err == io.EOF {
return nil
}
return err
}
*d = append(*d, row)
}
default:
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{} or implement decoder)")
}
}
func decodeOneRow(dec *json.Decoder, dest interface{}) error {
switch d := dest.(type) {
case *map[string]interface{}:
if err := dec.Decode(d); err != nil {
return err
}
return nil
default:
return fmt.Errorf("clickhouse: unsupported dest type for QueryRow (use *map[string]interface{})")
}
}

View File

@@ -0,0 +1,111 @@
// Package clickhouse 提供 ClickHouse 只读客户端,用于查询 logs_ingestFluent Bit 写入)。
// 配置优先从后台页面edgeSysSettings.clickhouseConfig读取其次 api.yaml最后环境变量。
package clickhouse
import (
"github.com/TeaOSLab/EdgeAPI/internal/configs"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"os"
"strconv"
"sync"
)
const (
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
defaultPort = 8123
defaultDB = "default"
)
var (
sharedConfig *Config
configOnce sync.Once
configLocker sync.Mutex
)
// Config ClickHouse 连接配置(仅查询,不从代码写库)
type Config struct {
Host string
Port int
User string
Password string
Database string
}
// SharedConfig 返回全局配置(优先从后台 DB 读取,其次 api.yaml最后环境变量
func SharedConfig() *Config {
configLocker.Lock()
defer configLocker.Unlock()
if sharedConfig != nil {
return sharedConfig
}
sharedConfig = loadConfig()
return sharedConfig
}
// ResetSharedConfig 清空缓存,下次 SharedConfig() 时重新从 DB/文件/环境变量加载(后台保存 ClickHouse 配置后调用)
func ResetSharedConfig() {
configLocker.Lock()
defer configLocker.Unlock()
sharedConfig = nil
}
func loadConfig() *Config {
cfg := &Config{Port: defaultPort, Database: defaultDB}
// 1) 优先从后台页面配置DB读取
if models.SharedSysSettingDAO != nil {
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
cfg.Host = dbCfg.Host
cfg.Port = dbCfg.Port
cfg.User = dbCfg.User
cfg.Password = dbCfg.Password
cfg.Database = dbCfg.Database
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
if cfg.Database == "" {
cfg.Database = defaultDB
}
return cfg
}
}
// 2) 其次 api.yaml
apiConfig, err := configs.SharedAPIConfig()
if err == nil && apiConfig != nil && apiConfig.ClickHouse != nil && apiConfig.ClickHouse.Host != "" {
ch := apiConfig.ClickHouse
cfg.Host = ch.Host
cfg.Port = ch.Port
cfg.User = ch.User
cfg.Password = ch.Password
cfg.Database = ch.Database
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
if cfg.Database == "" {
cfg.Database = defaultDB
}
return cfg
}
// 3) 最后环境变量
cfg.Host = os.Getenv(envHost)
cfg.User = os.Getenv(envUser)
cfg.Password = os.Getenv(envPassword)
cfg.Database = os.Getenv(envDatabase)
if cfg.Database == "" {
cfg.Database = defaultDB
}
if p := os.Getenv(envPort); p != "" {
if v, err := strconv.Atoi(p); err == nil {
cfg.Port = v
}
}
return cfg
}
// IsConfigured 是否已配置Host 非空即视为启用 ClickHouse 查询)
func (c *Config) IsConfigured() bool {
return c != nil && c.Host != ""
}

View File

@@ -0,0 +1,285 @@
// Package clickhouse 提供 logs_ingest 表的只读查询(列表分页),用于访问日志列表优先走 ClickHouse。
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// LogsIngestRow 对应 ClickHouse logs_ingest 表的一行(用于 List 结果与 RowToPB
type LogsIngestRow struct {
Timestamp time.Time
NodeId uint64
ClusterId uint64
ServerId uint64
Host string
IP string
Method string
Path string
Status uint16
BytesIn uint64
BytesOut uint64
CostMs uint32
UA string
Referer string
LogType string
TraceId string
FirewallPolicyId uint64
FirewallRuleGroupId uint64
FirewallRuleSetId uint64
FirewallRuleId uint64
RequestHeaders string
RequestBody string
ResponseHeaders string
ResponseBody string
}
// ListFilter 列表查询条件(与 ListHTTPAccessLogsRequest 对齐)
type ListFilter struct {
Day string
HourFrom string
HourTo string
Size int64
Reverse bool
HasError bool
HasFirewallPolicy bool
FirewallPolicyId int64
NodeId int64
ClusterId int64
LastRequestId string
ServerIds []int64
NodeIds []int64
}
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
type LogsIngestStore struct {
client *Client
}
// NewLogsIngestStore 创建 store内部使用共享 Client
func NewLogsIngestStore() *LogsIngestStore {
return &LogsIngestStore{client: NewClient()}
}
// Client 返回底层 Client供调用方判断 IsConfigured()
func (s *LogsIngestStore) Client() *Client {
return s.client
}
// List 按条件分页查询 logs_ingest返回行、下一页游标trace_id与错误
func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsIngestRow, nextCursor string, err error) {
if !s.client.IsConfigured() {
return nil, "", fmt.Errorf("clickhouse: not configured")
}
if f.Day == "" {
return nil, "", fmt.Errorf("clickhouse: day required")
}
table := "logs_ingest"
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest")
} else {
table = quoteIdent(table)
}
conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"}
if f.HourFrom != "" {
if _, err := strconv.Atoi(f.HourFrom); err == nil {
conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom)
}
}
if f.HourTo != "" {
if _, err := strconv.Atoi(f.HourTo); err == nil {
conditions = append(conditions, "toHour(timestamp) <= "+f.HourTo)
}
}
if len(f.ServerIds) > 0 {
parts := make([]string, 0, len(f.ServerIds))
for _, id := range f.ServerIds {
parts = append(parts, strconv.FormatInt(id, 10))
}
conditions = append(conditions, "server_id IN ("+strings.Join(parts, ",")+")")
}
if len(f.NodeIds) > 0 {
parts := make([]string, 0, len(f.NodeIds))
for _, id := range f.NodeIds {
parts = append(parts, strconv.FormatInt(id, 10))
}
conditions = append(conditions, "node_id IN ("+strings.Join(parts, ",")+")")
}
if f.NodeId > 0 {
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NodeId, 10))
}
if f.ClusterId > 0 {
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.ClusterId, 10))
}
if f.HasFirewallPolicy {
conditions = append(conditions, "firewall_policy_id > 0")
}
if f.FirewallPolicyId > 0 {
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
}
where := strings.Join(conditions, " AND ")
orderDir := "ASC"
if f.Reverse {
orderDir = "DESC"
}
limit := f.Size
if limit <= 0 {
limit = 20
}
if limit > 1000 {
limit = 1000
}
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
table, where, orderBy, limit+1)
var rawRows []map[string]interface{}
if err = s.client.Query(ctx, query, &rawRows); err != nil {
return nil, "", err
}
rows = make([]*LogsIngestRow, 0, len(rawRows))
for _, m := range rawRows {
r := mapToLogsIngestRow(m)
if r != nil {
rows = append(rows, r)
}
}
if len(rows) > int(limit) {
nextCursor = rows[limit].TraceId
rows = rows[:limit]
}
return rows, nextCursor, nil
}
func quoteIdent(name string) string {
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
}
func escapeString(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
r := &LogsIngestRow{}
u64 := func(key string) uint64 {
v, ok := m[key]
if !ok || v == nil {
return 0
}
switch x := v.(type) {
case float64:
return uint64(x)
case string:
n, _ := strconv.ParseUint(x, 10, 64)
return n
case json.Number:
n, _ := x.Int64()
return uint64(n)
}
return 0
}
u32 := func(key string) uint32 {
return uint32(u64(key))
}
str := func(key string) string {
if v, ok := m[key]; ok && v != nil {
if s, ok := v.(string); ok {
return s
}
}
return ""
}
ts := func(key string) time.Time {
v, ok := m[key]
if ok && v != nil {
switch x := v.(type) {
case string:
t, _ := time.Parse("2006-01-02 15:04:05", x)
return t
case float64:
return time.Unix(int64(x), 0)
case json.Number:
n, _ := x.Int64()
return time.Unix(n, 0)
}
}
return time.Time{}
}
r.Timestamp = ts("timestamp")
r.NodeId = u64("node_id")
r.ClusterId = u64("cluster_id")
r.ServerId = u64("server_id")
r.Host = str("host")
r.IP = str("ip")
r.Method = str("method")
r.Path = str("path")
r.Status = uint16(u64("status"))
r.BytesIn = u64("bytes_in")
r.BytesOut = u64("bytes_out")
r.CostMs = u32("cost_ms")
r.UA = str("ua")
r.Referer = str("referer")
r.LogType = str("log_type")
r.TraceId = str("trace_id")
r.FirewallPolicyId = u64("firewall_policy_id")
r.FirewallRuleGroupId = u64("firewall_rule_group_id")
r.FirewallRuleSetId = u64("firewall_rule_set_id")
r.FirewallRuleId = u64("firewall_rule_id")
r.RequestHeaders = str("request_headers")
r.RequestBody = str("request_body")
r.ResponseHeaders = str("response_headers")
r.ResponseBody = str("response_body")
return r
}
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog列表展示用
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
if r == nil {
return nil
}
a := &pb.HTTPAccessLog{
RequestId: r.TraceId,
ServerId: int64(r.ServerId),
NodeId: int64(r.NodeId),
Timestamp: r.Timestamp.Unix(),
Host: r.Host,
RawRemoteAddr: r.IP,
RemoteAddr: r.IP,
RequestMethod: r.Method,
RequestPath: r.Path,
Status: int32(r.Status),
RequestLength: int64(r.BytesIn),
BytesSent: int64(r.BytesOut),
RequestTime: float64(r.CostMs) / 1000,
UserAgent: r.UA,
Referer: r.Referer,
FirewallPolicyId: int64(r.FirewallPolicyId),
FirewallRuleGroupId: int64(r.FirewallRuleGroupId),
FirewallRuleSetId: int64(r.FirewallRuleSetId),
FirewallRuleId: int64(r.FirewallRuleId),
}
if r.TimeISO8601() != "" {
a.TimeISO8601 = r.TimeISO8601()
}
return a
}
// TimeISO8601 便于 RowToPB 使用
func (r *LogsIngestRow) TimeISO8601() string {
if r.Timestamp.IsZero() {
return ""
}
return r.Timestamp.UTC().Format("2006-01-02T15:04:05Z07:00")
}

View File

@@ -10,10 +10,20 @@ import (
var sharedAPIConfig *APIConfig = nil
// ClickHouseConfig 仅用于访问日志列表只读查询logs_ingest
type ClickHouseConfig struct {
Host string `yaml:"host" json:"host"`
Port int `yaml:"port" json:"port"`
User string `yaml:"user" json:"user"`
Password string `yaml:"password" json:"password"`
Database string `yaml:"database" json:"database"`
}
// APIConfig API节点配置
type APIConfig struct {
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
ClickHouse *ClickHouseConfig `yaml:"clickhouse,omitempty" json:"clickhouse,omitempty"`
numberId int64 // 数字ID
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "1.4.5" //1.3.9
Version = "1.4.6" //1.3.9
ProductName = "Edge API"
ProcessName = "edge-api"
@@ -17,5 +17,5 @@ const (
// 其他节点版本号,用来检测是否有需要升级的节点
NodeVersion = "1.4.5" //1.3.8.2
NodeVersion = "1.4.6" //1.3.8.2
)

View File

@@ -4,8 +4,8 @@
package teaconst
const (
DNSNodeVersion = "1.4.5" //1.3.8.2
UserNodeVersion = "1.4.5" //1.3.8.2
DNSNodeVersion = "1.4.6" //1.3.8.2
UserNodeVersion = "1.4.6" //1.3.8.2
ReportNodeVersion = "0.1.5"
DefaultMaxNodes int32 = 50

View File

@@ -107,7 +107,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
}
// CreatePolicy 创建策略
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool) (policyId int64, err error) {
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) {
var op = NewHTTPAccessLogPolicyOperator()
op.Name = name
op.Type = policyType
@@ -121,12 +121,15 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
op.IsOn = true
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.State = HTTPAccessLogPolicyStateEnabled
return this.SaveInt64(tx, op)
}
// UpdatePolicy 修改策略
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, isOn bool) error {
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -144,6 +147,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
var op = NewHTTPAccessLogPolicyOperator()
op.Id = policyId
op.Name = name
if policyType != "" {
op.Type = policyType
}
if len(optionsJSON) > 0 {
op.Options = optionsJSON
} else {
@@ -161,6 +167,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
op.IsPublic = isPublic
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.IsOn = isOn
return this.Save(tx, op)
}

View File

@@ -18,6 +18,7 @@ const (
HTTPAccessLogPolicyField_FirewallOnly dbs.FieldName = "firewallOnly" // 是否只记录防火墙相关
HTTPAccessLogPolicyField_Version dbs.FieldName = "version" // 版本号
HTTPAccessLogPolicyField_DisableDefaultDB dbs.FieldName = "disableDefaultDB" // 是否停止默认数据库存储
HTTPAccessLogPolicyField_WriteTargets dbs.FieldName = "writeTargets" // 写入目标 JSONfile/mysql/clickhouse
)
// HTTPAccessLogPolicy 访问日志策略
@@ -37,6 +38,7 @@ type HTTPAccessLogPolicy struct {
FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关
Version uint32 `field:"version"` // 版本号
DisableDefaultDB bool `field:"disableDefaultDB"` // 是否停止默认数据库存储
WriteTargets dbs.JSON `field:"writeTargets"` // 写入目标 JSON{"file":true,"mysql":true,"clickhouse":false}
}
type HTTPAccessLogPolicyOperator struct {
@@ -55,6 +57,7 @@ type HTTPAccessLogPolicyOperator struct {
FirewallOnly any // 是否只记录防火墙相关
Version any // 版本号
DisableDefaultDB any // 是否停止默认数据库存储
WriteTargets any // 写入目标 JSON
}
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

View File

@@ -1168,6 +1168,16 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
if config.GlobalServerConfig == nil {
config.GlobalServerConfig = nodeCluster.DecodeGlobalServerConfig()
}
// 注入公用访问日志策略的写入目标(供节点决定是否写文件、是否上报 API
if config.GlobalServerConfig != nil && clusterIndex == 0 {
publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx)
if publicPolicyId > 0 {
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
if publicPolicy != nil {
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
}
}
}
// 最大线程数、TCP连接数
if clusterIndex == 0 {
@@ -1972,6 +1982,25 @@ func (this *NodeDAO) FindEnabledNodesWithIds(tx *dbs.Tx, nodeIds []int64) (resul
return
}
// FindEnabledBasicNodesWithIds 根据一组ID查找节点基本信息id, name, clusterId用于批量填充日志列表的 Node/Cluster
func (this *NodeDAO) FindEnabledBasicNodesWithIds(tx *dbs.Tx, nodeIds []int64) (result []*Node, err error) {
if len(nodeIds) == 0 {
return nil, nil
}
idStrings := []string{}
for _, nodeId := range nodeIds {
idStrings = append(idStrings, numberutils.FormatInt64(nodeId))
}
_, err = this.Query(tx).
State(NodeStateEnabled).
Where("id IN ("+strings.Join(idStrings, ", ")+")").
Result("id", "name", "clusterId").
Slice(&result).
Reuse(false).
FindAll()
return
}
// DeleteNodeFromCluster 从集群中删除节点
func (this *NodeDAO) DeleteNodeFromCluster(tx *dbs.Tx, nodeId int64, clusterId int64) error {
one, err := this.Query(tx).

View File

@@ -65,3 +65,26 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
}
return config, nil
}
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
if err != nil {
return nil, err
}
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
if len(valueJSON) == 0 {
return out, nil
}
err = json.Unmarshal(valueJSON, out)
if err != nil {
return nil, err
}
if out.Port <= 0 {
out.Port = 8123
}
if out.Database == "" {
out.Database = "default"
}
return out, nil
}

View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
@@ -46,9 +47,8 @@ func (this *HTTPAccessLogService) CreateHTTPAccessLogs(ctx context.Context, req
return &pb.CreateHTTPAccessLogsResponse{}, nil
}
// ListHTTPAccessLogs 列出单页访问日志
// ListHTTPAccessLogs 列出单页访问日志(优先 ClickHouse否则 MySQLClickHouse 路径下节点/集群批量查询避免 N+1
func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *pb.ListHTTPAccessLogsRequest) (*pb.ListHTTPAccessLogsResponse, error) {
// 校验请求
_, userId, err := this.ValidateAdminAndUser(ctx, true)
if err != nil {
return nil, err
@@ -56,11 +56,8 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
var tx = this.NullTx()
// 检查服务ID
if userId > 0 {
req.UserId = userId
// 这里不用担心serverId <= 0 的情况因为如果userId>0则只会查询当前用户下的服务不会产生安全问题
if req.ServerId > 0 {
err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId)
if err != nil {
@@ -69,6 +66,17 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
}
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() && req.Day != "" {
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
if listErr != nil {
return nil, listErr
}
if resp != nil {
return resp, nil
}
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil {
return nil, err
@@ -82,8 +90,6 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
if err != nil {
return nil, err
}
// 节点 & 集群
pbNode, ok := pbNodeMap[a.NodeId]
if ok {
a.Node = pbNode
@@ -94,42 +100,131 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
if node != nil {
pbNode = &pb.Node{Id: int64(node.Id), Name: node.Name}
var clusterId = int64(node.ClusterId)
pbCluster, ok := pbClusterMap[clusterId]
if ok {
pbNode.NodeCluster = pbCluster
} else {
if !ok {
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, clusterId)
if err != nil {
return nil, err
}
if cluster != nil {
pbCluster = &pb.NodeCluster{
Id: int64(cluster.Id),
Name: cluster.Name,
}
pbNode.NodeCluster = pbCluster
pbCluster = &pb.NodeCluster{Id: int64(cluster.Id), Name: cluster.Name}
pbClusterMap[clusterId] = pbCluster
}
}
if pbCluster != nil {
pbNode.NodeCluster = pbCluster
}
pbNodeMap[a.NodeId] = pbNode
a.Node = pbNode
}
}
result = append(result, a)
}
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result, // TODO 仅仅为了兼容当用户节点版本大于0.0.8时可以删除
AccessLogs: result,
HasMore: hasMore,
RequestId: requestId,
}, nil
}
// listHTTPAccessLogsFromClickHouse 从 ClickHouse logs_ingest 查列表,并批量填充 Node/NodeCluster避免 N+1
func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.LogsIngestStore, req *pb.ListHTTPAccessLogsRequest, userId int64) (*pb.ListHTTPAccessLogsResponse, error) {
f := clickhouse.ListFilter{
Day: req.Day,
HourFrom: req.HourFrom,
HourTo: req.HourTo,
Size: req.Size,
Reverse: req.Reverse,
HasError: req.HasError,
HasFirewallPolicy: req.HasFirewallPolicy,
FirewallPolicyId: req.FirewallPolicyId,
NodeId: req.NodeId,
ClusterId: req.NodeClusterId,
LastRequestId: req.RequestId,
}
if req.ServerId > 0 {
f.ServerIds = []int64{req.ServerId}
} else if userId > 0 {
serverIds, err := models.SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
if err != nil {
return nil, err
}
if len(serverIds) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: nil, AccessLogs: nil, HasMore: false, RequestId: ""}, nil
}
f.ServerIds = serverIds
}
if req.NodeClusterId > 0 {
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, req.NodeClusterId)
if err != nil {
return nil, err
}
f.NodeIds = nodeIds
}
rows, nextCursor, err := store.List(ctx, f)
if err != nil {
return nil, err
}
if len(rows) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: []*pb.HTTPAccessLog{}, AccessLogs: []*pb.HTTPAccessLog{}, HasMore: false, RequestId: ""}, nil
}
result := make([]*pb.HTTPAccessLog, 0, len(rows))
nodeIdSet := make(map[int64]struct{})
for _, r := range rows {
result = append(result, clickhouse.RowToPB(r))
nodeIdSet[int64(r.NodeId)] = struct{}{}
}
nodeIds := make([]int64, 0, len(nodeIdSet))
for id := range nodeIdSet {
nodeIds = append(nodeIds, id)
}
nodes, err := models.SharedNodeDAO.FindEnabledBasicNodesWithIds(tx, nodeIds)
if err != nil {
return nil, err
}
clusterIds := make(map[int64]struct{})
for _, node := range nodes {
if node.ClusterId > 0 {
clusterIds[int64(node.ClusterId)] = struct{}{}
}
}
clusterIdList := make([]int64, 0, len(clusterIds))
for cid := range clusterIds {
clusterIdList = append(clusterIdList, cid)
}
clusters, _ := models.SharedNodeClusterDAO.FindEnabledNodeClustersWithIds(tx, clusterIdList)
clusterMap := make(map[int64]*pb.NodeCluster)
for _, c := range clusters {
clusterMap[int64(c.Id)] = &pb.NodeCluster{Id: int64(c.Id), Name: c.Name}
}
pbNodeMap := make(map[int64]*pb.Node)
for _, node := range nodes {
pbNode := &pb.Node{Id: int64(node.Id), Name: node.Name}
if c := clusterMap[int64(node.ClusterId)]; c != nil {
pbNode.NodeCluster = c
}
pbNodeMap[int64(node.Id)] = pbNode
}
for _, a := range result {
if n := pbNodeMap[a.NodeId]; n != nil {
a.Node = n
}
}
hasMore := nextCursor != ""
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result,
HasMore: hasMore,
RequestId: nextCursor,
}, nil
}
// FindHTTPAccessLog 查找单个日志
func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb.FindHTTPAccessLogRequest) (*pb.FindHTTPAccessLogResponse, error) {
// 校验请求

View File

@@ -11,7 +11,7 @@ import (
)
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return !accesslogs.SharedStorageManager.DisableDefaultDB()
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {

View File

@@ -53,6 +53,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
})
}
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -76,7 +77,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
}
// 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB)
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
if err != nil {
return nil, err
}
@@ -101,7 +102,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
}
// 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.IsOn)
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
if err != nil {
return nil, err
}
@@ -133,6 +134,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
}}, nil
}

View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
@@ -26,6 +27,11 @@ func (this *SysSettingService) UpdateSysSetting(ctx context.Context, req *pb.Upd
return nil, err
}
// 若为 ClickHouse 配置,清空缓存使下次读取生效
if req.Code == "clickhouseConfig" {
clickhouse.ResetSharedConfig()
}
return this.Success()
}

View File

@@ -113017,7 +113017,7 @@
"name": "edgeHTTPAccessLogPolicies",
"engine": "InnoDB",
"charset": "utf8mb4_unicode_ci",
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n `writeTargets` json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
"fields": [
{
"name": "id",
@@ -113078,6 +113078,10 @@
{
"name": "disableDefaultDB",
"definition": "tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储'"
},
{
"name": "writeTargets",
"definition": "json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse'"
}
],
"indexes": [

View File

@@ -1,7 +1,7 @@
FROM --platform=linux/amd64 alpine:latest
LABEL maintainer="goedge.cdn@gmail.com"
ENV TZ "Asia/Shanghai"
ENV VERSION 1.4.5
ENV VERSION 1.4.6
ENV ROOT_DIR /usr/local/goedge
ENV TAR_FILE edge-admin-linux-amd64-plus-v${VERSION}.zip

View File

@@ -1,9 +1,9 @@
package teaconst
const (
Version = "1.4.5" //1.3.9
Version = "1.4.6" //1.3.9
APINodeVersion = "1.4.5" //1.3.9
APINodeVersion = "1.4.6" //1.3.9
ProductName = "Edge Admin"
ProcessName = "edge-admin"

View File

@@ -0,0 +1,98 @@
//go:build plus
package db
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
"github.com/iwind/TeaGo/actions"
)
const clickhouseConfigCode = "clickhouseConfig"
type ClickHouseAction struct {
actionutils.ParentAction
}
func (this *ClickHouseAction) Init() {
this.Nav("db", "db", "clickhouse")
}
func (this *ClickHouseAction) RunGet(params struct{}) {
this.Data["mainTab"] = "clickhouse"
resp, err := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: clickhouseConfigCode})
if err != nil {
this.ErrorPage(err)
return
}
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
if len(resp.ValueJSON) > 0 {
_ = json.Unmarshal(resp.ValueJSON, cfg)
}
if cfg.Port <= 0 {
cfg.Port = 8123
}
if cfg.Database == "" {
cfg.Database = "default"
}
this.Data["config"] = map[string]interface{}{
"host": cfg.Host,
"port": cfg.Port,
"user": cfg.User,
"password": cfg.Password,
"database": cfg.Database,
}
this.Show()
}
func (this *ClickHouseAction) RunPost(params struct {
Host string
Port int
User string
Password string
Database string
Must *actions.Must
}) {
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
if params.Port <= 0 {
params.Port = 8123
}
if params.Database == "" {
params.Database = "default"
}
password := params.Password
if password == "" {
resp, _ := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: clickhouseConfigCode})
if len(resp.ValueJSON) > 0 {
var old systemconfigs.ClickHouseSetting
if json.Unmarshal(resp.ValueJSON, &old) == nil {
password = old.Password
}
}
}
cfg := &systemconfigs.ClickHouseSetting{
Host: params.Host,
Port: params.Port,
User: params.User,
Password: password,
Database: params.Database,
}
valueJSON, err := json.Marshal(cfg)
if err != nil {
this.ErrorPage(err)
return
}
_, err = this.RPC().SysSettingRPC().UpdateSysSetting(this.AdminContext(), &pb.UpdateSysSettingRequest{
Code: clickhouseConfigCode,
ValueJSON: valueJSON,
})
if err != nil {
this.ErrorPage(err)
return
}
this.Success()
}

View File

@@ -0,0 +1,33 @@
//go:build !plus
package db
import (
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
)
type ClickHouseAction struct {
actionutils.ParentAction
}
func (this *ClickHouseAction) Init() {
this.Nav("db", "db", "clickhouse")
}
func (this *ClickHouseAction) RunGet(params struct{}) {
this.Data["mainTab"] = "clickhouse"
this.Data["config"] = map[string]interface{}{
"host": "", "port": 8123, "user": "", "password": "", "database": "default",
}
this.Show()
}
func (this *ClickHouseAction) RunPost(params struct {
Host string
Port int
User string
Password string
Database string
}) {
this.Fail("请使用商业版以在页面上配置 ClickHouse")
}

View File

@@ -23,5 +23,6 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) {
var tabbar = actionutils.NewTabbar()
tabbar.Add(this.Lang(action, codes.DBNode_TabNodes), "", "/db", "", selectedTabbar == "db")
tabbar.Add(this.Lang(action, codes.DBNode_TabClickHouse), "", "/db/clickhouse", "", selectedTabbar == "clickhouse")
actionutils.SetTabbar(action, tabbar)
}

View File

@@ -24,6 +24,7 @@ func init() {
Get("/node", new(NodeAction)).
Get("/logs", new(LogsAction)).
Post("/status", new(StatusAction)).
GetPost("/clickhouse", new(ClickHouseAction)).
EndAll()
})
}

View File

@@ -87,8 +87,13 @@ func (this *CreatePopupAction) RunPost(params struct {
Field("type", params.Type).
Require("请选择存储类型")
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
if writeTargets == nil {
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
}
var options any = nil
switch params.Type {
switch baseType {
case serverconfigs.AccessLogStorageTypeFile:
params.Must.
Field("filePath", params.FilePath).
@@ -170,14 +175,21 @@ func (this *CreatePopupAction) RunPost(params struct {
this.ErrorPage(err)
return
}
writeTargetsMap := map[string]bool{
"file": writeTargets.File,
"mysql": writeTargets.MySQL,
"clickhouse": writeTargets.ClickHouse,
}
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
createResp, err := this.RPC().HTTPAccessLogPolicyRPC().CreateHTTPAccessLogPolicy(this.AdminContext(), &pb.CreateHTTPAccessLogPolicyRequest{
Name: params.Name,
Type: params.Type,
Type: baseType,
OptionsJSON: optionsJSON,
CondsJSON: nil, // TODO
IsPublic: params.IsPublic,
FirewallOnly: params.FirewallOnly,
DisableDefaultDB: params.DisableDefaultDB,
WriteTargetsJSON: writeTargetsJSON,
})
if err != nil {
this.ErrorPage(err)

View File

@@ -46,11 +46,16 @@ func (this *IndexAction) RunGet(params struct{}) {
return
}
}
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
if typeDisplay == "" {
typeDisplay = policy.Type
}
policyMaps = append(policyMaps, maps.Map{
"id": policy.Id,
"name": policy.Name,
"type": policy.Type,
"typeName": serverconfigs.FindAccessLogStorageTypeName(policy.Type),
"typeName": serverconfigs.FindAccessLogStorageTypeName(typeDisplay),
"isOn": policy.IsOn,
"isPublic": policy.IsPublic,
"firewallOnly": policy.FirewallOnly,

View File

@@ -36,11 +36,18 @@ func InitPolicy(parent *actionutils.ParentAction, policyId int64) error {
}
}
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
if typeDisplay == "" {
typeDisplay = policy.Type
}
parent.Data["policy"] = maps.Map{
"id": policy.Id,
"name": policy.Name,
"type": policy.Type,
"typeName": serverconfigs.FindAccessLogStorageTypeName(policy.Type),
"typeDisplay": typeDisplay,
"typeName": serverconfigs.FindAccessLogStorageTypeName(typeDisplay),
"isOn": policy.IsOn,
"isPublic": policy.IsPublic,
"firewallOnly": policy.FirewallOnly,

View File

@@ -39,6 +39,7 @@ func (this *UpdateAction) RunGet(params struct {
func (this *UpdateAction) RunPost(params struct {
PolicyId int64
Name string
Type string // 存储类型含组合file / file_mysql / file_clickhouse / file_mysql_clickhouse / es / tcp / syslog / command
// file
FilePath string
@@ -101,10 +102,17 @@ func (this *UpdateAction) RunPost(params struct {
params.Must.
Field("name", params.Name).
Require("请输入日志策略的名称")
Require("请输入日志策略的名称").
Field("type", params.Type).
Require("请选择存储类型")
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
if writeTargets == nil {
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
}
var options interface{} = nil
switch policy.Type {
switch baseType {
case serverconfigs.AccessLogStorageTypeFile:
params.Must.
Field("filePath", params.FilePath).
@@ -187,15 +195,23 @@ func (this *UpdateAction) RunPost(params struct {
this.ErrorPage(err)
return
}
writeTargetsMap := map[string]bool{
"file": writeTargets.File,
"mysql": writeTargets.MySQL,
"clickhouse": writeTargets.ClickHouse,
}
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
_, err = this.RPC().HTTPAccessLogPolicyRPC().UpdateHTTPAccessLogPolicy(this.AdminContext(), &pb.UpdateHTTPAccessLogPolicyRequest{
HttpAccessLogPolicyId: params.PolicyId,
Name: params.Name,
Type: baseType,
OptionsJSON: optionsJSON,
CondsJSON: nil, // TODO
IsOn: params.IsOn,
IsPublic: params.IsPublic,
FirewallOnly: params.FirewallOnly,
DisableDefaultDB: params.DisableDefaultDB,
WriteTargetsJSON: writeTargetsJSON,
})
if err != nil {
this.ErrorPage(err)

View File

@@ -4,6 +4,8 @@
package settingutils
import (
"strings"
"github.com/TeaOSLab/EdgeAdmin/internal/configloaders"
teaconst "github.com/TeaOSLab/EdgeAdmin/internal/const"
"github.com/TeaOSLab/EdgeAdmin/internal/plus"
@@ -46,7 +48,10 @@ func (this *AdvancedHelper) BeforeAction(actionPtr actions.ActionWrapper) (goNex
if plus.AllowComponent(plus.ComponentCodeUser) {
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabUserNodes), "", "/settings/userNodes", "", this.tab == "userNodes")
}
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabAccessLogDatabases), "", "/db", "", this.tab == "dbNodes")
// 外层始终显示「日志数据库」与「ClickHouse 配置」两个标签,不随点击变化
path := action.Request.URL.Path
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabAccessLogDatabases), "", "/db", "", (path == "/db" || strings.HasPrefix(path, "/db/")) && path != "/db/clickhouse")
tabbar.Add(this.Lang(actionPtr, codes.DBNode_TabClickHouse), "", "/db/clickhouse", "", path == "/db/clickhouse")
if teaconst.IsPlus {
// 目前仅在调试模式下使用
if Tea.IsTesting() {

View File

@@ -1,4 +1,4 @@
<first-menu>
<first-menu v-if="firstMenuItem !== 'clickhouse'">
<menu-item href="/db">所有节点</menu-item>
<span class="item disabled">|</span>
<menu-item :href="'/db/node?nodeId=' + node.id" code="node">"{{node.name}}"详情</menu-item>

View File

@@ -0,0 +1,46 @@
{$layout}
{$template "menu"}
<h3>ClickHouse 配置</h3>
<p class="comment">用于访问日志列表查询logs_ingest 表)。配置后,访问日志列表将优先从 ClickHouse 读取;不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。</p>
<form method="post" class="ui form" data-tea-action="$" data-tea-success="success" @submit.prevent="onSubmit">
<csrf-token></csrf-token>
<table class="ui table definition selectable">
<tr>
<td class="title">连接地址Host</td>
<td>
<input type="text" name="host" maxlength="200" ref="focus" placeholder="如 127.0.0.1 或 clickhouse.example.com" :value="config.host"/>
<p class="comment">ClickHouse 服务器地址。</p>
</td>
</tr>
<tr>
<td>端口Port</td>
<td>
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/>
<p class="comment">HTTP 接口端口,默认 8123。</p>
</td>
</tr>
<tr>
<td>用户名User</td>
<td>
<input type="text" name="user" maxlength="100" :value="config.user"/>
</td>
</tr>
<tr>
<td>密码Password</td>
<td>
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" value=""/>
<p class="comment">留空则不修改已保存的密码。</p>
</td>
</tr>
<tr>
<td>数据库名Database</td>
<td>
<input type="text" name="database" maxlength="100" placeholder="default" :value="config.database"/>
<p class="comment">logs_ingest 表所在库,默认 default。</p>
</td>
</tr>
</table>
<submit-btn></submit-btn>
</form>

View File

@@ -0,0 +1,12 @@
Tea.context(function () {
this.success = function () {
teaweb.success("保存成功")
}
this.onSubmit = function (e) {
e.preventDefault()
e.stopPropagation()
Tea.action("$").post().form(e.target).success(function () {
Tea.Vue.success()
})
}
})

View File

@@ -15,13 +15,14 @@
<td>
<select class="ui dropdown auto-width" name="type" v-model="type">
<option value="">[选择类型]</option>
<option v-for="type in types" :value="type.code">{{type.name}}</option>
<option v-for="t in types" :value="t.code">{{t.name}}</option>
</select>
<p class="comment">可选:文件、文件+MySQL、文件+ClickHouse、文件+MySQL+ClickHouse、ElasticSearch、TCP、Syslog、命令行等。</p>
</td>
</tr>
<!-- 文件 -->
<tbody v-show="type == 'file'">
<!-- 文件(含 文件 / 文件+MySQL / 文件+ClickHouse / 文件+MySQL+ClickHouse -->
<tbody v-show="type == 'file' || type == 'file_mysql' || type == 'file_clickhouse' || type == 'file_mysql_clickhouse'">
<tr>
<td>日志文件路径 *</td>
<td>

View File

@@ -16,12 +16,14 @@
<tr>
<td>存储类型 *</td>
<td>
{{policy.typeName}}
<select class="ui dropdown auto-width" name="type" v-model="policy.typeDisplay">
<option v-for="t in types" :value="t.code">{{t.name}}</option>
</select>
</td>
</tr>
<!-- 文件 -->
<tbody v-show="type == 'file'">
<tbody v-show="policy.typeDisplay == 'file' || policy.typeDisplay == 'file_mysql' || policy.typeDisplay == 'file_clickhouse' || policy.typeDisplay == 'file_mysql_clickhouse'">
<tr>
<td>日志文件路径 *</td>
<td>
@@ -52,7 +54,7 @@
</tbody>
<!-- Elastic Search -->
<tbody v-show="type == 'es'">
<tbody v-show="policy.typeDisplay == 'es'">
<tr>
<td>Endpoint *</td>
<td>
@@ -114,7 +116,7 @@
</tbody>
<!-- TCP Socket -->
<tbody v-show="type == 'tcp'">
<tbody v-show="policy.typeDisplay == 'tcp'">
<tr>
<td>网络协议 *</td>
<td>
@@ -134,7 +136,7 @@
</tbody>
<!-- Syslog -->
<tbody v-show="type == 'syslog'">
<tbody v-show="policy.typeDisplay == 'syslog'">
<tr>
<td>网络协议</td>
<td>
@@ -188,7 +190,7 @@
</tbody>
<!-- 命令行输入流 -->
<tbody v-show="type == 'command'">
<tbody v-show="policy.typeDisplay == 'command'">
<tr>
<td>可执行文件 *</td>
<td>

BIN
EdgeCommon/build/.DS_Store vendored Normal file

Binary file not shown.

View File

@@ -168,6 +168,7 @@ const (
DBNode_LogTruncateTable langs.MessageCode = "db_node@log_truncate_table" // 清空数据库节点 %d 数据表 %s 数据
DBNode_LogUpdateDBNode langs.MessageCode = "db_node@log_update_db_node" // 修改数据库节点 %d
DBNode_TabNodes langs.MessageCode = "db_node@tab_nodes" // 数据库节点
DBNode_TabClickHouse langs.MessageCode = "db_node@tab_clickhouse" // ClickHouse 配置
DDoSProtection_LogUpdateClusterDDoSProtection langs.MessageCode = "ddos_protection@log_update_cluster_ddos_protection" // 修改集群 %d 的DDOS防护设置
DDoSProtection_LogUpdateNodeDDoSProtection langs.MessageCode = "ddos_protection@log_update_node_ddos_protection" // 修改节点 %d 的DDOS防护设置
DNS_LogCreateDomain langs.MessageCode = "dns@log_create_domain" // 添加管理域名到DNS服务商 %d

View File

@@ -169,6 +169,7 @@ func init() {
"db_node@log_truncate_table": "",
"db_node@log_update_db_node": "",
"db_node@tab_nodes": "",
"db_node@tab_clickhouse": "ClickHouse",
"ddos_protection@log_update_cluster_ddos_protection": "",
"ddos_protection@log_update_node_ddos_protection": "",
"dns@log_create_domain": "",

View File

@@ -169,6 +169,7 @@ func init() {
"db_node@log_truncate_table": "清空数据库节点 %d 数据表 %s 数据",
"db_node@log_update_db_node": "修改数据库节点 %d",
"db_node@tab_nodes": "数据库节点",
"db_node@tab_clickhouse": "ClickHouse 配置",
"ddos_protection@log_update_cluster_ddos_protection": "修改集群 %d 的DDOS防护设置",
"ddos_protection@log_update_node_ddos_protection": "修改节点 %d 的DDOS防护设置",
"dns@log_create_domain": "添加管理域名到DNS服务商 %d",

View File

@@ -1,5 +1,6 @@
{
"tab_nodes": "数据库节点",
"tab_clickhouse": "ClickHouse 配置",
"log_create_db_node": "创建数据库节点 %d",
"log_delete_db_node": "删除数据库节点 %d",

View File

@@ -31,6 +31,7 @@ type HTTPAccessLogPolicy struct {
IsPublic bool `protobuf:"varint,7,opt,name=isPublic,proto3" json:"isPublic,omitempty"` // 是否公用
FirewallOnly bool `protobuf:"varint,8,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"` // 是否只记录WAF相关访问日志
DisableDefaultDB bool `protobuf:"varint,9,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"` // 停用默认数据库存储
WriteTargetsJSON []byte `protobuf:"bytes,10,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"` // 写入目标 JSON
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -128,6 +129,13 @@ func (x *HTTPAccessLogPolicy) GetDisableDefaultDB() bool {
return false
}
func (x *HTTPAccessLogPolicy) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
var File_models_model_http_access_log_policy_proto protoreflect.FileDescriptor
var file_models_model_http_access_log_policy_proto_rawDesc = []byte{

View File

@@ -164,6 +164,7 @@ type CreateHTTPAccessLogPolicyRequest struct {
IsPublic bool `protobuf:"varint,5,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
FirewallOnly bool `protobuf:"varint,6,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
DisableDefaultDB bool `protobuf:"varint,7,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
WriteTargetsJSON []byte `protobuf:"bytes,8,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -247,6 +248,13 @@ func (x *CreateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
return false
}
func (x *CreateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
type CreateHTTPAccessLogPolicyResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
@@ -296,12 +304,14 @@ type UpdateHTTPAccessLogPolicyRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
Type string `protobuf:"bytes,10,opt,name=type,proto3" json:"type,omitempty"`
IsOn bool `protobuf:"varint,3,opt,name=isOn,proto3" json:"isOn,omitempty"`
OptionsJSON []byte `protobuf:"bytes,4,opt,name=optionsJSON,proto3" json:"optionsJSON,omitempty"`
CondsJSON []byte `protobuf:"bytes,5,opt,name=condsJSON,proto3" json:"condsJSON,omitempty"`
IsPublic bool `protobuf:"varint,6,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
FirewallOnly bool `protobuf:"varint,7,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
DisableDefaultDB bool `protobuf:"varint,8,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
WriteTargetsJSON []byte `protobuf:"bytes,9,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -350,6 +360,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetName() string {
return ""
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetType() string {
if x != nil {
return x.Type
}
return ""
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetIsOn() bool {
if x != nil {
return x.IsOn
@@ -392,6 +409,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
return false
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
// 查找单个访问日志策略
type FindHTTPAccessLogPolicyRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`

View File

@@ -13,4 +13,5 @@ message HTTPAccessLogPolicy {
bool isPublic = 7; // 是否公用
bool firewallOnly = 8; // 是否只记录WAF相关访问日志
bool disableDefaultDB = 9; // 停用默认数据库存储
bytes writeTargetsJSON = 10; // 写入目标 JSON: {"file":true,"mysql":true,"clickhouse":false}
}

View File

@@ -55,6 +55,7 @@ message CreateHTTPAccessLogPolicyRequest {
bool isPublic = 5;
bool firewallOnly = 6;
bool disableDefaultDB = 7;
bytes writeTargetsJSON = 8;
}
message CreateHTTPAccessLogPolicyResponse {
@@ -65,12 +66,14 @@ message CreateHTTPAccessLogPolicyResponse {
message UpdateHTTPAccessLogPolicyRequest {
int64 httpAccessLogPolicyId = 1;
string name = 2;
string type = 10; // 存储类型file / es / tcp / syslog / command
bool isOn = 3;
bytes optionsJSON = 4;
bytes condsJSON = 5;
bool isPublic = 6;
bool firewallOnly = 7;
bool disableDefaultDB = 8;
bytes writeTargetsJSON = 9;
}
// 查找单个访问日志策略

View File

@@ -4,45 +4,31 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
)
// AccessLogStorageType 访问日志存储类型
// AccessLogStorageType 访问日志存储类型(含「存储+写入目标」组合)
type AccessLogStorageType = string
const (
AccessLogStorageTypeFile AccessLogStorageType = "file"
AccessLogStorageTypeES AccessLogStorageType = "es"
AccessLogStorageTypeTCP AccessLogStorageType = "tcp"
AccessLogStorageTypeSyslog AccessLogStorageType = "syslog"
AccessLogStorageTypeCommand AccessLogStorageType = "command"
AccessLogStorageTypeFile AccessLogStorageType = "file"
AccessLogStorageTypeFileMySQL AccessLogStorageType = "file_mysql"
AccessLogStorageTypeFileClickhouse AccessLogStorageType = "file_clickhouse"
AccessLogStorageTypeFileMySQLClickhouse AccessLogStorageType = "file_mysql_clickhouse"
AccessLogStorageTypeES AccessLogStorageType = "es"
AccessLogStorageTypeTCP AccessLogStorageType = "tcp"
AccessLogStorageTypeSyslog AccessLogStorageType = "syslog"
AccessLogStorageTypeCommand AccessLogStorageType = "command"
)
// FindAllAccessLogStorageTypes 所有存储引擎列表
// FindAllAccessLogStorageTypes 所有存储引擎列表(含写入目标组合:文件、文件+MySQL、文件+ClickHouse 等)
func FindAllAccessLogStorageTypes() []*shared.Definition {
return []*shared.Definition{
{
Name: "文件",
Code: AccessLogStorageTypeFile,
Description: "将日志存储在磁盘文件中",
},
{
Name: "ElasticSearch",
Code: AccessLogStorageTypeES,
Description: "将日志存储在ElasticSearch中",
},
{
Name: "TCP Socket",
Code: AccessLogStorageTypeTCP,
Description: "将日志通过TCP套接字输出",
},
{
Name: "Syslog",
Code: AccessLogStorageTypeSyslog,
Description: "将日志通过syslog输出仅支持Linux",
},
{
Name: "命令行输入流",
Code: AccessLogStorageTypeCommand,
Description: "启动一个命令通过读取stdin接收日志信息",
},
{Name: "文件", Code: AccessLogStorageTypeFile, Description: "节点写本地 JSON 文件"},
{Name: "文件+MySQL", Code: AccessLogStorageTypeFileMySQL, Description: "节点写文件 + API 写 MySQL"},
{Name: "文件+ClickHouse", Code: AccessLogStorageTypeFileClickhouse, Description: "节点写文件 + 落 ClickHouseFluent Bit 或 API 直写)"},
{Name: "文件+MySQL+ClickHouse", Code: AccessLogStorageTypeFileMySQLClickhouse, Description: "节点写文件 + API 写 MySQL + ClickHouse"},
{Name: "ElasticSearch", Code: AccessLogStorageTypeES, Description: "将日志存储在ElasticSearch中"},
{Name: "TCP Socket", Code: AccessLogStorageTypeTCP, Description: "将日志通过TCP套接字输出"},
{Name: "Syslog", Code: AccessLogStorageTypeSyslog, Description: "将日志通过syslog输出仅支持Linux"},
{Name: "命令行输入流", Code: AccessLogStorageTypeCommand, Description: "启动一个命令通过读取stdin接收日志信息"},
}
}
@@ -55,3 +41,65 @@ func FindAccessLogStorageTypeName(storageType string) string {
}
return ""
}
// IsFileBasedStorageType 是否为基于文件的存储(需要显示文件路径等配置)
func IsFileBasedStorageType(code string) bool {
switch code {
case AccessLogStorageTypeFile, AccessLogStorageTypeFileMySQL, AccessLogStorageTypeFileClickhouse, AccessLogStorageTypeFileMySQLClickhouse:
return true
default:
return false
}
}
// ParseStorageTypeAndWriteTargets 从下拉框选中的类型解析出「实际存储类型」与「写入目标」
// 用于创建/更新策略options 按 baseType 填(如 filewriteTargets 按组合填。
func ParseStorageTypeAndWriteTargets(selectedType string) (baseType string, writeTargets *AccessLogWriteTargets) {
writeTargets = &AccessLogWriteTargets{}
switch selectedType {
case AccessLogStorageTypeFile:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
case AccessLogStorageTypeFileMySQL:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.MySQL = true
case AccessLogStorageTypeFileClickhouse:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.ClickHouse = true
case AccessLogStorageTypeFileMySQLClickhouse:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.MySQL = true
writeTargets.ClickHouse = true
case AccessLogStorageTypeES, AccessLogStorageTypeTCP, AccessLogStorageTypeSyslog, AccessLogStorageTypeCommand:
baseType = selectedType
writeTargets.MySQL = true
default:
baseType = selectedType
writeTargets.File = true
writeTargets.MySQL = true
}
return baseType, writeTargets
}
// ComposeStorageTypeDisplay 根据策略的 Type + WriteTargets 得到下拉框显示用的类型 code用于编辑页回显
func ComposeStorageTypeDisplay(policyType string, writeTargets *AccessLogWriteTargets) string {
if policyType != AccessLogStorageTypeFile {
return policyType
}
if writeTargets == nil {
return AccessLogStorageTypeFile
}
if writeTargets.File && writeTargets.MySQL && writeTargets.ClickHouse {
return AccessLogStorageTypeFileMySQLClickhouse
}
if writeTargets.File && writeTargets.MySQL {
return AccessLogStorageTypeFileMySQL
}
if writeTargets.File && writeTargets.ClickHouse {
return AccessLogStorageTypeFileClickhouse
}
return AccessLogStorageTypeFile
}

View File

@@ -0,0 +1,49 @@
// Copyright 2025. All rights reserved.
package serverconfigs
import "encoding/json"
// AccessLogWriteTargets 访问日志写入目标(双写/单写文件、MySQL、ClickHouse
type AccessLogWriteTargets struct {
File bool `yaml:"file" json:"file"` // 写本地 JSON 文件(供 Fluent Bit → ClickHouse 或自用)
MySQL bool `yaml:"mysql" json:"mysql"` // 写 MySQL 默认库按日分表
ClickHouse bool `yaml:"clickhouse" json:"clickhouse"` // 需要落 ClickHouse文件+Fluent Bit 或 API 直写)
}
// NeedReportToAPI 是否需要上报到 API写 MySQL 或 API 直写 ClickHouse 时需要)
func (t *AccessLogWriteTargets) NeedReportToAPI() bool {
if t == nil {
return true // 兼容:未配置时保持原行为,上报
}
return t.MySQL || t.ClickHouse
}
// NeedWriteFile 节点是否需要写本地文件
func (t *AccessLogWriteTargets) NeedWriteFile() bool {
if t == nil {
return true // 兼容:未配置时保持原行为,写文件
}
return t.File
}
// ParseWriteTargetsFromPolicy 从策略的 writeTargets JSON 与旧字段解析;无 writeTargets 时按 type + disableDefaultDB 推断
func ParseWriteTargetsFromPolicy(writeTargetsJSON []byte, policyType string, disableDefaultDB bool) *AccessLogWriteTargets {
if len(writeTargetsJSON) > 0 {
var t AccessLogWriteTargets
if err := json.Unmarshal(writeTargetsJSON, &t); err == nil {
return &t
}
}
// 兼容旧策略type=file 视为写文件,!disableDefaultDB 视为写 MySQL
t := &AccessLogWriteTargets{
File: policyType == AccessLogStorageTypeFile,
MySQL: !disableDefaultDB,
ClickHouse: false,
}
if !t.File && !t.MySQL && !t.ClickHouse {
t.File = true
t.MySQL = true
}
return t
}

View File

@@ -1,10 +1,5 @@
package firewallconfigs
import (
"github.com/iwind/TeaGo/maps"
"regexp"
)
// AllCheckpoints all check points list
var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
{
@@ -307,86 +302,6 @@ var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
IsComposed: true,
Priority: 20,
},
{
Name: "CC统计",
Prefix: "cc",
Description: "统计某段时间段内的请求信息不推荐再使用请使用新的CC2统计代替。",
HasParams: true,
Params: []*KeyValue{
NewKeyValue("请求数", "requests"),
},
Options: []OptionInterface{
&FieldOption{
Type: "field",
Name: "统计周期",
Code: "period",
Value: "60",
IsRequired: false,
Size: 8,
Comment: "",
Placeholder: "",
RightLabel: "秒",
MaxLength: 8,
Validate: func(value string) (ok bool, message string) {
if regexp.MustCompile(`^\d+$`).MatchString(value) {
ok = true
return
}
message = "周期需要是一个整数数字"
return
},
},
&OptionsOption{
Type: "options",
Name: "用户识别读取来源",
Code: "userType",
Value: "",
IsRequired: false,
Size: 10,
Comment: "",
RightLabel: "",
Validate: nil,
Options: []maps.Map{
{
"name": "IP",
"value": "ip",
},
{
"name": "Cookie",
"value": "cookie",
},
{
"name": "URL参数",
"value": "get",
},
{
"name": "POST参数",
"value": "post",
},
{
"name": "HTTP Header",
"value": "header",
},
},
},
&FieldOption{
Type: "field",
Name: "用户识别字段",
Code: "userField",
Comment: "识别用户的唯一性字段在用户读取来源不是IP时使用",
},
&FieldOption{
Type: "field",
Name: "字段读取位置",
Code: "userIndex",
Size: 5,
MaxLength: 5,
Comment: "读取用户识别字段的位置从0开始比如user12345的数字ID 12345的位置就是5在用户读取来源不是IP时使用",
},
},
IsRequest: true,
Priority: 10,
},
{
Name: "响应状态码",
Prefix: "status",

View File

@@ -71,12 +71,13 @@ type GlobalServerConfig struct {
} `yaml:"tcpAll" json:"tcpAll"`
HTTPAccessLog struct {
IsOn bool `yaml:"isOn" json:"isOn"` // 是否启用此功能
EnableRequestHeaders bool `yaml:"enableRequestHeaders" json:"enableRequestHeaders"` // 记录请求Header
CommonRequestHeadersOnly bool `yaml:"commonRequestHeadersOnly" json:"commonRequestHeadersOnly"` // 只保留通用Header
EnableResponseHeaders bool `yaml:"enableResponseHeaders" json:"enableResponseHeaders"` // 记录响应Header
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
IsOn bool `yaml:"isOn" json:"isOn"` // 是否启用此功能
EnableRequestHeaders bool `yaml:"enableRequestHeaders" json:"enableRequestHeaders"` // 记录请求Header
CommonRequestHeadersOnly bool `yaml:"commonRequestHeadersOnly" json:"commonRequestHeadersOnly"` // 只保留通用Header
EnableResponseHeaders bool `yaml:"enableResponseHeaders" json:"enableResponseHeaders"` // 记录响应Header
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse双写/单写)
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
Stat struct {

View File

@@ -0,0 +1,12 @@
//go:build plus
package systemconfigs
// ClickHouseSetting 后台页面配置的 ClickHouse 连接(访问日志 logs_ingest 查询)
type ClickHouseSetting struct {
Host string `json:"host" yaml:"host"`
Port int `json:"port" yaml:"port"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
Database string `json:"database" yaml:"database"`
}

View File

@@ -15,4 +15,6 @@ const (
SettingCodeBillDay SettingCode = "billDay" // 账单日 YYYYMMDD
SettingCodeBillMonth SettingCode = "billMonth" // 账单月 YYYYMM
SettingCodeClickHouseConfig SettingCode = "clickhouseConfig" // ClickHouse 连接配置(访问日志 logs_ingest 查询)
)

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "1.4.5.1" //1.3.8.2
Version = "1.4.6" //1.3.8.2
ProductName = "Edge DNS"
ProcessName = "edge-dns"

View File

@@ -0,0 +1,206 @@
package accesslogs
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
)
var (
sharedFileWriter *FileWriter
sharedOnce sync.Once
)
// SharedFileWriter 返回全局本地日志文件写入器(单例)
func SharedFileWriter() *FileWriter {
sharedOnce.Do(func() {
sharedFileWriter = NewFileWriter()
})
return sharedFileWriter
}
const (
defaultLogDir = "/var/log/edge/edge-node"
envLogDir = "EDGE_LOG_DIR"
)
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 logrotate 与 Fluent Bit 采集
type FileWriter struct {
dir string
mu sync.Mutex
files map[string]*os.File // access.log, waf.log, error.log
inited bool
}
// NewFileWriter 创建本地日志文件写入器
func NewFileWriter() *FileWriter {
dir := os.Getenv(envLogDir)
if dir == "" {
dir = defaultLogDir
}
return &FileWriter{
dir: dir,
files: make(map[string]*os.File),
}
}
// Dir 返回当前配置的日志目录
func (w *FileWriter) Dir() string {
return w.dir
}
// IsEnabled 是否启用落盘(目录非空即视为启用)
func (w *FileWriter) IsEnabled() bool {
return w.dir != ""
}
// EnsureInit 在启动时预创建日志目录和空文件,便于 Fluent Bit 立即 tail无需等首条访问日志
func (w *FileWriter) EnsureInit() error {
if w.dir == "" {
return nil
}
return w.init()
}
// init 确保目录存在并打开三个日志文件(仅首次或 Reopen 时)
func (w *FileWriter) init() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.inited && len(w.files) > 0 {
return nil
}
if w.dir == "" {
return nil
}
if err := os.MkdirAll(w.dir, 0755); err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error())
return err
}
for _, name := range []string{"access.log", "waf.log", "error.log"} {
if w.files[name] != nil {
continue
}
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
continue
}
w.files[name] = fp
}
w.inited = true
return nil
}
// Write 将一条访问日志按 log_type 写入对应文件access.log / waf.log / error.log
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
if w.dir == "" {
return
}
if err := w.init(); err != nil || len(w.files) == 0 {
return
}
ingest, logType := FromHTTPAccessLog(l, clusterId)
line, err := json.Marshal(ingest)
if err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "marshal ingest log: "+err.Error())
return
}
var fileName string
switch logType {
case LogTypeWAF:
fileName = "waf.log"
case LogTypeError:
fileName = "error.log"
default:
fileName = "access.log"
}
w.mu.Lock()
fp := w.files[fileName]
w.mu.Unlock()
if fp == nil {
return
}
// 单行写入,末尾换行,便于 Fluent Bit / JSON 解析
_, err = fp.Write(append(line, '\n'))
if err != nil {
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
}
}
// WriteBatch 批量写入,减少锁竞争
func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
if w.dir == "" || len(logs) == 0 {
return
}
if err := w.init(); err != nil || len(w.files) == 0 {
return
}
w.mu.Lock()
accessFp := w.files["access.log"]
wafFp := w.files["waf.log"]
errorFp := w.files["error.log"]
w.mu.Unlock()
if accessFp == nil && wafFp == nil && errorFp == nil {
return
}
for _, l := range logs {
ingest, logType := FromHTTPAccessLog(l, clusterId)
line, err := json.Marshal(ingest)
if err != nil {
continue
}
line = append(line, '\n')
var fp *os.File
switch logType {
case LogTypeWAF:
fp = wafFp
case LogTypeError:
fp = errorFp
default:
fp = accessFp
}
if fp != nil {
_, _ = fp.Write(line)
}
}
}
// Reopen 关闭并重新打开所有日志文件(供 logrotate copytruncate 或 SIGHUP 后重开句柄)
func (w *FileWriter) Reopen() error {
if w.dir == "" {
return nil
}
w.mu.Lock()
defer w.mu.Unlock()
for name, f := range w.files {
if f != nil {
_ = f.Close()
w.files[name] = nil
}
}
w.inited = false
return w.init()
}
// Close 关闭所有已打开的文件
func (w *FileWriter) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
var lastErr error
for name, f := range w.files {
if f != nil {
if err := f.Close(); err != nil {
lastErr = err
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
}
w.files[name] = nil
}
}
w.inited = false
return lastErr
}

View File

@@ -0,0 +1,137 @@
// Package accesslogs 提供边缘节点访问日志落盘JSON Lines供 Fluent Bit 采集写入 ClickHouse。
package accesslogs
import (
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// LogType 与 Fluent Bit / logs_ingest 的 log_type 一致
const (
LogTypeAccess = "access"
LogTypeWAF = "waf"
LogTypeError = "error"
)
// 请求/响应 body 落盘最大长度(字节),超出截断,避免单条过大
const maxBodyLen = 512 * 1024
// IngestLog 单行 JSON 结构与方案文档、ClickHouse logs_ingest 表字段对齐
type IngestLog struct {
Timestamp int64 `json:"timestamp"`
NodeId int64 `json:"node_id"`
ClusterId int64 `json:"cluster_id"`
ServerId int64 `json:"server_id"`
Host string `json:"host"`
IP string `json:"ip"`
Method string `json:"method"`
Path string `json:"path"`
Status int32 `json:"status"`
BytesIn int64 `json:"bytes_in"`
BytesOut int64 `json:"bytes_out"`
CostMs int64 `json:"cost_ms"`
UA string `json:"ua"`
Referer string `json:"referer"`
LogType string `json:"log_type"`
TraceId string `json:"trace_id,omitempty"`
FirewallPolicyId int64 `json:"firewall_policy_id,omitempty"`
FirewallRuleGroupId int64 `json:"firewall_rule_group_id,omitempty"`
FirewallRuleSetId int64 `json:"firewall_rule_set_id,omitempty"`
FirewallRuleId int64 `json:"firewall_rule_id,omitempty"`
RequestHeaders string `json:"request_headers,omitempty"`
RequestBody string `json:"request_body,omitempty"`
ResponseHeaders string `json:"response_headers,omitempty"`
ResponseBody string `json:"response_body,omitempty"`
}
// stringsMapToJSON 将 map[string]*Strings 转为 JSON 字符串,便于落盘与 ClickHouse 存储
func stringsMapToJSON(m map[string]*pb.Strings) string {
if len(m) == 0 {
return ""
}
out := make(map[string]string, len(m))
for k, v := range m {
if v != nil && len(v.Values) > 0 {
out[k] = v.Values[0]
}
}
if len(out) == 0 {
return ""
}
b, _ := json.Marshal(out)
return string(b)
}
// truncateBody 截断 body 到最大长度,避免单条过大
func truncateBody(b []byte) string {
if len(b) == 0 {
return ""
}
s := string(b)
if len(s) > maxBodyLen {
return s[:maxBodyLen]
}
return s
}
// buildRequestBody 将查询串与请求体合并写入 request_body 字段(不新增字段)
func buildRequestBody(l *pb.HTTPAccessLog) string {
q := l.GetQueryString()
body := l.GetRequestBody()
if len(q) == 0 && len(body) == 0 {
return ""
}
if len(body) == 0 {
return truncateBody([]byte(q))
}
combined := make([]byte, 0, len(q)+1+len(body))
combined = append(combined, q...)
combined = append(combined, '\n')
combined = append(combined, body...)
return truncateBody(combined)
}
// FromHTTPAccessLog 从 pb.HTTPAccessLog 转为 IngestLog并决定 log_type
func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog, logType string) {
ingest = IngestLog{
Timestamp: l.GetTimestamp(),
NodeId: l.GetNodeId(),
ClusterId: clusterId,
ServerId: l.GetServerId(),
Host: l.GetHost(),
IP: l.GetRawRemoteAddr(),
Method: l.GetRequestMethod(),
Path: l.GetRequestPath(),
Status: l.GetStatus(),
BytesIn: l.GetRequestLength(),
BytesOut: l.GetBytesSent(),
CostMs: int64(l.GetRequestTime() * 1000),
UA: l.GetUserAgent(),
Referer: l.GetReferer(),
TraceId: l.GetRequestId(),
FirewallPolicyId: l.GetFirewallPolicyId(),
FirewallRuleGroupId: l.GetFirewallRuleGroupId(),
FirewallRuleSetId: l.GetFirewallRuleSetId(),
FirewallRuleId: l.GetFirewallRuleId(),
RequestHeaders: stringsMapToJSON(l.GetHeader()),
RequestBody: buildRequestBody(l),
ResponseHeaders: stringsMapToJSON(l.GetSentHeader()),
}
if ingest.IP == "" {
ingest.IP = l.GetRemoteAddr()
}
// 响应 body 当前 pb 未提供,若后续扩展可在此赋值
// ingest.ResponseBody = ...
// 与方案一致waf > error > access攻击日志通过 firewall_rule_id / firewall_policy_id 判断
if l.GetFirewallPolicyId() > 0 {
logType = LogTypeWAF
} else if len(l.GetErrors()) > 0 {
logType = LogTypeError
} else {
logType = LogTypeAccess
}
ingest.LogType = logType
return ingest, logType
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "1.4.5.2" //1.3.8.2
Version = "1.4.6" //1.3.8.2
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -3,6 +3,8 @@ package nodes
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -92,6 +94,22 @@ Loop:
return nil
}
var writeTargets *serverconfigs.AccessLogWriteTargets
if sharedNodeConfig != nil && sharedNodeConfig.GlobalServerConfig != nil {
writeTargets = sharedNodeConfig.GlobalServerConfig.HTTPAccessLog.WriteTargets
}
needWriteFile := writeTargets == nil || writeTargets.NeedWriteFile()
needReportAPI := writeTargets == nil || writeTargets.NeedReportToAPI()
// 落盘 JSON LinesFluent Bit 采集 → ClickHouse
if needWriteFile {
var clusterId int64
if sharedNodeConfig != nil {
clusterId = sharedNodeConfig.GroupId
}
accesslogs.SharedFileWriter().WriteBatch(accessLogs, clusterId)
}
// 发送到本地
if sharedHTTPAccessLogViewer.HasConns() {
for _, accessLog := range accessLogs {
@@ -99,7 +117,10 @@ Loop:
}
}
// 发送到API
// 发送到 API(写 MySQL 或 API 直写 ClickHouse 时需要)
if !needReportAPI {
return nil
}
if this.rpcClient == nil {
client, err := rpc.SharedRPC()
if err != nil {

View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/configs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
@@ -422,6 +423,9 @@ func (this *Node) syncConfig(taskVersion int64) error {
this.isLoaded = true
// 预创建本地日志目录与空文件,便于 Fluent Bit 立即 tail无需等首条访问日志
_ = accesslogs.SharedFileWriter().EnsureInit()
// 整体更新不需要再更新单个服务
this.updatingServerMap = map[int64]*serverconfigs.ServerConfig{}
@@ -569,9 +573,16 @@ func (this *Node) checkClusterConfig() error {
// 监听一些信号
func (this *Node) listenSignals() {
var queue = make(chan os.Signal, 8)
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT)
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGHUP)
goman.New(func() {
for range queue {
for sig := range queue {
if sig == syscall.SIGHUP {
// 供 logrotate 等旋转日志后重开句柄
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
}
continue
}
time.Sleep(100 * time.Millisecond)
utils.Exit()
return

View File

@@ -290,14 +290,6 @@ var AllCheckpoints = []*CheckpointDefinition{
Instance: new(RequestISPNameCheckpoint),
Priority: 90,
},
{
Name: "CC统计",
Prefix: "cc",
Description: "统计某段时间段内的请求信息",
HasParams: true,
Instance: new(CCCheckpoint),
Priority: 10,
},
{
Name: "CC统计",
Prefix: "cc2",

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "1.4.5" //1.3.8.2
Version = "1.4.6" //1.3.8.2
ProductName = "Edge User"
ProcessName = "edge-user"

BIN
default.etcd/.DS_Store vendored Normal file

Binary file not shown.

BIN
default.etcd/member/.DS_Store vendored Normal file

Binary file not shown.

BIN
default.etcd/member/snap/db Normal file

Binary file not shown.

Binary file not shown.

308
deploy/fluent-bit/README.md Normal file
View File

@@ -0,0 +1,308 @@
# 边缘节点日志链路部署Fluent Bit + ClickHouse
与 [日志链路调整方案](../../log-pipeline-migration-plan.md) 配套的配置与部署说明。本文档为 **Fluent Bit 部署教程**,按步骤即可在边缘节点或日志采集机上跑通采集 → ClickHouse 写入。
---
## Fluent Bit 跑在哪台机器上?
**Fluent Bit 应部署在每台 EdgeNode 机器上**(与 edge-node 同机),不要部署在 EdgeAPI 机器上。
- 日志文件(`/var/log/edge/edge-node/*.log`)是 **EdgeNode** 在本机写的,只有 EdgeNode 所在机器才有这些文件。
- Fluent Bit 使用 **tail** 插件读取本机路径,因此必须运行在 **有这些日志文件的机器** 上,即每台 EdgeNode。
- EdgeAPI 机器上没有边缘节点日志,只负责查询 ClickHouse/MySQL因此不需要在 EdgeAPI 上跑 Fluent Bit。
- **多台 EdgeNode 时**:每台 EdgeNode 各跑一份 Fluent Bit各自采集本机日志并上报到同一 ClickHouse。
---
## 一、前置条件
- **边缘节点EdgeNode** 已开启本地日志落盘,日志目录默认 `/var/log/edge/edge-node`,会生成 `access.log``waf.log``error.log`JSON Lines。由环境变量 `EDGE_LOG_DIR` 控制路径。
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest`见下文「五、ClickHouse 建表」)。
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123
---
## 二、安装 Fluent Bit
### 2.1 Ubuntu / Debian
```bash
# 添加 Fluent Bit 官方源并安装(以 Ubuntu 22.04 为例)
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
sudo apt-get install -y fluent-bit
# 或使用 TD Agent Bit 源(若需 ClickHouse 等扩展)
# 见https://docs.fluentbit.io/manual/installation/linux/ubuntu
```
### 2.2 CentOS / RHEL / Amazon Linux
```bash
# 使用官方 install 脚本
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
# 或 yum/dnf 安装(以提供的仓库为准)
# sudo yum install -y fluent-bit
```
### 2.3 使用二进制包
从 [Fluent Bit 官方 Release](https://github.com/fluent/fluent-bit/releases) 下载对应架构的 tarball解压后将 `bin/fluent-bit` 放到 PATH并确保其 **Output 插件支持 ClickHouse**(部分发行版或自编译需启用 `out_clickhouse`)。
---
## 三、部署配置文件
### 3.1 放置配置
将本目录下配置文件放到同一目录,例如 `/etc/fluent-bit/``/opt/edge/fluent-bit/`
```bash
sudo mkdir -p /etc/fluent-bit
sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
```
两文件需在同一目录,因 `fluent-bit.conf` 中有 `@INCLUDE clickhouse-upstream.conf`
### 3.2 修改 ClickHouse 地址(必做)
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port
- **单机**:保留一个 `[NODE]`,改 `Host``Port`(默认 8123
- **集群**:复制多段 `[NODE]`,每段一个节点,例如:
```ini
[UPSTREAM]
Name ch_backends
[NODE]
Name node-01
Host 192.168.1.10
Port 8123
[NODE]
Name node-02
Host 192.168.1.11
Port 8123
```
### 3.3 ClickHouse 账号密码(有密码时必做)
不在 `clickhouse-upstream.conf` 里配置密码,而是通过 **环境变量** 传给 Fluent Bit
- `CH_USER`ClickHouse 用户名(如 `default`)。
- `CH_PASSWORD`:对应用户的密码。
在 systemd 或启动脚本中设置(见下文「四、以 systemd 方式运行」)。
### 3.4 日志路径与 parsers.conf
- **日志路径**`fluent-bit.conf` 里 INPUT 的 `Path` 已为 `/var/log/edge/edge-node/*.log`,与 EdgeNode 默认落盘路径一致;若你改了 `EDGE_LOG_DIR`,请同步改此处的 `Path`
- **Parsers_File**:主配置引用了 `parsers.conf`。若安装包自带(如 `/etc/fluent-bit/parsers.conf`),无需改动;若启动报错找不到文件,可:
- 从 Fluent Bit 官方仓库复制 [conf/parsers.conf](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf) 到同一目录,或
- 在同一目录新建空文件 `parsers.conf`(仅当不使用任何 parser 时)。
### 3.5 数据与状态目录
Fluent Bit 会使用配置里的 `storage.path` 和 DB 路径,需保证进程有写权限:
```bash
sudo mkdir -p /var/lib/fluent-bit/storage
sudo chown -R <运行 fluent-bit 的用户>:<同组> /var/lib/fluent-bit
```
---
## 四、以 systemd 方式运行
### 4.1 使用自带服务(若安装包已提供)
若通过 apt/yum 安装,通常已有 `fluent-bit.service`。先改配置路径和环境变量:
```bash
# 编辑服务文件(路径以实际为准,如 /lib/systemd/system/fluent-bit.service
sudo systemctl edit fluent-bit --full
```
`[Service]` 中增加或修改:
- `EnvironmentFile` 指向你的环境变量文件,或直接写:
- `Environment="CH_USER=default"`
- `Environment="CH_PASSWORD=你的密码"`
- `ExecStart` 中的配置文件路径改为你的 `fluent-bit.conf`,例如:
- `ExecStart=/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf`
然后:
```bash
sudo systemctl daemon-reload
sudo systemctl enable fluent-bit
sudo systemctl start fluent-bit
sudo systemctl status fluent-bit
```
### 4.2 自定义 systemd 单元(无自带服务时)
新建 `/etc/systemd/system/fluent-bit-edge.service`
```ini
[Unit]
Description=Fluent Bit - Edge Node Logs to ClickHouse
After=network.target
[Service]
Type=simple
ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf
Restart=always
RestartSec=5
# ClickHouse 认证(按需修改)
Environment="CH_USER=default"
Environment="CH_PASSWORD=your_clickhouse_password"
[Install]
WantedBy=multi-user.target
```
若密码含特殊字符,建议用 `EnvironmentFile=/etc/fluent-bit/fluent-bit.env`,并在该文件中写:
```bash
CH_USER=default
CH_PASSWORD=your_clickhouse_password
```
然后:
```bash
sudo systemctl daemon-reload
sudo systemctl enable fluent-bit-edge
sudo systemctl start fluent-bit-edge
sudo systemctl status fluent-bit-edge
```
### 4.3 前台调试
不依赖 systemd 时可直接前台跑(便于看日志):
```bash
export CH_USER=default
export CH_PASSWORD=your_clickhouse_password
fluent-bit -c /etc/fluent-bit/fluent-bit.conf
```
---
## 五、ClickHouse 建表
平台EdgeAPI查询的是表 `logs_ingest`,需在 ClickHouse 中先建表。库名默认为 `default`,若使用其它库,需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。
在 ClickHouse 中执行(按需改库名或引擎):
```sql
CREATE TABLE IF NOT EXISTS default.logs_ingest
(
timestamp DateTime,
node_id UInt64,
cluster_id UInt64,
server_id UInt64,
host String,
ip String,
method String,
path String,
status UInt16,
bytes_in UInt64,
bytes_out UInt64,
cost_ms UInt32,
ua String,
referer String,
log_type String,
trace_id String,
firewall_policy_id UInt64 DEFAULT 0,
firewall_rule_group_id UInt64 DEFAULT 0,
firewall_rule_set_id UInt64 DEFAULT 0,
firewall_rule_id UInt64 DEFAULT 0,
request_headers String DEFAULT '',
request_body String DEFAULT '',
response_headers String DEFAULT '',
response_body String DEFAULT ''
)
ENGINE = MergeTree()
ORDER BY (timestamp, node_id, server_id, trace_id)
SETTINGS index_granularity = 8192;
```
- **log_type**`access` / `waf` / `error`;攻击日志同时看 **firewall_rule_id****firewall_policy_id** 是否大于 0与原有 MySQL 通过规则 ID 判断攻击日志一致)。
- **request_headers / response_headers**JSON 字符串;**request_body / response_body**:请求/响应体(单条建议限制长度,如 512KB
- **request_body 为空**:需在管理端为该站点/服务的「访问日志」策略中勾选「请求Body」后才会落盘默认未勾选。路径大致为站点/服务 → 访问日志 → 策略 → 记录字段 → 勾选「请求Body」。WAF 拦截且策略开启「记录请求Body」时也会记录。
- **response_body 为空**当前版本未实现proto 与节点均未支持响应体落盘),表中已预留字段,后续可扩展。
- **原有 MySQL 日志同步到 ClickHouse**:见 [mysql-to-clickhouse-migration.md](mysql-to-clickhouse-migration.md)。
若表已存在且缺少新字段,可执行:
```sql
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_policy_id UInt64 DEFAULT 0;
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_group_id UInt64 DEFAULT 0;
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_set_id UInt64 DEFAULT 0;
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_id UInt64 DEFAULT 0;
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_headers String DEFAULT '';
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_body String DEFAULT '';
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_headers String DEFAULT '';
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_body String DEFAULT '';
```
Fluent Bit 写入时使用 `json_date_key timestamp``json_date_format epoch`,会将 JSON 中的 `timestamp`Unix 秒)转为 DateTime。
---
## 六、验证与排错
1. **看 Fluent Bit 日志**
- systemd`journalctl -u fluent-bit-edge -f`(或你的服务名)
- 前台:直接看终端输出。
2. **看 ClickHouse 是否有数据**
```sql
SELECT count() FROM default.logs_ingest;
SELECT * FROM default.logs_ingest LIMIT 5;
```
3. **常见问题**
- **连接被拒**:检查 `clickhouse-upstream.conf` 的 Host/Port、防火墙、ClickHouse 的 `listen_host`。
- **认证失败**:检查 `CH_USER`、`CH_PASSWORD` 是否与 ClickHouse 用户一致,环境变量是否被 systemd 正确加载。
- **找不到 parsers.conf**:见上文 3.4。
- **没有新数据**:确认 EdgeNode 已写日志到 `Path` 下,且 Fluent Bit 对该目录有读权限;可用 `tail -f /var/log/edge/edge-node/access.log` 观察是否有新行。
- **Node 上没有 `/var/log/edge/edge-node/access.log`**见下文「八、Node 上找不到日志文件」。
---
## 七、与其它组件的关系(简要)
| 组件 | 说明 |
|------|------|
| **EdgeNode** | 日志落盘路径由 `EDGE_LOG_DIR` 控制,默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 |
| **平台EdgeAPI** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
---
## 八、Node 上找不到日志文件
若在 EdgeNode 机器上执行 `tail -f /var/log/edge/edge-node/access.log` 报 **No such file or directory**,按下面检查:
1. **EdgeNode 版本**
本地日志落盘是较新功能,需使用**包含该功能的 EdgeNode 构建**(当前仓库版本在首次加载配置时会预创建目录和三个空日志文件)。
2. **预创建目录(可选)**
若进程以非 root 运行,可先手动建目录并赋权,避免无权限创建 `/var/log/edge`
```bash
sudo mkdir -p /var/log/edge/edge-node
sudo chown <运行 edge-node 的用户>:<同组> /var/log/edge/edge-node
```
3. **重启 EdgeNode**
新版本在**首次成功加载节点配置后**会调用 `EnsureInit()`,自动创建 `/var/log/edge/edge-node` 及 `access.log`、`waf.log`、`error.log`。重启一次 edge-node 后再看目录下是否已有文件。
4. **自定义路径**
若通过环境变量 `EDGE_LOG_DIR` 指定了其它目录则日志在该目录下Fluent Bit 的 `Path` 需与之一致。
以上完成即完成 Fluent Bit 的部署与验证。

View File

@@ -0,0 +1,11 @@
# ClickHouse 上游配置(单机或集群只改此文件)
# 单机:保留一个 NODE集群按需增加 NODE
# 有密码时:不在此文件配置,通过环境变量 CH_USER、CH_PASSWORD 在 fluent-bit.conf 的 OUTPUT 中生效。
[UPSTREAM]
Name ch_backends
[NODE]
Name node-01
Host 127.0.0.1
Port 8123

View File

@@ -0,0 +1,34 @@
# Fluent Bit 主配置(边缘节点日志采集 → ClickHouse
# 生产环境将 INPUT 改为 tail 采集 /var/log/edge/edge-node/*.log
[SERVICE]
Flush 5
Log_Level info
Parsers_File parsers.conf
storage.path /var/lib/fluent-bit/storage
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 128MB
@INCLUDE clickhouse-upstream.conf
[INPUT]
Name tail
Path /var/log/edge/edge-node/*.log
Tag app.logs
Refresh_Interval 5
Read_from_Head false
DB /var/lib/fluent-bit/logs.db
Mem_Buf_Limit 128MB
Skip_Long_Lines On
[OUTPUT]
Name clickhouse
Match *
Upstream ch_backends
Table logs_ingest
Http_User ${CH_USER}
Http_Passwd ${CH_PASSWORD}
json_date_key timestamp
json_date_format epoch
Retry_Limit 10

View File

@@ -0,0 +1,11 @@
# logrotate 示例:边缘节点日志轮转
# 安装:放入 /etc/logrotate.d/edge-node 或 include 到主配置
/var/log/edge/edge-node/*.log {
daily
rotate 14
compress
missingok
notifempty
copytruncate
}

View File

@@ -0,0 +1,9 @@
# Fluent Bit 解析器(与主配置 Parsers_File 对应)
# 边缘节点日志为 JSON Lines使用 json 解析器即可
[PARSER]
Name json
Format json
Time_Key timestamp
Time_Format %s
Time_Keep On