主分支代码
This commit is contained in:
@@ -24,6 +24,7 @@ type StorageManager struct {
|
||||
|
||||
publicPolicyId int64
|
||||
disableDefaultDB bool
|
||||
writeTargets *serverconfigs.AccessLogWriteTargets // 公用策略的写入目标
|
||||
|
||||
locker sync.Mutex
|
||||
}
|
||||
@@ -79,12 +80,14 @@ func (this *StorageManager) Loop() error {
|
||||
|
||||
if int64(policy.Id) == publicPolicyId {
|
||||
this.disableDefaultDB = policy.DisableDefaultDB
|
||||
this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB)
|
||||
foundPolicy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPolicy {
|
||||
this.disableDefaultDB = false
|
||||
this.writeTargets = nil
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
@@ -160,6 +163,27 @@ func (this *StorageManager) DisableDefaultDB() bool {
|
||||
return this.disableDefaultDB
|
||||
}
|
||||
|
||||
// WriteMySQL 公用策略是否写入 MySQL(以 writeTargets 为准,无则用 disableDefaultDB)
|
||||
func (this *StorageManager) WriteMySQL() bool {
|
||||
if this.writeTargets != nil {
|
||||
return this.writeTargets.MySQL
|
||||
}
|
||||
return !this.disableDefaultDB
|
||||
}
|
||||
|
||||
// WriteClickHouse 公用策略是否写入 ClickHouse(文件+Fluent Bit 或后续 API 直写)
|
||||
func (this *StorageManager) WriteClickHouse() bool {
|
||||
if this.writeTargets != nil {
|
||||
return this.writeTargets.ClickHouse
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// WriteTargets 返回公用策略的写入目标(供节点配置注入等)
|
||||
func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets {
|
||||
return this.writeTargets
|
||||
}
|
||||
|
||||
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
|
||||
switch storageType {
|
||||
case serverconfigs.AccessLogStorageTypeFile:
|
||||
|
||||
134
EdgeAPI/internal/clickhouse/client.go
Normal file
134
EdgeAPI/internal/clickhouse/client.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client 通过 HTTP 接口执行只读查询(SELECT),返回 JSONEachRow 解析为 map 或结构体
|
||||
type Client struct {
|
||||
cfg *Config
|
||||
httpCli *http.Client
|
||||
}
|
||||
|
||||
// NewClient 使用共享配置创建客户端
|
||||
func NewClient() *Client {
|
||||
cfg := SharedConfig()
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
httpCli: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// IsConfigured 是否已配置
|
||||
func (c *Client) IsConfigured() bool {
|
||||
return c.cfg != nil && c.cfg.IsConfigured()
|
||||
}
|
||||
|
||||
// Query 执行 SELECT,将每行 JSON 解析到 dest 切片;dest 元素类型需为 *struct 或 map
|
||||
func (c *Client) Query(ctx context.Context, query string, dest interface{}) error {
|
||||
if !c.IsConfigured() {
|
||||
return fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
// 强制 JSONEachRow 便于解析
|
||||
q := strings.TrimSpace(query)
|
||||
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
|
||||
query = q + " FORMAT JSONEachRow"
|
||||
}
|
||||
u := c.buildURL(query)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.cfg.User != "" || c.cfg.Password != "" {
|
||||
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
|
||||
}
|
||||
resp, err := c.httpCli.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
return decodeRows(dec, dest)
|
||||
}
|
||||
|
||||
// QueryRow 执行仅返回一行的查询,将结果解析到 dest(*struct 或 *map)
|
||||
func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) error {
|
||||
if !c.IsConfigured() {
|
||||
return fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
q := strings.TrimSpace(query)
|
||||
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
|
||||
query = q + " FORMAT JSONEachRow"
|
||||
}
|
||||
u := c.buildURL(query)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.cfg.User != "" || c.cfg.Password != "" {
|
||||
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
|
||||
}
|
||||
resp, err := c.httpCli.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
return decodeOneRow(dec, dest)
|
||||
}
|
||||
|
||||
func (c *Client) buildURL(query string) string {
|
||||
rawURL := fmt.Sprintf("http://%s:%d/?query=%s&database=%s",
|
||||
c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
|
||||
return rawURL
|
||||
}
|
||||
|
||||
// decodeRows 将 JSONEachRow 流解析到 slice;元素类型须为 *struct 或 *map[string]interface{}
|
||||
func decodeRows(dec *json.Decoder, dest interface{}) error {
|
||||
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
|
||||
switch d := dest.(type) {
|
||||
case *[]map[string]interface{}:
|
||||
*d = (*d)[:0]
|
||||
for {
|
||||
var row map[string]interface{}
|
||||
if err := dec.Decode(&row); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
*d = append(*d, row)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{} or implement decoder)")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeOneRow(dec *json.Decoder, dest interface{}) error {
|
||||
switch d := dest.(type) {
|
||||
case *map[string]interface{}:
|
||||
if err := dec.Decode(d); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("clickhouse: unsupported dest type for QueryRow (use *map[string]interface{})")
|
||||
}
|
||||
}
|
||||
111
EdgeAPI/internal/clickhouse/config.go
Normal file
111
EdgeAPI/internal/clickhouse/config.go
Normal file
@@ -0,0 +1,111 @@
|
||||
// Package clickhouse 提供 ClickHouse 只读客户端,用于查询 logs_ingest(Fluent Bit 写入)。
|
||||
// 配置优先从后台页面(edgeSysSettings.clickhouseConfig)读取,其次 api.yaml,最后环境变量。
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
envHost = "CLICKHOUSE_HOST"
|
||||
envPort = "CLICKHOUSE_PORT"
|
||||
envUser = "CLICKHOUSE_USER"
|
||||
envPassword = "CLICKHOUSE_PASSWORD"
|
||||
envDatabase = "CLICKHOUSE_DATABASE"
|
||||
defaultPort = 8123
|
||||
defaultDB = "default"
|
||||
)
|
||||
|
||||
var (
|
||||
sharedConfig *Config
|
||||
configOnce sync.Once
|
||||
configLocker sync.Mutex
|
||||
)
|
||||
|
||||
// Config ClickHouse 连接配置(仅查询,不从代码写库)
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
}
|
||||
|
||||
// SharedConfig 返回全局配置(优先从后台 DB 读取,其次 api.yaml,最后环境变量)
|
||||
func SharedConfig() *Config {
|
||||
configLocker.Lock()
|
||||
defer configLocker.Unlock()
|
||||
if sharedConfig != nil {
|
||||
return sharedConfig
|
||||
}
|
||||
sharedConfig = loadConfig()
|
||||
return sharedConfig
|
||||
}
|
||||
|
||||
// ResetSharedConfig 清空缓存,下次 SharedConfig() 时重新从 DB/文件/环境变量加载(后台保存 ClickHouse 配置后调用)
|
||||
func ResetSharedConfig() {
|
||||
configLocker.Lock()
|
||||
defer configLocker.Unlock()
|
||||
sharedConfig = nil
|
||||
}
|
||||
|
||||
func loadConfig() *Config {
|
||||
cfg := &Config{Port: defaultPort, Database: defaultDB}
|
||||
// 1) 优先从后台页面配置(DB)读取
|
||||
if models.SharedSysSettingDAO != nil {
|
||||
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
|
||||
cfg.Host = dbCfg.Host
|
||||
cfg.Port = dbCfg.Port
|
||||
cfg.User = dbCfg.User
|
||||
cfg.Password = dbCfg.Password
|
||||
cfg.Database = dbCfg.Database
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
}
|
||||
// 2) 其次 api.yaml
|
||||
apiConfig, err := configs.SharedAPIConfig()
|
||||
if err == nil && apiConfig != nil && apiConfig.ClickHouse != nil && apiConfig.ClickHouse.Host != "" {
|
||||
ch := apiConfig.ClickHouse
|
||||
cfg.Host = ch.Host
|
||||
cfg.Port = ch.Port
|
||||
cfg.User = ch.User
|
||||
cfg.Password = ch.Password
|
||||
cfg.Database = ch.Database
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
// 3) 最后环境变量
|
||||
cfg.Host = os.Getenv(envHost)
|
||||
cfg.User = os.Getenv(envUser)
|
||||
cfg.Password = os.Getenv(envPassword)
|
||||
cfg.Database = os.Getenv(envDatabase)
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
if p := os.Getenv(envPort); p != "" {
|
||||
if v, err := strconv.Atoi(p); err == nil {
|
||||
cfg.Port = v
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// IsConfigured 是否已配置(Host 非空即视为启用 ClickHouse 查询)
|
||||
func (c *Config) IsConfigured() bool {
|
||||
return c != nil && c.Host != ""
|
||||
}
|
||||
285
EdgeAPI/internal/clickhouse/logs_ingest_store.go
Normal file
285
EdgeAPI/internal/clickhouse/logs_ingest_store.go
Normal file
@@ -0,0 +1,285 @@
|
||||
// Package clickhouse 提供 logs_ingest 表的只读查询(列表分页),用于访问日志列表优先走 ClickHouse。
|
||||
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
|
||||
// LogsIngestRow 对应 ClickHouse logs_ingest 表的一行(用于 List 结果与 RowToPB)
|
||||
type LogsIngestRow struct {
|
||||
Timestamp time.Time
|
||||
NodeId uint64
|
||||
ClusterId uint64
|
||||
ServerId uint64
|
||||
Host string
|
||||
IP string
|
||||
Method string
|
||||
Path string
|
||||
Status uint16
|
||||
BytesIn uint64
|
||||
BytesOut uint64
|
||||
CostMs uint32
|
||||
UA string
|
||||
Referer string
|
||||
LogType string
|
||||
TraceId string
|
||||
FirewallPolicyId uint64
|
||||
FirewallRuleGroupId uint64
|
||||
FirewallRuleSetId uint64
|
||||
FirewallRuleId uint64
|
||||
RequestHeaders string
|
||||
RequestBody string
|
||||
ResponseHeaders string
|
||||
ResponseBody string
|
||||
}
|
||||
|
||||
// ListFilter 列表查询条件(与 ListHTTPAccessLogsRequest 对齐)
|
||||
type ListFilter struct {
|
||||
Day string
|
||||
HourFrom string
|
||||
HourTo string
|
||||
Size int64
|
||||
Reverse bool
|
||||
HasError bool
|
||||
HasFirewallPolicy bool
|
||||
FirewallPolicyId int64
|
||||
NodeId int64
|
||||
ClusterId int64
|
||||
LastRequestId string
|
||||
ServerIds []int64
|
||||
NodeIds []int64
|
||||
}
|
||||
|
||||
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
|
||||
type LogsIngestStore struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewLogsIngestStore 创建 store,内部使用共享 Client
|
||||
func NewLogsIngestStore() *LogsIngestStore {
|
||||
return &LogsIngestStore{client: NewClient()}
|
||||
}
|
||||
|
||||
// Client 返回底层 Client,供调用方判断 IsConfigured()
|
||||
func (s *LogsIngestStore) Client() *Client {
|
||||
return s.client
|
||||
}
|
||||
|
||||
// List 按条件分页查询 logs_ingest,返回行、下一页游标(trace_id)与错误
|
||||
func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsIngestRow, nextCursor string, err error) {
|
||||
if !s.client.IsConfigured() {
|
||||
return nil, "", fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
if f.Day == "" {
|
||||
return nil, "", fmt.Errorf("clickhouse: day required")
|
||||
}
|
||||
table := "logs_ingest"
|
||||
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
|
||||
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest")
|
||||
} else {
|
||||
table = quoteIdent(table)
|
||||
}
|
||||
|
||||
conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"}
|
||||
if f.HourFrom != "" {
|
||||
if _, err := strconv.Atoi(f.HourFrom); err == nil {
|
||||
conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom)
|
||||
}
|
||||
}
|
||||
if f.HourTo != "" {
|
||||
if _, err := strconv.Atoi(f.HourTo); err == nil {
|
||||
conditions = append(conditions, "toHour(timestamp) <= "+f.HourTo)
|
||||
}
|
||||
}
|
||||
if len(f.ServerIds) > 0 {
|
||||
parts := make([]string, 0, len(f.ServerIds))
|
||||
for _, id := range f.ServerIds {
|
||||
parts = append(parts, strconv.FormatInt(id, 10))
|
||||
}
|
||||
conditions = append(conditions, "server_id IN ("+strings.Join(parts, ",")+")")
|
||||
}
|
||||
if len(f.NodeIds) > 0 {
|
||||
parts := make([]string, 0, len(f.NodeIds))
|
||||
for _, id := range f.NodeIds {
|
||||
parts = append(parts, strconv.FormatInt(id, 10))
|
||||
}
|
||||
conditions = append(conditions, "node_id IN ("+strings.Join(parts, ",")+")")
|
||||
}
|
||||
if f.NodeId > 0 {
|
||||
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NodeId, 10))
|
||||
}
|
||||
if f.ClusterId > 0 {
|
||||
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.ClusterId, 10))
|
||||
}
|
||||
if f.HasFirewallPolicy {
|
||||
conditions = append(conditions, "firewall_policy_id > 0")
|
||||
}
|
||||
if f.FirewallPolicyId > 0 {
|
||||
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
|
||||
}
|
||||
|
||||
where := strings.Join(conditions, " AND ")
|
||||
orderDir := "ASC"
|
||||
if f.Reverse {
|
||||
orderDir = "DESC"
|
||||
}
|
||||
limit := f.Size
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
|
||||
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||
table, where, orderBy, limit+1)
|
||||
|
||||
var rawRows []map[string]interface{}
|
||||
if err = s.client.Query(ctx, query, &rawRows); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
rows = make([]*LogsIngestRow, 0, len(rawRows))
|
||||
for _, m := range rawRows {
|
||||
r := mapToLogsIngestRow(m)
|
||||
if r != nil {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
if len(rows) > int(limit) {
|
||||
nextCursor = rows[limit].TraceId
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return rows, nextCursor, nil
|
||||
}
|
||||
|
||||
func quoteIdent(name string) string {
|
||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||
}
|
||||
|
||||
func escapeString(s string) string {
|
||||
return strings.ReplaceAll(s, "'", "''")
|
||||
}
|
||||
|
||||
func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
|
||||
r := &LogsIngestRow{}
|
||||
u64 := func(key string) uint64 {
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return uint64(x)
|
||||
case string:
|
||||
n, _ := strconv.ParseUint(x, 10, 64)
|
||||
return n
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return uint64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
u32 := func(key string) uint32 {
|
||||
return uint32(u64(key))
|
||||
}
|
||||
str := func(key string) string {
|
||||
if v, ok := m[key]; ok && v != nil {
|
||||
if s, ok := v.(string); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
ts := func(key string) time.Time {
|
||||
v, ok := m[key]
|
||||
if ok && v != nil {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
t, _ := time.Parse("2006-01-02 15:04:05", x)
|
||||
return t
|
||||
case float64:
|
||||
return time.Unix(int64(x), 0)
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
r.Timestamp = ts("timestamp")
|
||||
r.NodeId = u64("node_id")
|
||||
r.ClusterId = u64("cluster_id")
|
||||
r.ServerId = u64("server_id")
|
||||
r.Host = str("host")
|
||||
r.IP = str("ip")
|
||||
r.Method = str("method")
|
||||
r.Path = str("path")
|
||||
r.Status = uint16(u64("status"))
|
||||
r.BytesIn = u64("bytes_in")
|
||||
r.BytesOut = u64("bytes_out")
|
||||
r.CostMs = u32("cost_ms")
|
||||
r.UA = str("ua")
|
||||
r.Referer = str("referer")
|
||||
r.LogType = str("log_type")
|
||||
r.TraceId = str("trace_id")
|
||||
r.FirewallPolicyId = u64("firewall_policy_id")
|
||||
r.FirewallRuleGroupId = u64("firewall_rule_group_id")
|
||||
r.FirewallRuleSetId = u64("firewall_rule_set_id")
|
||||
r.FirewallRuleId = u64("firewall_rule_id")
|
||||
r.RequestHeaders = str("request_headers")
|
||||
r.RequestBody = str("request_body")
|
||||
r.ResponseHeaders = str("response_headers")
|
||||
r.ResponseBody = str("response_body")
|
||||
return r
|
||||
}
|
||||
|
||||
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用)
|
||||
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
a := &pb.HTTPAccessLog{
|
||||
RequestId: r.TraceId,
|
||||
ServerId: int64(r.ServerId),
|
||||
NodeId: int64(r.NodeId),
|
||||
Timestamp: r.Timestamp.Unix(),
|
||||
Host: r.Host,
|
||||
RawRemoteAddr: r.IP,
|
||||
RemoteAddr: r.IP,
|
||||
RequestMethod: r.Method,
|
||||
RequestPath: r.Path,
|
||||
Status: int32(r.Status),
|
||||
RequestLength: int64(r.BytesIn),
|
||||
BytesSent: int64(r.BytesOut),
|
||||
RequestTime: float64(r.CostMs) / 1000,
|
||||
UserAgent: r.UA,
|
||||
Referer: r.Referer,
|
||||
FirewallPolicyId: int64(r.FirewallPolicyId),
|
||||
FirewallRuleGroupId: int64(r.FirewallRuleGroupId),
|
||||
FirewallRuleSetId: int64(r.FirewallRuleSetId),
|
||||
FirewallRuleId: int64(r.FirewallRuleId),
|
||||
}
|
||||
if r.TimeISO8601() != "" {
|
||||
a.TimeISO8601 = r.TimeISO8601()
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// TimeISO8601 便于 RowToPB 使用
|
||||
func (r *LogsIngestRow) TimeISO8601() string {
|
||||
if r.Timestamp.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return r.Timestamp.UTC().Format("2006-01-02T15:04:05Z07:00")
|
||||
}
|
||||
@@ -10,10 +10,20 @@ import (
|
||||
|
||||
var sharedAPIConfig *APIConfig = nil
|
||||
|
||||
// ClickHouseConfig 仅用于访问日志列表只读查询(logs_ingest)
|
||||
type ClickHouseConfig struct {
|
||||
Host string `yaml:"host" json:"host"`
|
||||
Port int `yaml:"port" json:"port"`
|
||||
User string `yaml:"user" json:"user"`
|
||||
Password string `yaml:"password" json:"password"`
|
||||
Database string `yaml:"database" json:"database"`
|
||||
}
|
||||
|
||||
// APIConfig API节点配置
|
||||
type APIConfig struct {
|
||||
NodeId string `yaml:"nodeId" json:"nodeId"`
|
||||
Secret string `yaml:"secret" json:"secret"`
|
||||
ClickHouse *ClickHouseConfig `yaml:"clickhouse,omitempty" json:"clickhouse,omitempty"`
|
||||
|
||||
numberId int64 // 数字ID
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5" //1.3.9
|
||||
Version = "1.4.6" //1.3.9
|
||||
|
||||
ProductName = "Edge API"
|
||||
ProcessName = "edge-api"
|
||||
@@ -17,5 +17,5 @@ const (
|
||||
|
||||
// 其他节点版本号,用来检测是否有需要升级的节点
|
||||
|
||||
NodeVersion = "1.4.5" //1.3.8.2
|
||||
NodeVersion = "1.4.6" //1.3.8.2
|
||||
)
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
DNSNodeVersion = "1.4.5" //1.3.8.2
|
||||
UserNodeVersion = "1.4.5" //1.3.8.2
|
||||
DNSNodeVersion = "1.4.6" //1.3.8.2
|
||||
UserNodeVersion = "1.4.6" //1.3.8.2
|
||||
ReportNodeVersion = "0.1.5"
|
||||
|
||||
DefaultMaxNodes int32 = 50
|
||||
|
||||
@@ -107,7 +107,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
|
||||
}
|
||||
|
||||
// CreatePolicy 创建策略
|
||||
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool) (policyId int64, err error) {
|
||||
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) {
|
||||
var op = NewHTTPAccessLogPolicyOperator()
|
||||
op.Name = name
|
||||
op.Type = policyType
|
||||
@@ -121,12 +121,15 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
|
||||
op.IsOn = true
|
||||
op.FirewallOnly = firewallOnly
|
||||
op.DisableDefaultDB = disableDefaultDB
|
||||
if len(writeTargetsJSON) > 0 {
|
||||
op.WriteTargets = writeTargetsJSON
|
||||
}
|
||||
op.State = HTTPAccessLogPolicyStateEnabled
|
||||
return this.SaveInt64(tx, op)
|
||||
}
|
||||
|
||||
// UpdatePolicy 修改策略
|
||||
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, isOn bool) error {
|
||||
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error {
|
||||
if policyId <= 0 {
|
||||
return errors.New("invalid policyId")
|
||||
}
|
||||
@@ -144,6 +147,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
|
||||
var op = NewHTTPAccessLogPolicyOperator()
|
||||
op.Id = policyId
|
||||
op.Name = name
|
||||
if policyType != "" {
|
||||
op.Type = policyType
|
||||
}
|
||||
if len(optionsJSON) > 0 {
|
||||
op.Options = optionsJSON
|
||||
} else {
|
||||
@@ -161,6 +167,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
|
||||
op.IsPublic = isPublic
|
||||
op.FirewallOnly = firewallOnly
|
||||
op.DisableDefaultDB = disableDefaultDB
|
||||
if len(writeTargetsJSON) > 0 {
|
||||
op.WriteTargets = writeTargetsJSON
|
||||
}
|
||||
op.IsOn = isOn
|
||||
return this.Save(tx, op)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
HTTPAccessLogPolicyField_FirewallOnly dbs.FieldName = "firewallOnly" // 是否只记录防火墙相关
|
||||
HTTPAccessLogPolicyField_Version dbs.FieldName = "version" // 版本号
|
||||
HTTPAccessLogPolicyField_DisableDefaultDB dbs.FieldName = "disableDefaultDB" // 是否停止默认数据库存储
|
||||
HTTPAccessLogPolicyField_WriteTargets dbs.FieldName = "writeTargets" // 写入目标 JSON:file/mysql/clickhouse
|
||||
)
|
||||
|
||||
// HTTPAccessLogPolicy 访问日志策略
|
||||
@@ -37,6 +38,7 @@ type HTTPAccessLogPolicy struct {
|
||||
FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关
|
||||
Version uint32 `field:"version"` // 版本号
|
||||
DisableDefaultDB bool `field:"disableDefaultDB"` // 是否停止默认数据库存储
|
||||
WriteTargets dbs.JSON `field:"writeTargets"` // 写入目标 JSON:{"file":true,"mysql":true,"clickhouse":false}
|
||||
}
|
||||
|
||||
type HTTPAccessLogPolicyOperator struct {
|
||||
@@ -55,6 +57,7 @@ type HTTPAccessLogPolicyOperator struct {
|
||||
FirewallOnly any // 是否只记录防火墙相关
|
||||
Version any // 版本号
|
||||
DisableDefaultDB any // 是否停止默认数据库存储
|
||||
WriteTargets any // 写入目标 JSON
|
||||
}
|
||||
|
||||
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {
|
||||
|
||||
@@ -1168,6 +1168,16 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
|
||||
if config.GlobalServerConfig == nil {
|
||||
config.GlobalServerConfig = nodeCluster.DecodeGlobalServerConfig()
|
||||
}
|
||||
// 注入公用访问日志策略的写入目标(供节点决定是否写文件、是否上报 API)
|
||||
if config.GlobalServerConfig != nil && clusterIndex == 0 {
|
||||
publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx)
|
||||
if publicPolicyId > 0 {
|
||||
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
|
||||
if publicPolicy != nil {
|
||||
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 最大线程数、TCP连接数
|
||||
if clusterIndex == 0 {
|
||||
@@ -1972,6 +1982,25 @@ func (this *NodeDAO) FindEnabledNodesWithIds(tx *dbs.Tx, nodeIds []int64) (resul
|
||||
return
|
||||
}
|
||||
|
||||
// FindEnabledBasicNodesWithIds 根据一组ID查找节点基本信息(id, name, clusterId),用于批量填充日志列表的 Node/Cluster
|
||||
func (this *NodeDAO) FindEnabledBasicNodesWithIds(tx *dbs.Tx, nodeIds []int64) (result []*Node, err error) {
|
||||
if len(nodeIds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
idStrings := []string{}
|
||||
for _, nodeId := range nodeIds {
|
||||
idStrings = append(idStrings, numberutils.FormatInt64(nodeId))
|
||||
}
|
||||
_, err = this.Query(tx).
|
||||
State(NodeStateEnabled).
|
||||
Where("id IN ("+strings.Join(idStrings, ", ")+")").
|
||||
Result("id", "name", "clusterId").
|
||||
Slice(&result).
|
||||
Reuse(false).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteNodeFromCluster 从集群中删除节点
|
||||
func (this *NodeDAO) DeleteNodeFromCluster(tx *dbs.Tx, nodeId int64, clusterId int64) error {
|
||||
one, err := this.Query(tx).
|
||||
|
||||
@@ -65,3 +65,26 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
|
||||
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
||||
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
|
||||
if len(valueJSON) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
err = json.Unmarshal(valueJSON, out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out.Port <= 0 {
|
||||
out.Port = 8123
|
||||
}
|
||||
if out.Database == "" {
|
||||
out.Database = "default"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
|
||||
@@ -46,9 +47,8 @@ func (this *HTTPAccessLogService) CreateHTTPAccessLogs(ctx context.Context, req
|
||||
return &pb.CreateHTTPAccessLogsResponse{}, nil
|
||||
}
|
||||
|
||||
// ListHTTPAccessLogs 列出单页访问日志
|
||||
// ListHTTPAccessLogs 列出单页访问日志(优先 ClickHouse,否则 MySQL;ClickHouse 路径下节点/集群批量查询避免 N+1)
|
||||
func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *pb.ListHTTPAccessLogsRequest) (*pb.ListHTTPAccessLogsResponse, error) {
|
||||
// 校验请求
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -56,11 +56,8 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
// 检查服务ID
|
||||
if userId > 0 {
|
||||
req.UserId = userId
|
||||
|
||||
// 这里不用担心serverId <= 0 的情况,因为如果userId>0,则只会查询当前用户下的服务,不会产生安全问题
|
||||
if req.ServerId > 0 {
|
||||
err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId)
|
||||
if err != nil {
|
||||
@@ -69,6 +66,17 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
}
|
||||
}
|
||||
|
||||
store := clickhouse.NewLogsIngestStore()
|
||||
if store.Client().IsConfigured() && req.Day != "" {
|
||||
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
|
||||
if listErr != nil {
|
||||
return nil, listErr
|
||||
}
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -82,8 +90,6 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 节点 & 集群
|
||||
pbNode, ok := pbNodeMap[a.NodeId]
|
||||
if ok {
|
||||
a.Node = pbNode
|
||||
@@ -94,42 +100,131 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
}
|
||||
if node != nil {
|
||||
pbNode = &pb.Node{Id: int64(node.Id), Name: node.Name}
|
||||
|
||||
var clusterId = int64(node.ClusterId)
|
||||
pbCluster, ok := pbClusterMap[clusterId]
|
||||
if ok {
|
||||
pbNode.NodeCluster = pbCluster
|
||||
} else {
|
||||
if !ok {
|
||||
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, clusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cluster != nil {
|
||||
pbCluster = &pb.NodeCluster{
|
||||
Id: int64(cluster.Id),
|
||||
Name: cluster.Name,
|
||||
}
|
||||
pbNode.NodeCluster = pbCluster
|
||||
pbCluster = &pb.NodeCluster{Id: int64(cluster.Id), Name: cluster.Name}
|
||||
pbClusterMap[clusterId] = pbCluster
|
||||
}
|
||||
}
|
||||
|
||||
if pbCluster != nil {
|
||||
pbNode.NodeCluster = pbCluster
|
||||
}
|
||||
pbNodeMap[a.NodeId] = pbNode
|
||||
a.Node = pbNode
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, a)
|
||||
}
|
||||
|
||||
return &pb.ListHTTPAccessLogsResponse{
|
||||
HttpAccessLogs: result,
|
||||
AccessLogs: result, // TODO 仅仅为了兼容,当用户节点版本大于0.0.8时可以删除
|
||||
AccessLogs: result,
|
||||
HasMore: hasMore,
|
||||
RequestId: requestId,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// listHTTPAccessLogsFromClickHouse 从 ClickHouse logs_ingest 查列表,并批量填充 Node/NodeCluster(避免 N+1)
|
||||
func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.LogsIngestStore, req *pb.ListHTTPAccessLogsRequest, userId int64) (*pb.ListHTTPAccessLogsResponse, error) {
|
||||
f := clickhouse.ListFilter{
|
||||
Day: req.Day,
|
||||
HourFrom: req.HourFrom,
|
||||
HourTo: req.HourTo,
|
||||
Size: req.Size,
|
||||
Reverse: req.Reverse,
|
||||
HasError: req.HasError,
|
||||
HasFirewallPolicy: req.HasFirewallPolicy,
|
||||
FirewallPolicyId: req.FirewallPolicyId,
|
||||
NodeId: req.NodeId,
|
||||
ClusterId: req.NodeClusterId,
|
||||
LastRequestId: req.RequestId,
|
||||
}
|
||||
if req.ServerId > 0 {
|
||||
f.ServerIds = []int64{req.ServerId}
|
||||
} else if userId > 0 {
|
||||
serverIds, err := models.SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(serverIds) == 0 {
|
||||
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: nil, AccessLogs: nil, HasMore: false, RequestId: ""}, nil
|
||||
}
|
||||
f.ServerIds = serverIds
|
||||
}
|
||||
if req.NodeClusterId > 0 {
|
||||
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, req.NodeClusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.NodeIds = nodeIds
|
||||
}
|
||||
|
||||
rows, nextCursor, err := store.List(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: []*pb.HTTPAccessLog{}, AccessLogs: []*pb.HTTPAccessLog{}, HasMore: false, RequestId: ""}, nil
|
||||
}
|
||||
|
||||
result := make([]*pb.HTTPAccessLog, 0, len(rows))
|
||||
nodeIdSet := make(map[int64]struct{})
|
||||
for _, r := range rows {
|
||||
result = append(result, clickhouse.RowToPB(r))
|
||||
nodeIdSet[int64(r.NodeId)] = struct{}{}
|
||||
}
|
||||
nodeIds := make([]int64, 0, len(nodeIdSet))
|
||||
for id := range nodeIdSet {
|
||||
nodeIds = append(nodeIds, id)
|
||||
}
|
||||
nodes, err := models.SharedNodeDAO.FindEnabledBasicNodesWithIds(tx, nodeIds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clusterIds := make(map[int64]struct{})
|
||||
for _, node := range nodes {
|
||||
if node.ClusterId > 0 {
|
||||
clusterIds[int64(node.ClusterId)] = struct{}{}
|
||||
}
|
||||
}
|
||||
clusterIdList := make([]int64, 0, len(clusterIds))
|
||||
for cid := range clusterIds {
|
||||
clusterIdList = append(clusterIdList, cid)
|
||||
}
|
||||
clusters, _ := models.SharedNodeClusterDAO.FindEnabledNodeClustersWithIds(tx, clusterIdList)
|
||||
clusterMap := make(map[int64]*pb.NodeCluster)
|
||||
for _, c := range clusters {
|
||||
clusterMap[int64(c.Id)] = &pb.NodeCluster{Id: int64(c.Id), Name: c.Name}
|
||||
}
|
||||
pbNodeMap := make(map[int64]*pb.Node)
|
||||
for _, node := range nodes {
|
||||
pbNode := &pb.Node{Id: int64(node.Id), Name: node.Name}
|
||||
if c := clusterMap[int64(node.ClusterId)]; c != nil {
|
||||
pbNode.NodeCluster = c
|
||||
}
|
||||
pbNodeMap[int64(node.Id)] = pbNode
|
||||
}
|
||||
for _, a := range result {
|
||||
if n := pbNodeMap[a.NodeId]; n != nil {
|
||||
a.Node = n
|
||||
}
|
||||
}
|
||||
|
||||
hasMore := nextCursor != ""
|
||||
return &pb.ListHTTPAccessLogsResponse{
|
||||
HttpAccessLogs: result,
|
||||
AccessLogs: result,
|
||||
HasMore: hasMore,
|
||||
RequestId: nextCursor,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FindHTTPAccessLog 查找单个日志
|
||||
func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb.FindHTTPAccessLogRequest) (*pb.FindHTTPAccessLogResponse, error) {
|
||||
// 校验请求
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
|
||||
return !accesslogs.SharedStorageManager.DisableDefaultDB()
|
||||
return accesslogs.SharedStorageManager.WriteMySQL()
|
||||
}
|
||||
|
||||
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
|
||||
IsPublic: policy.IsPublic,
|
||||
FirewallOnly: policy.FirewallOnly == 1,
|
||||
DisableDefaultDB: policy.DisableDefaultDB,
|
||||
WriteTargetsJSON: policy.WriteTargets,
|
||||
})
|
||||
}
|
||||
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
|
||||
@@ -76,7 +77,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
|
||||
}
|
||||
|
||||
// 创建
|
||||
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB)
|
||||
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -101,7 +102,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
|
||||
}
|
||||
|
||||
// 保存修改
|
||||
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.IsOn)
|
||||
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -133,6 +134,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
|
||||
IsPublic: policy.IsPublic,
|
||||
FirewallOnly: policy.FirewallOnly == 1,
|
||||
DisableDefaultDB: policy.DisableDefaultDB,
|
||||
WriteTargetsJSON: policy.WriteTargets,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
@@ -26,6 +27,11 @@ func (this *SysSettingService) UpdateSysSetting(ctx context.Context, req *pb.Upd
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 若为 ClickHouse 配置,清空缓存使下次读取生效
|
||||
if req.Code == "clickhouseConfig" {
|
||||
clickhouse.ResetSharedConfig()
|
||||
}
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
|
||||
@@ -113017,7 +113017,7 @@
|
||||
"name": "edgeHTTPAccessLogPolicies",
|
||||
"engine": "InnoDB",
|
||||
"charset": "utf8mb4_unicode_ci",
|
||||
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
|
||||
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n `writeTargets` json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
@@ -113078,6 +113078,10 @@
|
||||
{
|
||||
"name": "disableDefaultDB",
|
||||
"definition": "tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储'"
|
||||
},
|
||||
{
|
||||
"name": "writeTargets",
|
||||
"definition": "json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse'"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
FROM --platform=linux/amd64 alpine:latest
|
||||
LABEL maintainer="goedge.cdn@gmail.com"
|
||||
ENV TZ "Asia/Shanghai"
|
||||
ENV VERSION 1.4.5
|
||||
ENV VERSION 1.4.6
|
||||
ENV ROOT_DIR /usr/local/goedge
|
||||
ENV TAR_FILE edge-admin-linux-amd64-plus-v${VERSION}.zip
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5" //1.3.9
|
||||
Version = "1.4.6" //1.3.9
|
||||
|
||||
APINodeVersion = "1.4.5" //1.3.9
|
||||
APINodeVersion = "1.4.6" //1.3.9
|
||||
|
||||
ProductName = "Edge Admin"
|
||||
ProcessName = "edge-admin"
|
||||
|
||||
98
EdgeAdmin/internal/web/actions/default/db/clickhouse.go
Normal file
98
EdgeAdmin/internal/web/actions/default/db/clickhouse.go
Normal file
@@ -0,0 +1,98 @@
|
||||
//go:build plus
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/langs/codes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||
"github.com/iwind/TeaGo/actions"
|
||||
)
|
||||
|
||||
const clickhouseConfigCode = "clickhouseConfig"
|
||||
|
||||
type ClickHouseAction struct {
|
||||
actionutils.ParentAction
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) Init() {
|
||||
this.Nav("db", "db", "clickhouse")
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||
this.Data["mainTab"] = "clickhouse"
|
||||
resp, err := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: clickhouseConfigCode})
|
||||
if err != nil {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
cfg := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
|
||||
if len(resp.ValueJSON) > 0 {
|
||||
_ = json.Unmarshal(resp.ValueJSON, cfg)
|
||||
}
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = 8123
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = "default"
|
||||
}
|
||||
this.Data["config"] = map[string]interface{}{
|
||||
"host": cfg.Host,
|
||||
"port": cfg.Port,
|
||||
"user": cfg.User,
|
||||
"password": cfg.Password,
|
||||
"database": cfg.Database,
|
||||
}
|
||||
this.Show()
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) RunPost(params struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
|
||||
Must *actions.Must
|
||||
}) {
|
||||
defer this.CreateLogInfo(codes.DBNode_LogUpdateDBNode, 0)
|
||||
if params.Port <= 0 {
|
||||
params.Port = 8123
|
||||
}
|
||||
if params.Database == "" {
|
||||
params.Database = "default"
|
||||
}
|
||||
password := params.Password
|
||||
if password == "" {
|
||||
resp, _ := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: clickhouseConfigCode})
|
||||
if len(resp.ValueJSON) > 0 {
|
||||
var old systemconfigs.ClickHouseSetting
|
||||
if json.Unmarshal(resp.ValueJSON, &old) == nil {
|
||||
password = old.Password
|
||||
}
|
||||
}
|
||||
}
|
||||
cfg := &systemconfigs.ClickHouseSetting{
|
||||
Host: params.Host,
|
||||
Port: params.Port,
|
||||
User: params.User,
|
||||
Password: password,
|
||||
Database: params.Database,
|
||||
}
|
||||
valueJSON, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
_, err = this.RPC().SysSettingRPC().UpdateSysSetting(this.AdminContext(), &pb.UpdateSysSettingRequest{
|
||||
Code: clickhouseConfigCode,
|
||||
ValueJSON: valueJSON,
|
||||
})
|
||||
if err != nil {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
this.Success()
|
||||
}
|
||||
33
EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go
Normal file
33
EdgeAdmin/internal/web/actions/default/db/clickhouse_stub.go
Normal file
@@ -0,0 +1,33 @@
|
||||
//go:build !plus
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
|
||||
)
|
||||
|
||||
type ClickHouseAction struct {
|
||||
actionutils.ParentAction
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) Init() {
|
||||
this.Nav("db", "db", "clickhouse")
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) RunGet(params struct{}) {
|
||||
this.Data["mainTab"] = "clickhouse"
|
||||
this.Data["config"] = map[string]interface{}{
|
||||
"host": "", "port": 8123, "user": "", "password": "", "database": "default",
|
||||
}
|
||||
this.Show()
|
||||
}
|
||||
|
||||
func (this *ClickHouseAction) RunPost(params struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
}) {
|
||||
this.Fail("请使用商业版以在页面上配置 ClickHouse")
|
||||
}
|
||||
@@ -23,5 +23,6 @@ func (this *Helper) BeforeAction(action *actions.ActionObject) {
|
||||
|
||||
var tabbar = actionutils.NewTabbar()
|
||||
tabbar.Add(this.Lang(action, codes.DBNode_TabNodes), "", "/db", "", selectedTabbar == "db")
|
||||
tabbar.Add(this.Lang(action, codes.DBNode_TabClickHouse), "", "/db/clickhouse", "", selectedTabbar == "clickhouse")
|
||||
actionutils.SetTabbar(action, tabbar)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ func init() {
|
||||
Get("/node", new(NodeAction)).
|
||||
Get("/logs", new(LogsAction)).
|
||||
Post("/status", new(StatusAction)).
|
||||
GetPost("/clickhouse", new(ClickHouseAction)).
|
||||
EndAll()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -87,8 +87,13 @@ func (this *CreatePopupAction) RunPost(params struct {
|
||||
Field("type", params.Type).
|
||||
Require("请选择存储类型")
|
||||
|
||||
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
|
||||
if writeTargets == nil {
|
||||
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
|
||||
}
|
||||
|
||||
var options any = nil
|
||||
switch params.Type {
|
||||
switch baseType {
|
||||
case serverconfigs.AccessLogStorageTypeFile:
|
||||
params.Must.
|
||||
Field("filePath", params.FilePath).
|
||||
@@ -170,14 +175,21 @@ func (this *CreatePopupAction) RunPost(params struct {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
writeTargetsMap := map[string]bool{
|
||||
"file": writeTargets.File,
|
||||
"mysql": writeTargets.MySQL,
|
||||
"clickhouse": writeTargets.ClickHouse,
|
||||
}
|
||||
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
|
||||
createResp, err := this.RPC().HTTPAccessLogPolicyRPC().CreateHTTPAccessLogPolicy(this.AdminContext(), &pb.CreateHTTPAccessLogPolicyRequest{
|
||||
Name: params.Name,
|
||||
Type: params.Type,
|
||||
Type: baseType,
|
||||
OptionsJSON: optionsJSON,
|
||||
CondsJSON: nil, // TODO
|
||||
IsPublic: params.IsPublic,
|
||||
FirewallOnly: params.FirewallOnly,
|
||||
DisableDefaultDB: params.DisableDefaultDB,
|
||||
WriteTargetsJSON: writeTargetsJSON,
|
||||
})
|
||||
if err != nil {
|
||||
this.ErrorPage(err)
|
||||
|
||||
@@ -46,11 +46,16 @@ func (this *IndexAction) RunGet(params struct{}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
|
||||
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
|
||||
if typeDisplay == "" {
|
||||
typeDisplay = policy.Type
|
||||
}
|
||||
policyMaps = append(policyMaps, maps.Map{
|
||||
"id": policy.Id,
|
||||
"name": policy.Name,
|
||||
"type": policy.Type,
|
||||
"typeName": serverconfigs.FindAccessLogStorageTypeName(policy.Type),
|
||||
"typeName": serverconfigs.FindAccessLogStorageTypeName(typeDisplay),
|
||||
"isOn": policy.IsOn,
|
||||
"isPublic": policy.IsPublic,
|
||||
"firewallOnly": policy.FirewallOnly,
|
||||
|
||||
@@ -36,11 +36,18 @@ func InitPolicy(parent *actionutils.ParentAction, policyId int64) error {
|
||||
}
|
||||
}
|
||||
|
||||
writeTargets := serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargetsJSON, policy.Type, policy.DisableDefaultDB)
|
||||
typeDisplay := serverconfigs.ComposeStorageTypeDisplay(policy.Type, writeTargets)
|
||||
if typeDisplay == "" {
|
||||
typeDisplay = policy.Type
|
||||
}
|
||||
|
||||
parent.Data["policy"] = maps.Map{
|
||||
"id": policy.Id,
|
||||
"name": policy.Name,
|
||||
"type": policy.Type,
|
||||
"typeName": serverconfigs.FindAccessLogStorageTypeName(policy.Type),
|
||||
"typeDisplay": typeDisplay,
|
||||
"typeName": serverconfigs.FindAccessLogStorageTypeName(typeDisplay),
|
||||
"isOn": policy.IsOn,
|
||||
"isPublic": policy.IsPublic,
|
||||
"firewallOnly": policy.FirewallOnly,
|
||||
|
||||
@@ -39,6 +39,7 @@ func (this *UpdateAction) RunGet(params struct {
|
||||
func (this *UpdateAction) RunPost(params struct {
|
||||
PolicyId int64
|
||||
Name string
|
||||
Type string // 存储类型(含组合:file / file_mysql / file_clickhouse / file_mysql_clickhouse / es / tcp / syslog / command)
|
||||
|
||||
// file
|
||||
FilePath string
|
||||
@@ -101,10 +102,17 @@ func (this *UpdateAction) RunPost(params struct {
|
||||
|
||||
params.Must.
|
||||
Field("name", params.Name).
|
||||
Require("请输入日志策略的名称")
|
||||
Require("请输入日志策略的名称").
|
||||
Field("type", params.Type).
|
||||
Require("请选择存储类型")
|
||||
|
||||
baseType, writeTargets := serverconfigs.ParseStorageTypeAndWriteTargets(params.Type)
|
||||
if writeTargets == nil {
|
||||
writeTargets = &serverconfigs.AccessLogWriteTargets{File: true, MySQL: true}
|
||||
}
|
||||
|
||||
var options interface{} = nil
|
||||
switch policy.Type {
|
||||
switch baseType {
|
||||
case serverconfigs.AccessLogStorageTypeFile:
|
||||
params.Must.
|
||||
Field("filePath", params.FilePath).
|
||||
@@ -187,15 +195,23 @@ func (this *UpdateAction) RunPost(params struct {
|
||||
this.ErrorPage(err)
|
||||
return
|
||||
}
|
||||
writeTargetsMap := map[string]bool{
|
||||
"file": writeTargets.File,
|
||||
"mysql": writeTargets.MySQL,
|
||||
"clickhouse": writeTargets.ClickHouse,
|
||||
}
|
||||
writeTargetsJSON, _ := json.Marshal(writeTargetsMap)
|
||||
_, err = this.RPC().HTTPAccessLogPolicyRPC().UpdateHTTPAccessLogPolicy(this.AdminContext(), &pb.UpdateHTTPAccessLogPolicyRequest{
|
||||
HttpAccessLogPolicyId: params.PolicyId,
|
||||
Name: params.Name,
|
||||
Type: baseType,
|
||||
OptionsJSON: optionsJSON,
|
||||
CondsJSON: nil, // TODO
|
||||
IsOn: params.IsOn,
|
||||
IsPublic: params.IsPublic,
|
||||
FirewallOnly: params.FirewallOnly,
|
||||
DisableDefaultDB: params.DisableDefaultDB,
|
||||
WriteTargetsJSON: writeTargetsJSON,
|
||||
})
|
||||
if err != nil {
|
||||
this.ErrorPage(err)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package settingutils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/configloaders"
|
||||
teaconst "github.com/TeaOSLab/EdgeAdmin/internal/const"
|
||||
"github.com/TeaOSLab/EdgeAdmin/internal/plus"
|
||||
@@ -46,7 +48,10 @@ func (this *AdvancedHelper) BeforeAction(actionPtr actions.ActionWrapper) (goNex
|
||||
if plus.AllowComponent(plus.ComponentCodeUser) {
|
||||
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabUserNodes), "", "/settings/userNodes", "", this.tab == "userNodes")
|
||||
}
|
||||
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabAccessLogDatabases), "", "/db", "", this.tab == "dbNodes")
|
||||
// 外层始终显示「日志数据库」与「ClickHouse 配置」两个标签,不随点击变化
|
||||
path := action.Request.URL.Path
|
||||
tabbar.Add(this.Lang(actionPtr, codes.AdminSetting_TabAccessLogDatabases), "", "/db", "", (path == "/db" || strings.HasPrefix(path, "/db/")) && path != "/db/clickhouse")
|
||||
tabbar.Add(this.Lang(actionPtr, codes.DBNode_TabClickHouse), "", "/db/clickhouse", "", path == "/db/clickhouse")
|
||||
if teaconst.IsPlus {
|
||||
// 目前仅在调试模式下使用
|
||||
if Tea.IsTesting() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<first-menu>
|
||||
<first-menu v-if="firstMenuItem !== 'clickhouse'">
|
||||
<menu-item href="/db">所有节点</menu-item>
|
||||
<span class="item disabled">|</span>
|
||||
<menu-item :href="'/db/node?nodeId=' + node.id" code="node">"{{node.name}}"详情</menu-item>
|
||||
|
||||
46
EdgeAdmin/web/views/@default/db/clickHouse.html
Normal file
46
EdgeAdmin/web/views/@default/db/clickHouse.html
Normal file
@@ -0,0 +1,46 @@
|
||||
{$layout}
|
||||
{$template "menu"}
|
||||
|
||||
<h3>ClickHouse 配置</h3>
|
||||
<p class="comment">用于访问日志列表查询(logs_ingest 表)。配置后,访问日志列表将优先从 ClickHouse 读取;不配置则仅从 MySQL 读取。留空表示不使用 ClickHouse。</p>
|
||||
|
||||
<form method="post" class="ui form" data-tea-action="$" data-tea-success="success" @submit.prevent="onSubmit">
|
||||
<csrf-token></csrf-token>
|
||||
<table class="ui table definition selectable">
|
||||
<tr>
|
||||
<td class="title">连接地址(Host)</td>
|
||||
<td>
|
||||
<input type="text" name="host" maxlength="200" ref="focus" placeholder="如 127.0.0.1 或 clickhouse.example.com" :value="config.host"/>
|
||||
<p class="comment">ClickHouse 服务器地址。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>端口(Port)</td>
|
||||
<td>
|
||||
<input type="number" name="port" min="1" max="65535" style="width:6em" :value="config.port"/>
|
||||
<p class="comment">HTTP 接口端口,默认 8123。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>用户名(User)</td>
|
||||
<td>
|
||||
<input type="text" name="user" maxlength="100" :value="config.user"/>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>密码(Password)</td>
|
||||
<td>
|
||||
<input type="password" name="password" maxlength="200" placeholder="不修改请留空" value=""/>
|
||||
<p class="comment">留空则不修改已保存的密码。</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>数据库名(Database)</td>
|
||||
<td>
|
||||
<input type="text" name="database" maxlength="100" placeholder="default" :value="config.database"/>
|
||||
<p class="comment">logs_ingest 表所在库,默认 default。</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<submit-btn></submit-btn>
|
||||
</form>
|
||||
12
EdgeAdmin/web/views/@default/db/clickHouse.js
Normal file
12
EdgeAdmin/web/views/@default/db/clickHouse.js
Normal file
@@ -0,0 +1,12 @@
|
||||
Tea.context(function () {
|
||||
this.success = function () {
|
||||
teaweb.success("保存成功")
|
||||
}
|
||||
this.onSubmit = function (e) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
Tea.action("$").post().form(e.target).success(function () {
|
||||
Tea.Vue.success()
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -15,13 +15,14 @@
|
||||
<td>
|
||||
<select class="ui dropdown auto-width" name="type" v-model="type">
|
||||
<option value="">[选择类型]</option>
|
||||
<option v-for="type in types" :value="type.code">{{type.name}}</option>
|
||||
<option v-for="t in types" :value="t.code">{{t.name}}</option>
|
||||
</select>
|
||||
<p class="comment">可选:文件、文件+MySQL、文件+ClickHouse、文件+MySQL+ClickHouse、ElasticSearch、TCP、Syslog、命令行等。</p>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- 文件 -->
|
||||
<tbody v-show="type == 'file'">
|
||||
<!-- 文件(含 文件 / 文件+MySQL / 文件+ClickHouse / 文件+MySQL+ClickHouse) -->
|
||||
<tbody v-show="type == 'file' || type == 'file_mysql' || type == 'file_clickhouse' || type == 'file_mysql_clickhouse'">
|
||||
<tr>
|
||||
<td>日志文件路径 *</td>
|
||||
<td>
|
||||
|
||||
@@ -16,12 +16,14 @@
|
||||
<tr>
|
||||
<td>存储类型 *</td>
|
||||
<td>
|
||||
{{policy.typeName}}
|
||||
<select class="ui dropdown auto-width" name="type" v-model="policy.typeDisplay">
|
||||
<option v-for="t in types" :value="t.code">{{t.name}}</option>
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- 文件 -->
|
||||
<tbody v-show="type == 'file'">
|
||||
<tbody v-show="policy.typeDisplay == 'file' || policy.typeDisplay == 'file_mysql' || policy.typeDisplay == 'file_clickhouse' || policy.typeDisplay == 'file_mysql_clickhouse'">
|
||||
<tr>
|
||||
<td>日志文件路径 *</td>
|
||||
<td>
|
||||
@@ -52,7 +54,7 @@
|
||||
</tbody>
|
||||
|
||||
<!-- Elastic Search -->
|
||||
<tbody v-show="type == 'es'">
|
||||
<tbody v-show="policy.typeDisplay == 'es'">
|
||||
<tr>
|
||||
<td>Endpoint *</td>
|
||||
<td>
|
||||
@@ -114,7 +116,7 @@
|
||||
</tbody>
|
||||
|
||||
<!-- TCP Socket -->
|
||||
<tbody v-show="type == 'tcp'">
|
||||
<tbody v-show="policy.typeDisplay == 'tcp'">
|
||||
<tr>
|
||||
<td>网络协议 *</td>
|
||||
<td>
|
||||
@@ -134,7 +136,7 @@
|
||||
</tbody>
|
||||
|
||||
<!-- Syslog -->
|
||||
<tbody v-show="type == 'syslog'">
|
||||
<tbody v-show="policy.typeDisplay == 'syslog'">
|
||||
<tr>
|
||||
<td>网络协议</td>
|
||||
<td>
|
||||
@@ -188,7 +190,7 @@
|
||||
</tbody>
|
||||
|
||||
<!-- 命令行输入流 -->
|
||||
<tbody v-show="type == 'command'">
|
||||
<tbody v-show="policy.typeDisplay == 'command'">
|
||||
<tr>
|
||||
<td>可执行文件 *</td>
|
||||
<td>
|
||||
|
||||
BIN
EdgeCommon/build/.DS_Store
vendored
Normal file
BIN
EdgeCommon/build/.DS_Store
vendored
Normal file
Binary file not shown.
@@ -168,6 +168,7 @@ const (
|
||||
DBNode_LogTruncateTable langs.MessageCode = "db_node@log_truncate_table" // 清空数据库节点 %d 数据表 %s 数据
|
||||
DBNode_LogUpdateDBNode langs.MessageCode = "db_node@log_update_db_node" // 修改数据库节点 %d
|
||||
DBNode_TabNodes langs.MessageCode = "db_node@tab_nodes" // 数据库节点
|
||||
DBNode_TabClickHouse langs.MessageCode = "db_node@tab_clickhouse" // ClickHouse 配置
|
||||
DDoSProtection_LogUpdateClusterDDoSProtection langs.MessageCode = "ddos_protection@log_update_cluster_ddos_protection" // 修改集群 %d 的DDOS防护设置
|
||||
DDoSProtection_LogUpdateNodeDDoSProtection langs.MessageCode = "ddos_protection@log_update_node_ddos_protection" // 修改节点 %d 的DDOS防护设置
|
||||
DNS_LogCreateDomain langs.MessageCode = "dns@log_create_domain" // 添加管理域名到DNS服务商 %d
|
||||
|
||||
@@ -169,6 +169,7 @@ func init() {
|
||||
"db_node@log_truncate_table": "",
|
||||
"db_node@log_update_db_node": "",
|
||||
"db_node@tab_nodes": "",
|
||||
"db_node@tab_clickhouse": "ClickHouse",
|
||||
"ddos_protection@log_update_cluster_ddos_protection": "",
|
||||
"ddos_protection@log_update_node_ddos_protection": "",
|
||||
"dns@log_create_domain": "",
|
||||
|
||||
@@ -169,6 +169,7 @@ func init() {
|
||||
"db_node@log_truncate_table": "清空数据库节点 %d 数据表 %s 数据",
|
||||
"db_node@log_update_db_node": "修改数据库节点 %d",
|
||||
"db_node@tab_nodes": "数据库节点",
|
||||
"db_node@tab_clickhouse": "ClickHouse 配置",
|
||||
"ddos_protection@log_update_cluster_ddos_protection": "修改集群 %d 的DDOS防护设置",
|
||||
"ddos_protection@log_update_node_ddos_protection": "修改节点 %d 的DDOS防护设置",
|
||||
"dns@log_create_domain": "添加管理域名到DNS服务商 %d",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"tab_nodes": "数据库节点",
|
||||
"tab_clickhouse": "ClickHouse 配置",
|
||||
|
||||
"log_create_db_node": "创建数据库节点 %d",
|
||||
"log_delete_db_node": "删除数据库节点 %d",
|
||||
|
||||
@@ -31,6 +31,7 @@ type HTTPAccessLogPolicy struct {
|
||||
IsPublic bool `protobuf:"varint,7,opt,name=isPublic,proto3" json:"isPublic,omitempty"` // 是否公用
|
||||
FirewallOnly bool `protobuf:"varint,8,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"` // 是否只记录WAF相关访问日志
|
||||
DisableDefaultDB bool `protobuf:"varint,9,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"` // 停用默认数据库存储
|
||||
WriteTargetsJSON []byte `protobuf:"bytes,10,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"` // 写入目标 JSON
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -128,6 +129,13 @@ func (x *HTTPAccessLogPolicy) GetDisableDefaultDB() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *HTTPAccessLogPolicy) GetWriteTargetsJSON() []byte {
|
||||
if x != nil {
|
||||
return x.WriteTargetsJSON
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_models_model_http_access_log_policy_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_models_model_http_access_log_policy_proto_rawDesc = []byte{
|
||||
|
||||
@@ -164,6 +164,7 @@ type CreateHTTPAccessLogPolicyRequest struct {
|
||||
IsPublic bool `protobuf:"varint,5,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
|
||||
FirewallOnly bool `protobuf:"varint,6,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
|
||||
DisableDefaultDB bool `protobuf:"varint,7,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
|
||||
WriteTargetsJSON []byte `protobuf:"bytes,8,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -247,6 +248,13 @@ func (x *CreateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *CreateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
|
||||
if x != nil {
|
||||
return x.WriteTargetsJSON
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type CreateHTTPAccessLogPolicyResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
|
||||
@@ -296,12 +304,14 @@ type UpdateHTTPAccessLogPolicyRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
|
||||
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
|
||||
Type string `protobuf:"bytes,10,opt,name=type,proto3" json:"type,omitempty"`
|
||||
IsOn bool `protobuf:"varint,3,opt,name=isOn,proto3" json:"isOn,omitempty"`
|
||||
OptionsJSON []byte `protobuf:"bytes,4,opt,name=optionsJSON,proto3" json:"optionsJSON,omitempty"`
|
||||
CondsJSON []byte `protobuf:"bytes,5,opt,name=condsJSON,proto3" json:"condsJSON,omitempty"`
|
||||
IsPublic bool `protobuf:"varint,6,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
|
||||
FirewallOnly bool `protobuf:"varint,7,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
|
||||
DisableDefaultDB bool `protobuf:"varint,8,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
|
||||
WriteTargetsJSON []byte `protobuf:"bytes,9,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -350,6 +360,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetName() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpdateHTTPAccessLogPolicyRequest) GetType() string {
|
||||
if x != nil {
|
||||
return x.Type
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpdateHTTPAccessLogPolicyRequest) GetIsOn() bool {
|
||||
if x != nil {
|
||||
return x.IsOn
|
||||
@@ -392,6 +409,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *UpdateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
|
||||
if x != nil {
|
||||
return x.WriteTargetsJSON
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 查找单个访问日志策略
|
||||
type FindHTTPAccessLogPolicyRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
|
||||
@@ -13,4 +13,5 @@ message HTTPAccessLogPolicy {
|
||||
bool isPublic = 7; // 是否公用
|
||||
bool firewallOnly = 8; // 是否只记录WAF相关访问日志
|
||||
bool disableDefaultDB = 9; // 停用默认数据库存储
|
||||
bytes writeTargetsJSON = 10; // 写入目标 JSON: {"file":true,"mysql":true,"clickhouse":false}
|
||||
}
|
||||
@@ -55,6 +55,7 @@ message CreateHTTPAccessLogPolicyRequest {
|
||||
bool isPublic = 5;
|
||||
bool firewallOnly = 6;
|
||||
bool disableDefaultDB = 7;
|
||||
bytes writeTargetsJSON = 8;
|
||||
}
|
||||
|
||||
message CreateHTTPAccessLogPolicyResponse {
|
||||
@@ -65,12 +66,14 @@ message CreateHTTPAccessLogPolicyResponse {
|
||||
message UpdateHTTPAccessLogPolicyRequest {
|
||||
int64 httpAccessLogPolicyId = 1;
|
||||
string name = 2;
|
||||
string type = 10; // 存储类型:file / es / tcp / syslog / command
|
||||
bool isOn = 3;
|
||||
bytes optionsJSON = 4;
|
||||
bytes condsJSON = 5;
|
||||
bool isPublic = 6;
|
||||
bool firewallOnly = 7;
|
||||
bool disableDefaultDB = 8;
|
||||
bytes writeTargetsJSON = 9;
|
||||
}
|
||||
|
||||
// 查找单个访问日志策略
|
||||
|
||||
@@ -4,45 +4,31 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
)
|
||||
|
||||
// AccessLogStorageType 访问日志存储类型
|
||||
// AccessLogStorageType 访问日志存储类型(含「存储+写入目标」组合)
|
||||
type AccessLogStorageType = string
|
||||
|
||||
const (
|
||||
AccessLogStorageTypeFile AccessLogStorageType = "file"
|
||||
AccessLogStorageTypeFileMySQL AccessLogStorageType = "file_mysql"
|
||||
AccessLogStorageTypeFileClickhouse AccessLogStorageType = "file_clickhouse"
|
||||
AccessLogStorageTypeFileMySQLClickhouse AccessLogStorageType = "file_mysql_clickhouse"
|
||||
AccessLogStorageTypeES AccessLogStorageType = "es"
|
||||
AccessLogStorageTypeTCP AccessLogStorageType = "tcp"
|
||||
AccessLogStorageTypeSyslog AccessLogStorageType = "syslog"
|
||||
AccessLogStorageTypeCommand AccessLogStorageType = "command"
|
||||
)
|
||||
|
||||
// FindAllAccessLogStorageTypes 所有存储引擎列表
|
||||
// FindAllAccessLogStorageTypes 所有存储引擎列表(含写入目标组合:文件、文件+MySQL、文件+ClickHouse 等)
|
||||
func FindAllAccessLogStorageTypes() []*shared.Definition {
|
||||
return []*shared.Definition{
|
||||
{
|
||||
Name: "文件",
|
||||
Code: AccessLogStorageTypeFile,
|
||||
Description: "将日志存储在磁盘文件中",
|
||||
},
|
||||
{
|
||||
Name: "ElasticSearch",
|
||||
Code: AccessLogStorageTypeES,
|
||||
Description: "将日志存储在ElasticSearch中",
|
||||
},
|
||||
{
|
||||
Name: "TCP Socket",
|
||||
Code: AccessLogStorageTypeTCP,
|
||||
Description: "将日志通过TCP套接字输出",
|
||||
},
|
||||
{
|
||||
Name: "Syslog",
|
||||
Code: AccessLogStorageTypeSyslog,
|
||||
Description: "将日志通过syslog输出,仅支持Linux",
|
||||
},
|
||||
{
|
||||
Name: "命令行输入流",
|
||||
Code: AccessLogStorageTypeCommand,
|
||||
Description: "启动一个命令通过读取stdin接收日志信息",
|
||||
},
|
||||
{Name: "文件", Code: AccessLogStorageTypeFile, Description: "节点写本地 JSON 文件"},
|
||||
{Name: "文件+MySQL", Code: AccessLogStorageTypeFileMySQL, Description: "节点写文件 + API 写 MySQL"},
|
||||
{Name: "文件+ClickHouse", Code: AccessLogStorageTypeFileClickhouse, Description: "节点写文件 + 落 ClickHouse(Fluent Bit 或 API 直写)"},
|
||||
{Name: "文件+MySQL+ClickHouse", Code: AccessLogStorageTypeFileMySQLClickhouse, Description: "节点写文件 + API 写 MySQL + ClickHouse"},
|
||||
{Name: "ElasticSearch", Code: AccessLogStorageTypeES, Description: "将日志存储在ElasticSearch中"},
|
||||
{Name: "TCP Socket", Code: AccessLogStorageTypeTCP, Description: "将日志通过TCP套接字输出"},
|
||||
{Name: "Syslog", Code: AccessLogStorageTypeSyslog, Description: "将日志通过syslog输出,仅支持Linux"},
|
||||
{Name: "命令行输入流", Code: AccessLogStorageTypeCommand, Description: "启动一个命令通过读取stdin接收日志信息"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,3 +41,65 @@ func FindAccessLogStorageTypeName(storageType string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// IsFileBasedStorageType 是否为基于文件的存储(需要显示文件路径等配置)
|
||||
func IsFileBasedStorageType(code string) bool {
|
||||
switch code {
|
||||
case AccessLogStorageTypeFile, AccessLogStorageTypeFileMySQL, AccessLogStorageTypeFileClickhouse, AccessLogStorageTypeFileMySQLClickhouse:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ParseStorageTypeAndWriteTargets 从下拉框选中的类型解析出「实际存储类型」与「写入目标」
|
||||
// 用于创建/更新策略:options 按 baseType 填(如 file),writeTargets 按组合填。
|
||||
func ParseStorageTypeAndWriteTargets(selectedType string) (baseType string, writeTargets *AccessLogWriteTargets) {
|
||||
writeTargets = &AccessLogWriteTargets{}
|
||||
switch selectedType {
|
||||
case AccessLogStorageTypeFile:
|
||||
baseType = AccessLogStorageTypeFile
|
||||
writeTargets.File = true
|
||||
case AccessLogStorageTypeFileMySQL:
|
||||
baseType = AccessLogStorageTypeFile
|
||||
writeTargets.File = true
|
||||
writeTargets.MySQL = true
|
||||
case AccessLogStorageTypeFileClickhouse:
|
||||
baseType = AccessLogStorageTypeFile
|
||||
writeTargets.File = true
|
||||
writeTargets.ClickHouse = true
|
||||
case AccessLogStorageTypeFileMySQLClickhouse:
|
||||
baseType = AccessLogStorageTypeFile
|
||||
writeTargets.File = true
|
||||
writeTargets.MySQL = true
|
||||
writeTargets.ClickHouse = true
|
||||
case AccessLogStorageTypeES, AccessLogStorageTypeTCP, AccessLogStorageTypeSyslog, AccessLogStorageTypeCommand:
|
||||
baseType = selectedType
|
||||
writeTargets.MySQL = true
|
||||
default:
|
||||
baseType = selectedType
|
||||
writeTargets.File = true
|
||||
writeTargets.MySQL = true
|
||||
}
|
||||
return baseType, writeTargets
|
||||
}
|
||||
|
||||
// ComposeStorageTypeDisplay 根据策略的 Type + WriteTargets 得到下拉框显示用的类型 code(用于编辑页回显)
|
||||
func ComposeStorageTypeDisplay(policyType string, writeTargets *AccessLogWriteTargets) string {
|
||||
if policyType != AccessLogStorageTypeFile {
|
||||
return policyType
|
||||
}
|
||||
if writeTargets == nil {
|
||||
return AccessLogStorageTypeFile
|
||||
}
|
||||
if writeTargets.File && writeTargets.MySQL && writeTargets.ClickHouse {
|
||||
return AccessLogStorageTypeFileMySQLClickhouse
|
||||
}
|
||||
if writeTargets.File && writeTargets.MySQL {
|
||||
return AccessLogStorageTypeFileMySQL
|
||||
}
|
||||
if writeTargets.File && writeTargets.ClickHouse {
|
||||
return AccessLogStorageTypeFileClickhouse
|
||||
}
|
||||
return AccessLogStorageTypeFile
|
||||
}
|
||||
|
||||
49
EdgeCommon/pkg/serverconfigs/access_log_write_targets.go
Normal file
49
EdgeCommon/pkg/serverconfigs/access_log_write_targets.go
Normal file
@@ -0,0 +1,49 @@
|
||||
// Copyright 2025. All rights reserved.
|
||||
|
||||
package serverconfigs
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// AccessLogWriteTargets 访问日志写入目标(双写/单写:文件、MySQL、ClickHouse)
|
||||
type AccessLogWriteTargets struct {
|
||||
File bool `yaml:"file" json:"file"` // 写本地 JSON 文件(供 Fluent Bit → ClickHouse 或自用)
|
||||
MySQL bool `yaml:"mysql" json:"mysql"` // 写 MySQL 默认库按日分表
|
||||
ClickHouse bool `yaml:"clickhouse" json:"clickhouse"` // 需要落 ClickHouse(文件+Fluent Bit 或 API 直写)
|
||||
}
|
||||
|
||||
// NeedReportToAPI 是否需要上报到 API(写 MySQL 或 API 直写 ClickHouse 时需要)
|
||||
func (t *AccessLogWriteTargets) NeedReportToAPI() bool {
|
||||
if t == nil {
|
||||
return true // 兼容:未配置时保持原行为,上报
|
||||
}
|
||||
return t.MySQL || t.ClickHouse
|
||||
}
|
||||
|
||||
// NeedWriteFile 节点是否需要写本地文件
|
||||
func (t *AccessLogWriteTargets) NeedWriteFile() bool {
|
||||
if t == nil {
|
||||
return true // 兼容:未配置时保持原行为,写文件
|
||||
}
|
||||
return t.File
|
||||
}
|
||||
|
||||
// ParseWriteTargetsFromPolicy 从策略的 writeTargets JSON 与旧字段解析;无 writeTargets 时按 type + disableDefaultDB 推断
|
||||
func ParseWriteTargetsFromPolicy(writeTargetsJSON []byte, policyType string, disableDefaultDB bool) *AccessLogWriteTargets {
|
||||
if len(writeTargetsJSON) > 0 {
|
||||
var t AccessLogWriteTargets
|
||||
if err := json.Unmarshal(writeTargetsJSON, &t); err == nil {
|
||||
return &t
|
||||
}
|
||||
}
|
||||
// 兼容旧策略:type=file 视为写文件,!disableDefaultDB 视为写 MySQL
|
||||
t := &AccessLogWriteTargets{
|
||||
File: policyType == AccessLogStorageTypeFile,
|
||||
MySQL: !disableDefaultDB,
|
||||
ClickHouse: false,
|
||||
}
|
||||
if !t.File && !t.MySQL && !t.ClickHouse {
|
||||
t.File = true
|
||||
t.MySQL = true
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -1,10 +1,5 @@
|
||||
package firewallconfigs
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// AllCheckpoints all check points list
|
||||
var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
|
||||
{
|
||||
@@ -307,86 +302,6 @@ var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
|
||||
IsComposed: true,
|
||||
Priority: 20,
|
||||
},
|
||||
{
|
||||
Name: "CC统计(旧)",
|
||||
Prefix: "cc",
|
||||
Description: "统计某段时间段内的请求信息(不推荐再使用,请使用新的CC2统计代替)。",
|
||||
HasParams: true,
|
||||
Params: []*KeyValue{
|
||||
NewKeyValue("请求数", "requests"),
|
||||
},
|
||||
Options: []OptionInterface{
|
||||
&FieldOption{
|
||||
Type: "field",
|
||||
Name: "统计周期",
|
||||
Code: "period",
|
||||
Value: "60",
|
||||
IsRequired: false,
|
||||
Size: 8,
|
||||
Comment: "",
|
||||
Placeholder: "",
|
||||
RightLabel: "秒",
|
||||
MaxLength: 8,
|
||||
Validate: func(value string) (ok bool, message string) {
|
||||
if regexp.MustCompile(`^\d+$`).MatchString(value) {
|
||||
ok = true
|
||||
return
|
||||
}
|
||||
message = "周期需要是一个整数数字"
|
||||
return
|
||||
},
|
||||
},
|
||||
&OptionsOption{
|
||||
Type: "options",
|
||||
Name: "用户识别读取来源",
|
||||
Code: "userType",
|
||||
Value: "",
|
||||
IsRequired: false,
|
||||
Size: 10,
|
||||
Comment: "",
|
||||
RightLabel: "",
|
||||
Validate: nil,
|
||||
Options: []maps.Map{
|
||||
{
|
||||
"name": "IP",
|
||||
"value": "ip",
|
||||
},
|
||||
{
|
||||
"name": "Cookie",
|
||||
"value": "cookie",
|
||||
},
|
||||
{
|
||||
"name": "URL参数",
|
||||
"value": "get",
|
||||
},
|
||||
{
|
||||
"name": "POST参数",
|
||||
"value": "post",
|
||||
},
|
||||
{
|
||||
"name": "HTTP Header",
|
||||
"value": "header",
|
||||
},
|
||||
},
|
||||
},
|
||||
&FieldOption{
|
||||
Type: "field",
|
||||
Name: "用户识别字段",
|
||||
Code: "userField",
|
||||
Comment: "识别用户的唯一性字段,在用户读取来源不是IP时使用",
|
||||
},
|
||||
&FieldOption{
|
||||
Type: "field",
|
||||
Name: "字段读取位置",
|
||||
Code: "userIndex",
|
||||
Size: 5,
|
||||
MaxLength: 5,
|
||||
Comment: "读取用户识别字段的位置,从0开始,比如user12345的数字ID 12345的位置就是5,在用户读取来源不是IP时使用",
|
||||
},
|
||||
},
|
||||
IsRequest: true,
|
||||
Priority: 10,
|
||||
},
|
||||
{
|
||||
Name: "响应状态码",
|
||||
Prefix: "status",
|
||||
|
||||
@@ -77,6 +77,7 @@ type GlobalServerConfig struct {
|
||||
EnableResponseHeaders bool `yaml:"enableResponseHeaders" json:"enableResponseHeaders"` // 记录响应Header
|
||||
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
|
||||
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
|
||||
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标:文件/MySQL/ClickHouse(双写/单写)
|
||||
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
|
||||
|
||||
Stat struct {
|
||||
|
||||
12
EdgeCommon/pkg/systemconfigs/clickhouse_setting.go
Normal file
12
EdgeCommon/pkg/systemconfigs/clickhouse_setting.go
Normal file
@@ -0,0 +1,12 @@
|
||||
//go:build plus
|
||||
|
||||
package systemconfigs
|
||||
|
||||
// ClickHouseSetting 后台页面配置的 ClickHouse 连接(访问日志 logs_ingest 查询)
|
||||
type ClickHouseSetting struct {
|
||||
Host string `json:"host" yaml:"host"`
|
||||
Port int `json:"port" yaml:"port"`
|
||||
User string `json:"user" yaml:"user"`
|
||||
Password string `json:"password" yaml:"password"`
|
||||
Database string `json:"database" yaml:"database"`
|
||||
}
|
||||
@@ -15,4 +15,6 @@ const (
|
||||
|
||||
SettingCodeBillDay SettingCode = "billDay" // 账单日 YYYYMMDD
|
||||
SettingCodeBillMonth SettingCode = "billMonth" // 账单月 YYYYMM
|
||||
|
||||
SettingCodeClickHouseConfig SettingCode = "clickhouseConfig" // ClickHouse 连接配置(访问日志 logs_ingest 查询)
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5.1" //1.3.8.2
|
||||
Version = "1.4.6" //1.3.8.2
|
||||
|
||||
ProductName = "Edge DNS"
|
||||
ProcessName = "edge-dns"
|
||||
|
||||
206
EdgeNode/internal/accesslogs/file_writer.go
Normal file
206
EdgeNode/internal/accesslogs/file_writer.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package accesslogs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
)
|
||||
|
||||
var (
|
||||
sharedFileWriter *FileWriter
|
||||
sharedOnce sync.Once
|
||||
)
|
||||
|
||||
// SharedFileWriter 返回全局本地日志文件写入器(单例)
|
||||
func SharedFileWriter() *FileWriter {
|
||||
sharedOnce.Do(func() {
|
||||
sharedFileWriter = NewFileWriter()
|
||||
})
|
||||
return sharedFileWriter
|
||||
}
|
||||
|
||||
const (
|
||||
defaultLogDir = "/var/log/edge/edge-node"
|
||||
envLogDir = "EDGE_LOG_DIR"
|
||||
)
|
||||
|
||||
// FileWriter 将访问/WAF/错误日志以 JSON Lines 写入本地文件,便于 logrotate 与 Fluent Bit 采集
|
||||
type FileWriter struct {
|
||||
dir string
|
||||
mu sync.Mutex
|
||||
files map[string]*os.File // access.log, waf.log, error.log
|
||||
inited bool
|
||||
}
|
||||
|
||||
// NewFileWriter 创建本地日志文件写入器
|
||||
func NewFileWriter() *FileWriter {
|
||||
dir := os.Getenv(envLogDir)
|
||||
if dir == "" {
|
||||
dir = defaultLogDir
|
||||
}
|
||||
return &FileWriter{
|
||||
dir: dir,
|
||||
files: make(map[string]*os.File),
|
||||
}
|
||||
}
|
||||
|
||||
// Dir 返回当前配置的日志目录
|
||||
func (w *FileWriter) Dir() string {
|
||||
return w.dir
|
||||
}
|
||||
|
||||
// IsEnabled 是否启用落盘(目录非空即视为启用)
|
||||
func (w *FileWriter) IsEnabled() bool {
|
||||
return w.dir != ""
|
||||
}
|
||||
|
||||
// EnsureInit 在启动时预创建日志目录和空文件,便于 Fluent Bit 立即 tail,无需等首条访问日志
|
||||
func (w *FileWriter) EnsureInit() error {
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
return w.init()
|
||||
}
|
||||
|
||||
// init 确保目录存在并打开三个日志文件(仅首次或 Reopen 时)
|
||||
func (w *FileWriter) init() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.inited && len(w.files) > 0 {
|
||||
return nil
|
||||
}
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(w.dir, 0755); err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "mkdir log dir failed: "+err.Error())
|
||||
return err
|
||||
}
|
||||
for _, name := range []string{"access.log", "waf.log", "error.log"} {
|
||||
if w.files[name] != nil {
|
||||
continue
|
||||
}
|
||||
fp, err := os.OpenFile(filepath.Join(w.dir, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "open "+name+" failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
w.files[name] = fp
|
||||
}
|
||||
w.inited = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write 将一条访问日志按 log_type 写入对应文件(access.log / waf.log / error.log)
|
||||
func (w *FileWriter) Write(l *pb.HTTPAccessLog, clusterId int64) {
|
||||
if w.dir == "" {
|
||||
return
|
||||
}
|
||||
if err := w.init(); err != nil || len(w.files) == 0 {
|
||||
return
|
||||
}
|
||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
||||
line, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "marshal ingest log: "+err.Error())
|
||||
return
|
||||
}
|
||||
var fileName string
|
||||
switch logType {
|
||||
case LogTypeWAF:
|
||||
fileName = "waf.log"
|
||||
case LogTypeError:
|
||||
fileName = "error.log"
|
||||
default:
|
||||
fileName = "access.log"
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp := w.files[fileName]
|
||||
w.mu.Unlock()
|
||||
if fp == nil {
|
||||
return
|
||||
}
|
||||
// 单行写入,末尾换行,便于 Fluent Bit / JSON 解析
|
||||
_, err = fp.Write(append(line, '\n'))
|
||||
if err != nil {
|
||||
remotelogs.Error("ACCESS_LOG_FILE", "write "+fileName+" failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// WriteBatch 批量写入,减少锁竞争
|
||||
func (w *FileWriter) WriteBatch(logs []*pb.HTTPAccessLog, clusterId int64) {
|
||||
if w.dir == "" || len(logs) == 0 {
|
||||
return
|
||||
}
|
||||
if err := w.init(); err != nil || len(w.files) == 0 {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
accessFp := w.files["access.log"]
|
||||
wafFp := w.files["waf.log"]
|
||||
errorFp := w.files["error.log"]
|
||||
w.mu.Unlock()
|
||||
if accessFp == nil && wafFp == nil && errorFp == nil {
|
||||
return
|
||||
}
|
||||
for _, l := range logs {
|
||||
ingest, logType := FromHTTPAccessLog(l, clusterId)
|
||||
line, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
line = append(line, '\n')
|
||||
var fp *os.File
|
||||
switch logType {
|
||||
case LogTypeWAF:
|
||||
fp = wafFp
|
||||
case LogTypeError:
|
||||
fp = errorFp
|
||||
default:
|
||||
fp = accessFp
|
||||
}
|
||||
if fp != nil {
|
||||
_, _ = fp.Write(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen 关闭并重新打开所有日志文件(供 logrotate copytruncate 或 SIGHUP 后重开句柄)
|
||||
func (w *FileWriter) Reopen() error {
|
||||
if w.dir == "" {
|
||||
return nil
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
w.files[name] = nil
|
||||
}
|
||||
}
|
||||
w.inited = false
|
||||
return w.init()
|
||||
}
|
||||
|
||||
// Close 关闭所有已打开的文件
|
||||
func (w *FileWriter) Close() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var lastErr error
|
||||
for name, f := range w.files {
|
||||
if f != nil {
|
||||
if err := f.Close(); err != nil {
|
||||
lastErr = err
|
||||
remotelogs.Error("ACCESS_LOG_FILE", fmt.Sprintf("close %s: %v", name, err))
|
||||
}
|
||||
w.files[name] = nil
|
||||
}
|
||||
}
|
||||
w.inited = false
|
||||
return lastErr
|
||||
}
|
||||
137
EdgeNode/internal/accesslogs/ingest_log.go
Normal file
137
EdgeNode/internal/accesslogs/ingest_log.go
Normal file
@@ -0,0 +1,137 @@
|
||||
// Package accesslogs 提供边缘节点访问日志落盘(JSON Lines),供 Fluent Bit 采集写入 ClickHouse。
|
||||
package accesslogs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
|
||||
// LogType 与 Fluent Bit / logs_ingest 的 log_type 一致
|
||||
const (
|
||||
LogTypeAccess = "access"
|
||||
LogTypeWAF = "waf"
|
||||
LogTypeError = "error"
|
||||
)
|
||||
|
||||
// 请求/响应 body 落盘最大长度(字节),超出截断,避免单条过大
|
||||
const maxBodyLen = 512 * 1024
|
||||
|
||||
// IngestLog 单行 JSON 结构与方案文档、ClickHouse logs_ingest 表字段对齐
|
||||
type IngestLog struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
NodeId int64 `json:"node_id"`
|
||||
ClusterId int64 `json:"cluster_id"`
|
||||
ServerId int64 `json:"server_id"`
|
||||
Host string `json:"host"`
|
||||
IP string `json:"ip"`
|
||||
Method string `json:"method"`
|
||||
Path string `json:"path"`
|
||||
Status int32 `json:"status"`
|
||||
BytesIn int64 `json:"bytes_in"`
|
||||
BytesOut int64 `json:"bytes_out"`
|
||||
CostMs int64 `json:"cost_ms"`
|
||||
UA string `json:"ua"`
|
||||
Referer string `json:"referer"`
|
||||
LogType string `json:"log_type"`
|
||||
TraceId string `json:"trace_id,omitempty"`
|
||||
FirewallPolicyId int64 `json:"firewall_policy_id,omitempty"`
|
||||
FirewallRuleGroupId int64 `json:"firewall_rule_group_id,omitempty"`
|
||||
FirewallRuleSetId int64 `json:"firewall_rule_set_id,omitempty"`
|
||||
FirewallRuleId int64 `json:"firewall_rule_id,omitempty"`
|
||||
RequestHeaders string `json:"request_headers,omitempty"`
|
||||
RequestBody string `json:"request_body,omitempty"`
|
||||
ResponseHeaders string `json:"response_headers,omitempty"`
|
||||
ResponseBody string `json:"response_body,omitempty"`
|
||||
}
|
||||
|
||||
// stringsMapToJSON 将 map[string]*Strings 转为 JSON 字符串,便于落盘与 ClickHouse 存储
|
||||
func stringsMapToJSON(m map[string]*pb.Strings) string {
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := make(map[string]string, len(m))
|
||||
for k, v := range m {
|
||||
if v != nil && len(v.Values) > 0 {
|
||||
out[k] = v.Values[0]
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return ""
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// truncateBody 截断 body 到最大长度,避免单条过大
|
||||
func truncateBody(b []byte) string {
|
||||
if len(b) == 0 {
|
||||
return ""
|
||||
}
|
||||
s := string(b)
|
||||
if len(s) > maxBodyLen {
|
||||
return s[:maxBodyLen]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// buildRequestBody 将查询串与请求体合并写入 request_body 字段(不新增字段)
|
||||
func buildRequestBody(l *pb.HTTPAccessLog) string {
|
||||
q := l.GetQueryString()
|
||||
body := l.GetRequestBody()
|
||||
if len(q) == 0 && len(body) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return truncateBody([]byte(q))
|
||||
}
|
||||
combined := make([]byte, 0, len(q)+1+len(body))
|
||||
combined = append(combined, q...)
|
||||
combined = append(combined, '\n')
|
||||
combined = append(combined, body...)
|
||||
return truncateBody(combined)
|
||||
}
|
||||
|
||||
// FromHTTPAccessLog 从 pb.HTTPAccessLog 转为 IngestLog,并决定 log_type
|
||||
func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog, logType string) {
|
||||
ingest = IngestLog{
|
||||
Timestamp: l.GetTimestamp(),
|
||||
NodeId: l.GetNodeId(),
|
||||
ClusterId: clusterId,
|
||||
ServerId: l.GetServerId(),
|
||||
Host: l.GetHost(),
|
||||
IP: l.GetRawRemoteAddr(),
|
||||
Method: l.GetRequestMethod(),
|
||||
Path: l.GetRequestPath(),
|
||||
Status: l.GetStatus(),
|
||||
BytesIn: l.GetRequestLength(),
|
||||
BytesOut: l.GetBytesSent(),
|
||||
CostMs: int64(l.GetRequestTime() * 1000),
|
||||
UA: l.GetUserAgent(),
|
||||
Referer: l.GetReferer(),
|
||||
TraceId: l.GetRequestId(),
|
||||
FirewallPolicyId: l.GetFirewallPolicyId(),
|
||||
FirewallRuleGroupId: l.GetFirewallRuleGroupId(),
|
||||
FirewallRuleSetId: l.GetFirewallRuleSetId(),
|
||||
FirewallRuleId: l.GetFirewallRuleId(),
|
||||
RequestHeaders: stringsMapToJSON(l.GetHeader()),
|
||||
RequestBody: buildRequestBody(l),
|
||||
ResponseHeaders: stringsMapToJSON(l.GetSentHeader()),
|
||||
}
|
||||
if ingest.IP == "" {
|
||||
ingest.IP = l.GetRemoteAddr()
|
||||
}
|
||||
// 响应 body 当前 pb 未提供,若后续扩展可在此赋值
|
||||
// ingest.ResponseBody = ...
|
||||
|
||||
// 与方案一致:waf > error > access;攻击日志通过 firewall_rule_id / firewall_policy_id 判断
|
||||
if l.GetFirewallPolicyId() > 0 {
|
||||
logType = LogTypeWAF
|
||||
} else if len(l.GetErrors()) > 0 {
|
||||
logType = LogTypeError
|
||||
} else {
|
||||
logType = LogTypeAccess
|
||||
}
|
||||
ingest.LogType = logType
|
||||
return ingest, logType
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5.2" //1.3.8.2
|
||||
Version = "1.4.6" //1.3.8.2
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -3,6 +3,8 @@ package nodes
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
@@ -92,6 +94,22 @@ Loop:
|
||||
return nil
|
||||
}
|
||||
|
||||
var writeTargets *serverconfigs.AccessLogWriteTargets
|
||||
if sharedNodeConfig != nil && sharedNodeConfig.GlobalServerConfig != nil {
|
||||
writeTargets = sharedNodeConfig.GlobalServerConfig.HTTPAccessLog.WriteTargets
|
||||
}
|
||||
needWriteFile := writeTargets == nil || writeTargets.NeedWriteFile()
|
||||
needReportAPI := writeTargets == nil || writeTargets.NeedReportToAPI()
|
||||
|
||||
// 落盘 JSON Lines(Fluent Bit 采集 → ClickHouse)
|
||||
if needWriteFile {
|
||||
var clusterId int64
|
||||
if sharedNodeConfig != nil {
|
||||
clusterId = sharedNodeConfig.GroupId
|
||||
}
|
||||
accesslogs.SharedFileWriter().WriteBatch(accessLogs, clusterId)
|
||||
}
|
||||
|
||||
// 发送到本地
|
||||
if sharedHTTPAccessLogViewer.HasConns() {
|
||||
for _, accessLog := range accessLogs {
|
||||
@@ -99,7 +117,10 @@ Loop:
|
||||
}
|
||||
}
|
||||
|
||||
// 发送到API
|
||||
// 发送到 API(写 MySQL 或 API 直写 ClickHouse 时需要)
|
||||
if !needReportAPI {
|
||||
return nil
|
||||
}
|
||||
if this.rpcClient == nil {
|
||||
client, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/accesslogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
@@ -422,6 +423,9 @@ func (this *Node) syncConfig(taskVersion int64) error {
|
||||
|
||||
this.isLoaded = true
|
||||
|
||||
// 预创建本地日志目录与空文件,便于 Fluent Bit 立即 tail,无需等首条访问日志
|
||||
_ = accesslogs.SharedFileWriter().EnsureInit()
|
||||
|
||||
// 整体更新不需要再更新单个服务
|
||||
this.updatingServerMap = map[int64]*serverconfigs.ServerConfig{}
|
||||
|
||||
@@ -569,9 +573,16 @@ func (this *Node) checkClusterConfig() error {
|
||||
// 监听一些信号
|
||||
func (this *Node) listenSignals() {
|
||||
var queue = make(chan os.Signal, 8)
|
||||
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT)
|
||||
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
goman.New(func() {
|
||||
for range queue {
|
||||
for sig := range queue {
|
||||
if sig == syscall.SIGHUP {
|
||||
// 供 logrotate 等旋转日志后重开句柄
|
||||
if err := accesslogs.SharedFileWriter().Reopen(); err != nil {
|
||||
remotelogs.Error("NODE", "access log file reopen: "+err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
utils.Exit()
|
||||
return
|
||||
|
||||
@@ -290,14 +290,6 @@ var AllCheckpoints = []*CheckpointDefinition{
|
||||
Instance: new(RequestISPNameCheckpoint),
|
||||
Priority: 90,
|
||||
},
|
||||
{
|
||||
Name: "CC统计(旧)",
|
||||
Prefix: "cc",
|
||||
Description: "统计某段时间段内的请求信息",
|
||||
HasParams: true,
|
||||
Instance: new(CCCheckpoint),
|
||||
Priority: 10,
|
||||
},
|
||||
{
|
||||
Name: "CC统计(新)",
|
||||
Prefix: "cc2",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5" //1.3.8.2
|
||||
Version = "1.4.6" //1.3.8.2
|
||||
|
||||
ProductName = "Edge User"
|
||||
ProcessName = "edge-user"
|
||||
|
||||
BIN
default.etcd/.DS_Store
vendored
Normal file
BIN
default.etcd/.DS_Store
vendored
Normal file
Binary file not shown.
BIN
default.etcd/member/.DS_Store
vendored
Normal file
BIN
default.etcd/member/.DS_Store
vendored
Normal file
Binary file not shown.
BIN
default.etcd/member/snap/db
Normal file
BIN
default.etcd/member/snap/db
Normal file
Binary file not shown.
BIN
default.etcd/member/wal/0.tmp
Normal file
BIN
default.etcd/member/wal/0.tmp
Normal file
Binary file not shown.
BIN
default.etcd/member/wal/0000000000000000-0000000000000000.wal
Normal file
BIN
default.etcd/member/wal/0000000000000000-0000000000000000.wal
Normal file
Binary file not shown.
308
deploy/fluent-bit/README.md
Normal file
308
deploy/fluent-bit/README.md
Normal file
@@ -0,0 +1,308 @@
|
||||
# 边缘节点日志链路部署(Fluent Bit + ClickHouse)
|
||||
|
||||
与 [日志链路调整方案](../../log-pipeline-migration-plan.md) 配套的配置与部署说明。本文档为 **Fluent Bit 部署教程**,按步骤即可在边缘节点或日志采集机上跑通采集 → ClickHouse 写入。
|
||||
|
||||
---
|
||||
|
||||
## Fluent Bit 跑在哪台机器上?
|
||||
|
||||
**Fluent Bit 应部署在每台 EdgeNode 机器上**(与 edge-node 同机),不要部署在 EdgeAPI 机器上。
|
||||
|
||||
- 日志文件(`/var/log/edge/edge-node/*.log`)是 **EdgeNode** 在本机写的,只有 EdgeNode 所在机器才有这些文件。
|
||||
- Fluent Bit 使用 **tail** 插件读取本机路径,因此必须运行在 **有这些日志文件的机器** 上,即每台 EdgeNode。
|
||||
- EdgeAPI 机器上没有边缘节点日志,只负责查询 ClickHouse/MySQL,因此不需要在 EdgeAPI 上跑 Fluent Bit。
|
||||
- **多台 EdgeNode 时**:每台 EdgeNode 各跑一份 Fluent Bit,各自采集本机日志并上报到同一 ClickHouse。
|
||||
|
||||
---
|
||||
|
||||
## 一、前置条件
|
||||
|
||||
- **边缘节点(EdgeNode)** 已开启本地日志落盘,日志目录默认 `/var/log/edge/edge-node`,会生成 `access.log`、`waf.log`、`error.log`(JSON Lines)。由环境变量 `EDGE_LOG_DIR` 控制路径。
|
||||
- **ClickHouse** 已安装并可访问(单机或集群),且已创建好 `logs_ingest` 表(见下文「五、ClickHouse 建表」)。
|
||||
- 若 Fluent Bit 与 ClickHouse 不在同一台机,需保证网络可达(默认 HTTP 端口 8123)。
|
||||
|
||||
---
|
||||
|
||||
## 二、安装 Fluent Bit
|
||||
|
||||
### 2.1 Ubuntu / Debian
|
||||
|
||||
```bash
|
||||
# 添加 Fluent Bit 官方源并安装(以 Ubuntu 22.04 为例)
|
||||
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
|
||||
sudo apt-get install -y fluent-bit
|
||||
|
||||
# 或使用 TD Agent Bit 源(若需 ClickHouse 等扩展)
|
||||
# 见:https://docs.fluentbit.io/manual/installation/linux/ubuntu
|
||||
```
|
||||
|
||||
### 2.2 CentOS / RHEL / Amazon Linux
|
||||
|
||||
```bash
|
||||
# 使用官方 install 脚本
|
||||
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
|
||||
|
||||
# 或 yum/dnf 安装(以提供的仓库为准)
|
||||
# sudo yum install -y fluent-bit
|
||||
```
|
||||
|
||||
### 2.3 使用二进制包
|
||||
|
||||
从 [Fluent Bit 官方 Release](https://github.com/fluent/fluent-bit/releases) 下载对应架构的 tarball,解压后将 `bin/fluent-bit` 放到 PATH,并确保其 **Output 插件支持 ClickHouse**(部分发行版或自编译需启用 `out_clickhouse`)。
|
||||
|
||||
---
|
||||
|
||||
## 三、部署配置文件
|
||||
|
||||
### 3.1 放置配置
|
||||
|
||||
将本目录下配置文件放到同一目录,例如 `/etc/fluent-bit/` 或 `/opt/edge/fluent-bit/`:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /etc/fluent-bit
|
||||
sudo cp fluent-bit.conf clickhouse-upstream.conf /etc/fluent-bit/
|
||||
```
|
||||
|
||||
两文件需在同一目录,因 `fluent-bit.conf` 中有 `@INCLUDE clickhouse-upstream.conf`。
|
||||
|
||||
### 3.2 修改 ClickHouse 地址(必做)
|
||||
|
||||
编辑 `clickhouse-upstream.conf`,按实际环境填写 ClickHouse 的 Host/Port:
|
||||
|
||||
- **单机**:保留一个 `[NODE]`,改 `Host`、`Port`(默认 8123)。
|
||||
- **集群**:复制多段 `[NODE]`,每段一个节点,例如:
|
||||
|
||||
```ini
|
||||
[UPSTREAM]
|
||||
Name ch_backends
|
||||
|
||||
[NODE]
|
||||
Name node-01
|
||||
Host 192.168.1.10
|
||||
Port 8123
|
||||
|
||||
[NODE]
|
||||
Name node-02
|
||||
Host 192.168.1.11
|
||||
Port 8123
|
||||
```
|
||||
|
||||
### 3.3 ClickHouse 账号密码(有密码时必做)
|
||||
|
||||
不在 `clickhouse-upstream.conf` 里配置密码,而是通过 **环境变量** 传给 Fluent Bit:
|
||||
|
||||
- `CH_USER`:ClickHouse 用户名(如 `default`)。
|
||||
- `CH_PASSWORD`:对应用户的密码。
|
||||
|
||||
在 systemd 或启动脚本中设置(见下文「四、以 systemd 方式运行」)。
|
||||
|
||||
### 3.4 日志路径与 parsers.conf
|
||||
|
||||
- **日志路径**:`fluent-bit.conf` 里 INPUT 的 `Path` 已为 `/var/log/edge/edge-node/*.log`,与 EdgeNode 默认落盘路径一致;若你改了 `EDGE_LOG_DIR`,请同步改此处的 `Path`。
|
||||
- **Parsers_File**:主配置引用了 `parsers.conf`。若安装包自带(如 `/etc/fluent-bit/parsers.conf`),无需改动;若启动报错找不到文件,可:
|
||||
- 从 Fluent Bit 官方仓库复制 [conf/parsers.conf](https://github.com/fluent/fluent-bit/blob/master/conf/parsers.conf) 到同一目录,或
|
||||
- 在同一目录新建空文件 `parsers.conf`(仅当不使用任何 parser 时)。
|
||||
|
||||
### 3.5 数据与状态目录
|
||||
|
||||
Fluent Bit 会使用配置里的 `storage.path` 和 DB 路径,需保证进程有写权限:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /var/lib/fluent-bit/storage
|
||||
sudo chown -R <运行 fluent-bit 的用户>:<同组> /var/lib/fluent-bit
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 四、以 systemd 方式运行
|
||||
|
||||
### 4.1 使用自带服务(若安装包已提供)
|
||||
|
||||
若通过 apt/yum 安装,通常已有 `fluent-bit.service`。先改配置路径和环境变量:
|
||||
|
||||
```bash
|
||||
# 编辑服务文件(路径以实际为准,如 /lib/systemd/system/fluent-bit.service)
|
||||
sudo systemctl edit fluent-bit --full
|
||||
```
|
||||
|
||||
在 `[Service]` 中增加或修改:
|
||||
|
||||
- `EnvironmentFile` 指向你的环境变量文件,或直接写:
|
||||
- `Environment="CH_USER=default"`
|
||||
- `Environment="CH_PASSWORD=你的密码"`
|
||||
- `ExecStart` 中的配置文件路径改为你的 `fluent-bit.conf`,例如:
|
||||
- `ExecStart=/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf`
|
||||
|
||||
然后:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable fluent-bit
|
||||
sudo systemctl start fluent-bit
|
||||
sudo systemctl status fluent-bit
|
||||
```
|
||||
|
||||
### 4.2 自定义 systemd 单元(无自带服务时)
|
||||
|
||||
新建 `/etc/systemd/system/fluent-bit-edge.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Fluent Bit - Edge Node Logs to ClickHouse
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.conf
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
# ClickHouse 认证(按需修改)
|
||||
Environment="CH_USER=default"
|
||||
Environment="CH_PASSWORD=your_clickhouse_password"
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
若密码含特殊字符,建议用 `EnvironmentFile=/etc/fluent-bit/fluent-bit.env`,并在该文件中写:
|
||||
|
||||
```bash
|
||||
CH_USER=default
|
||||
CH_PASSWORD=your_clickhouse_password
|
||||
```
|
||||
|
||||
然后:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable fluent-bit-edge
|
||||
sudo systemctl start fluent-bit-edge
|
||||
sudo systemctl status fluent-bit-edge
|
||||
```
|
||||
|
||||
### 4.3 前台调试
|
||||
|
||||
不依赖 systemd 时可直接前台跑(便于看日志):
|
||||
|
||||
```bash
|
||||
export CH_USER=default
|
||||
export CH_PASSWORD=your_clickhouse_password
|
||||
fluent-bit -c /etc/fluent-bit/fluent-bit.conf
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 五、ClickHouse 建表
|
||||
|
||||
平台(EdgeAPI)查询的是表 `logs_ingest`,需在 ClickHouse 中先建表。库名默认为 `default`,若使用其它库,需与 EdgeAPI 的 `CLICKHOUSE_DATABASE` 一致。
|
||||
|
||||
在 ClickHouse 中执行(按需改库名或引擎):
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS default.logs_ingest
|
||||
(
|
||||
timestamp DateTime,
|
||||
node_id UInt64,
|
||||
cluster_id UInt64,
|
||||
server_id UInt64,
|
||||
host String,
|
||||
ip String,
|
||||
method String,
|
||||
path String,
|
||||
status UInt16,
|
||||
bytes_in UInt64,
|
||||
bytes_out UInt64,
|
||||
cost_ms UInt32,
|
||||
ua String,
|
||||
referer String,
|
||||
log_type String,
|
||||
trace_id String,
|
||||
firewall_policy_id UInt64 DEFAULT 0,
|
||||
firewall_rule_group_id UInt64 DEFAULT 0,
|
||||
firewall_rule_set_id UInt64 DEFAULT 0,
|
||||
firewall_rule_id UInt64 DEFAULT 0,
|
||||
request_headers String DEFAULT '',
|
||||
request_body String DEFAULT '',
|
||||
response_headers String DEFAULT '',
|
||||
response_body String DEFAULT ''
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (timestamp, node_id, server_id, trace_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
```
|
||||
|
||||
- **log_type**:`access` / `waf` / `error`;攻击日志同时看 **firewall_rule_id** 或 **firewall_policy_id** 是否大于 0(与原有 MySQL 通过规则 ID 判断攻击日志一致)。
|
||||
- **request_headers / response_headers**:JSON 字符串;**request_body / response_body**:请求/响应体(单条建议限制长度,如 512KB)。
|
||||
- **request_body 为空**:需在管理端为该站点/服务的「访问日志」策略中勾选「请求Body」后才会落盘;默认未勾选。路径大致为:站点/服务 → 访问日志 → 策略 → 记录字段 → 勾选「请求Body」。WAF 拦截且策略开启「记录请求Body」时也会记录。
|
||||
- **response_body 为空**:当前版本未实现(proto 与节点均未支持响应体落盘),表中已预留字段,后续可扩展。
|
||||
- **原有 MySQL 日志同步到 ClickHouse**:见 [mysql-to-clickhouse-migration.md](mysql-to-clickhouse-migration.md)。
|
||||
|
||||
若表已存在且缺少新字段,可执行:
|
||||
|
||||
```sql
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_policy_id UInt64 DEFAULT 0;
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_group_id UInt64 DEFAULT 0;
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_set_id UInt64 DEFAULT 0;
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS firewall_rule_id UInt64 DEFAULT 0;
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_headers String DEFAULT '';
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS request_body String DEFAULT '';
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_headers String DEFAULT '';
|
||||
ALTER TABLE default.logs_ingest ADD COLUMN IF NOT EXISTS response_body String DEFAULT '';
|
||||
```
|
||||
|
||||
Fluent Bit 写入时使用 `json_date_key timestamp` 和 `json_date_format epoch`,会将 JSON 中的 `timestamp`(Unix 秒)转为 DateTime。
|
||||
|
||||
---
|
||||
|
||||
## 六、验证与排错
|
||||
|
||||
1. **看 Fluent Bit 日志**
|
||||
- systemd:`journalctl -u fluent-bit-edge -f`(或你的服务名)
|
||||
- 前台:直接看终端输出。
|
||||
|
||||
2. **看 ClickHouse 是否有数据**
|
||||
```sql
|
||||
SELECT count() FROM default.logs_ingest;
|
||||
SELECT * FROM default.logs_ingest LIMIT 5;
|
||||
```
|
||||
|
||||
3. **常见问题**
|
||||
- **连接被拒**:检查 `clickhouse-upstream.conf` 的 Host/Port、防火墙、ClickHouse 的 `listen_host`。
|
||||
- **认证失败**:检查 `CH_USER`、`CH_PASSWORD` 是否与 ClickHouse 用户一致,环境变量是否被 systemd 正确加载。
|
||||
- **找不到 parsers.conf**:见上文 3.4。
|
||||
- **没有新数据**:确认 EdgeNode 已写日志到 `Path` 下,且 Fluent Bit 对该目录有读权限;可用 `tail -f /var/log/edge/edge-node/access.log` 观察是否有新行。
|
||||
- **Node 上没有 `/var/log/edge/edge-node/access.log`**:见下文「八、Node 上找不到日志文件」。
|
||||
|
||||
---
|
||||
|
||||
## 七、与其它组件的关系(简要)
|
||||
|
||||
| 组件 | 说明 |
|
||||
|------|------|
|
||||
| **EdgeNode** | 日志落盘路径由 `EDGE_LOG_DIR` 控制,默认 `/var/log/edge/edge-node`;生成 `access.log`、`waf.log`、`error.log`;支持 SIGHUP 重开句柄,可与 logrotate 的 `copytruncate` 配合。 |
|
||||
| **logrotate** | 使用 `deploy/fluent-bit/logrotate.conf` 示例做轮转,避免磁盘占满。 |
|
||||
| **平台(EdgeAPI)** | 配置 ClickHouse 只读连接(`CLICKHOUSE_HOST`、`CLICKHOUSE_PORT`、`CLICKHOUSE_USER`、`CLICKHOUSE_PASSWORD`、`CLICKHOUSE_DATABASE`);当请求带 `Day` 且已配置 ClickHouse 时,访问日志列表查询走 ClickHouse。 |
|
||||
|
||||
---
|
||||
|
||||
## 八、Node 上找不到日志文件
|
||||
|
||||
若在 EdgeNode 机器上执行 `tail -f /var/log/edge/edge-node/access.log` 报 **No such file or directory**,按下面检查:
|
||||
|
||||
1. **EdgeNode 版本**
|
||||
本地日志落盘是较新功能,需使用**包含该功能的 EdgeNode 构建**(当前仓库版本在首次加载配置时会预创建目录和三个空日志文件)。
|
||||
|
||||
2. **预创建目录(可选)**
|
||||
若进程以非 root 运行,可先手动建目录并赋权,避免无权限创建 `/var/log/edge`:
|
||||
```bash
|
||||
sudo mkdir -p /var/log/edge/edge-node
|
||||
sudo chown <运行 edge-node 的用户>:<同组> /var/log/edge/edge-node
|
||||
```
|
||||
|
||||
3. **重启 EdgeNode**
|
||||
新版本在**首次成功加载节点配置后**会调用 `EnsureInit()`,自动创建 `/var/log/edge/edge-node` 及 `access.log`、`waf.log`、`error.log`。重启一次 edge-node 后再看目录下是否已有文件。
|
||||
|
||||
4. **自定义路径**
|
||||
若通过环境变量 `EDGE_LOG_DIR` 指定了其它目录,则日志在该目录下;Fluent Bit 的 `Path` 需与之一致。
|
||||
|
||||
以上完成即完成 Fluent Bit 的部署与验证。
|
||||
11
deploy/fluent-bit/clickhouse-upstream.conf
Normal file
11
deploy/fluent-bit/clickhouse-upstream.conf
Normal file
@@ -0,0 +1,11 @@
|
||||
# ClickHouse 上游配置(单机或集群只改此文件)
|
||||
# 单机:保留一个 NODE;集群:按需增加 NODE
|
||||
# 有密码时:不在此文件配置,通过环境变量 CH_USER、CH_PASSWORD 在 fluent-bit.conf 的 OUTPUT 中生效。
|
||||
|
||||
[UPSTREAM]
|
||||
Name ch_backends
|
||||
|
||||
[NODE]
|
||||
Name node-01
|
||||
Host 127.0.0.1
|
||||
Port 8123
|
||||
34
deploy/fluent-bit/fluent-bit.conf
Normal file
34
deploy/fluent-bit/fluent-bit.conf
Normal file
@@ -0,0 +1,34 @@
|
||||
# Fluent Bit 主配置(边缘节点日志采集 → ClickHouse)
|
||||
# 生产环境将 INPUT 改为 tail 采集 /var/log/edge/edge-node/*.log
|
||||
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 128MB
|
||||
|
||||
@INCLUDE clickhouse-upstream.conf
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-node/*.log
|
||||
Tag app.logs
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name clickhouse
|
||||
Match *
|
||||
Upstream ch_backends
|
||||
Table logs_ingest
|
||||
Http_User ${CH_USER}
|
||||
Http_Passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
11
deploy/fluent-bit/logrotate.conf
Normal file
11
deploy/fluent-bit/logrotate.conf
Normal file
@@ -0,0 +1,11 @@
|
||||
# logrotate 示例:边缘节点日志轮转
|
||||
# 安装:放入 /etc/logrotate.d/edge-node 或 include 到主配置
|
||||
|
||||
/var/log/edge/edge-node/*.log {
|
||||
daily
|
||||
rotate 14
|
||||
compress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
}
|
||||
9
deploy/fluent-bit/parsers.conf
Normal file
9
deploy/fluent-bit/parsers.conf
Normal file
@@ -0,0 +1,9 @@
|
||||
# Fluent Bit 解析器(与主配置 Parsers_File 对应)
|
||||
# 边缘节点日志为 JSON Lines,使用 json 解析器即可
|
||||
|
||||
[PARSER]
|
||||
Name json
|
||||
Format json
|
||||
Time_Key timestamp
|
||||
Time_Format %s
|
||||
Time_Keep On
|
||||
Reference in New Issue
Block a user