主分支代码

This commit is contained in:
robin
2026-02-07 20:30:31 +08:00
parent 3b042d1dad
commit bc223fd1aa
65 changed files with 1969 additions and 188 deletions

View File

@@ -24,6 +24,7 @@ type StorageManager struct {
publicPolicyId int64
disableDefaultDB bool
writeTargets *serverconfigs.AccessLogWriteTargets // 公用策略的写入目标
locker sync.Mutex
}
@@ -79,12 +80,14 @@ func (this *StorageManager) Loop() error {
if int64(policy.Id) == publicPolicyId {
this.disableDefaultDB = policy.DisableDefaultDB
this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB)
foundPolicy = true
}
}
}
if !foundPolicy {
this.disableDefaultDB = false
this.writeTargets = nil
}
this.locker.Lock()
@@ -160,6 +163,27 @@ func (this *StorageManager) DisableDefaultDB() bool {
return this.disableDefaultDB
}
// WriteMySQL 公用策略是否写入 MySQL以 writeTargets 为准,无则用 disableDefaultDB
func (this *StorageManager) WriteMySQL() bool {
if this.writeTargets != nil {
return this.writeTargets.MySQL
}
return !this.disableDefaultDB
}
// WriteClickHouse 公用策略是否写入 ClickHouse文件+Fluent Bit 或后续 API 直写)
func (this *StorageManager) WriteClickHouse() bool {
if this.writeTargets != nil {
return this.writeTargets.ClickHouse
}
return false
}
// WriteTargets 返回公用策略的写入目标(供节点配置注入等)
func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets {
return this.writeTargets
}
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
switch storageType {
case serverconfigs.AccessLogStorageTypeFile:

View File

@@ -0,0 +1,134 @@
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// Client 通过 HTTP 接口执行只读查询SELECT返回 JSONEachRow 解析为 map 或结构体
type Client struct {
cfg *Config
httpCli *http.Client
}
// NewClient 使用共享配置创建客户端
func NewClient() *Client {
cfg := SharedConfig()
return &Client{
cfg: cfg,
httpCli: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// IsConfigured 是否已配置
func (c *Client) IsConfigured() bool {
return c.cfg != nil && c.cfg.IsConfigured()
}
// Query 执行 SELECT将每行 JSON 解析到 dest 切片dest 元素类型需为 *struct 或 map
func (c *Client) Query(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
// 强制 JSONEachRow 便于解析
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeRows(dec, dest)
}
// QueryRow 执行仅返回一行的查询,将结果解析到 dest*struct 或 *map
func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeOneRow(dec, dest)
}
func (c *Client) buildURL(query string) string {
rawURL := fmt.Sprintf("http://%s:%d/?query=%s&database=%s",
c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
return rawURL
}
// decodeRows 将 JSONEachRow 流解析到 slice元素类型须为 *struct 或 *map[string]interface{}
func decodeRows(dec *json.Decoder, dest interface{}) error {
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
switch d := dest.(type) {
case *[]map[string]interface{}:
*d = (*d)[:0]
for {
var row map[string]interface{}
if err := dec.Decode(&row); err != nil {
if err == io.EOF {
return nil
}
return err
}
*d = append(*d, row)
}
default:
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{} or implement decoder)")
}
}
func decodeOneRow(dec *json.Decoder, dest interface{}) error {
switch d := dest.(type) {
case *map[string]interface{}:
if err := dec.Decode(d); err != nil {
return err
}
return nil
default:
return fmt.Errorf("clickhouse: unsupported dest type for QueryRow (use *map[string]interface{})")
}
}

View File

@@ -0,0 +1,111 @@
// Package clickhouse 提供 ClickHouse 只读客户端,用于查询 logs_ingestFluent Bit 写入)。
// 配置优先从后台页面edgeSysSettings.clickhouseConfig读取其次 api.yaml最后环境变量。
package clickhouse
import (
"github.com/TeaOSLab/EdgeAPI/internal/configs"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"os"
"strconv"
"sync"
)
const (
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
defaultPort = 8123
defaultDB = "default"
)
var (
sharedConfig *Config
configOnce sync.Once
configLocker sync.Mutex
)
// Config ClickHouse 连接配置(仅查询,不从代码写库)
type Config struct {
Host string
Port int
User string
Password string
Database string
}
// SharedConfig 返回全局配置(优先从后台 DB 读取,其次 api.yaml最后环境变量
func SharedConfig() *Config {
configLocker.Lock()
defer configLocker.Unlock()
if sharedConfig != nil {
return sharedConfig
}
sharedConfig = loadConfig()
return sharedConfig
}
// ResetSharedConfig 清空缓存,下次 SharedConfig() 时重新从 DB/文件/环境变量加载(后台保存 ClickHouse 配置后调用)
func ResetSharedConfig() {
configLocker.Lock()
defer configLocker.Unlock()
sharedConfig = nil
}
func loadConfig() *Config {
cfg := &Config{Port: defaultPort, Database: defaultDB}
// 1) 优先从后台页面配置DB读取
if models.SharedSysSettingDAO != nil {
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
cfg.Host = dbCfg.Host
cfg.Port = dbCfg.Port
cfg.User = dbCfg.User
cfg.Password = dbCfg.Password
cfg.Database = dbCfg.Database
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
if cfg.Database == "" {
cfg.Database = defaultDB
}
return cfg
}
}
// 2) 其次 api.yaml
apiConfig, err := configs.SharedAPIConfig()
if err == nil && apiConfig != nil && apiConfig.ClickHouse != nil && apiConfig.ClickHouse.Host != "" {
ch := apiConfig.ClickHouse
cfg.Host = ch.Host
cfg.Port = ch.Port
cfg.User = ch.User
cfg.Password = ch.Password
cfg.Database = ch.Database
if cfg.Port <= 0 {
cfg.Port = defaultPort
}
if cfg.Database == "" {
cfg.Database = defaultDB
}
return cfg
}
// 3) 最后环境变量
cfg.Host = os.Getenv(envHost)
cfg.User = os.Getenv(envUser)
cfg.Password = os.Getenv(envPassword)
cfg.Database = os.Getenv(envDatabase)
if cfg.Database == "" {
cfg.Database = defaultDB
}
if p := os.Getenv(envPort); p != "" {
if v, err := strconv.Atoi(p); err == nil {
cfg.Port = v
}
}
return cfg
}
// IsConfigured 是否已配置Host 非空即视为启用 ClickHouse 查询)
func (c *Config) IsConfigured() bool {
return c != nil && c.Host != ""
}

View File

@@ -0,0 +1,285 @@
// Package clickhouse 提供 logs_ingest 表的只读查询(列表分页),用于访问日志列表优先走 ClickHouse。
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
// LogsIngestRow 对应 ClickHouse logs_ingest 表的一行(用于 List 结果与 RowToPB
type LogsIngestRow struct {
Timestamp time.Time
NodeId uint64
ClusterId uint64
ServerId uint64
Host string
IP string
Method string
Path string
Status uint16
BytesIn uint64
BytesOut uint64
CostMs uint32
UA string
Referer string
LogType string
TraceId string
FirewallPolicyId uint64
FirewallRuleGroupId uint64
FirewallRuleSetId uint64
FirewallRuleId uint64
RequestHeaders string
RequestBody string
ResponseHeaders string
ResponseBody string
}
// ListFilter 列表查询条件(与 ListHTTPAccessLogsRequest 对齐)
type ListFilter struct {
Day string
HourFrom string
HourTo string
Size int64
Reverse bool
HasError bool
HasFirewallPolicy bool
FirewallPolicyId int64
NodeId int64
ClusterId int64
LastRequestId string
ServerIds []int64
NodeIds []int64
}
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
type LogsIngestStore struct {
client *Client
}
// NewLogsIngestStore 创建 store内部使用共享 Client
func NewLogsIngestStore() *LogsIngestStore {
return &LogsIngestStore{client: NewClient()}
}
// Client 返回底层 Client供调用方判断 IsConfigured()
func (s *LogsIngestStore) Client() *Client {
return s.client
}
// List 按条件分页查询 logs_ingest返回行、下一页游标trace_id与错误
func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsIngestRow, nextCursor string, err error) {
if !s.client.IsConfigured() {
return nil, "", fmt.Errorf("clickhouse: not configured")
}
if f.Day == "" {
return nil, "", fmt.Errorf("clickhouse: day required")
}
table := "logs_ingest"
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest")
} else {
table = quoteIdent(table)
}
conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"}
if f.HourFrom != "" {
if _, err := strconv.Atoi(f.HourFrom); err == nil {
conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom)
}
}
if f.HourTo != "" {
if _, err := strconv.Atoi(f.HourTo); err == nil {
conditions = append(conditions, "toHour(timestamp) <= "+f.HourTo)
}
}
if len(f.ServerIds) > 0 {
parts := make([]string, 0, len(f.ServerIds))
for _, id := range f.ServerIds {
parts = append(parts, strconv.FormatInt(id, 10))
}
conditions = append(conditions, "server_id IN ("+strings.Join(parts, ",")+")")
}
if len(f.NodeIds) > 0 {
parts := make([]string, 0, len(f.NodeIds))
for _, id := range f.NodeIds {
parts = append(parts, strconv.FormatInt(id, 10))
}
conditions = append(conditions, "node_id IN ("+strings.Join(parts, ",")+")")
}
if f.NodeId > 0 {
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NodeId, 10))
}
if f.ClusterId > 0 {
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.ClusterId, 10))
}
if f.HasFirewallPolicy {
conditions = append(conditions, "firewall_policy_id > 0")
}
if f.FirewallPolicyId > 0 {
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
}
where := strings.Join(conditions, " AND ")
orderDir := "ASC"
if f.Reverse {
orderDir = "DESC"
}
limit := f.Size
if limit <= 0 {
limit = 20
}
if limit > 1000 {
limit = 1000
}
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
table, where, orderBy, limit+1)
var rawRows []map[string]interface{}
if err = s.client.Query(ctx, query, &rawRows); err != nil {
return nil, "", err
}
rows = make([]*LogsIngestRow, 0, len(rawRows))
for _, m := range rawRows {
r := mapToLogsIngestRow(m)
if r != nil {
rows = append(rows, r)
}
}
if len(rows) > int(limit) {
nextCursor = rows[limit].TraceId
rows = rows[:limit]
}
return rows, nextCursor, nil
}
func quoteIdent(name string) string {
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
}
func escapeString(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
r := &LogsIngestRow{}
u64 := func(key string) uint64 {
v, ok := m[key]
if !ok || v == nil {
return 0
}
switch x := v.(type) {
case float64:
return uint64(x)
case string:
n, _ := strconv.ParseUint(x, 10, 64)
return n
case json.Number:
n, _ := x.Int64()
return uint64(n)
}
return 0
}
u32 := func(key string) uint32 {
return uint32(u64(key))
}
str := func(key string) string {
if v, ok := m[key]; ok && v != nil {
if s, ok := v.(string); ok {
return s
}
}
return ""
}
ts := func(key string) time.Time {
v, ok := m[key]
if ok && v != nil {
switch x := v.(type) {
case string:
t, _ := time.Parse("2006-01-02 15:04:05", x)
return t
case float64:
return time.Unix(int64(x), 0)
case json.Number:
n, _ := x.Int64()
return time.Unix(n, 0)
}
}
return time.Time{}
}
r.Timestamp = ts("timestamp")
r.NodeId = u64("node_id")
r.ClusterId = u64("cluster_id")
r.ServerId = u64("server_id")
r.Host = str("host")
r.IP = str("ip")
r.Method = str("method")
r.Path = str("path")
r.Status = uint16(u64("status"))
r.BytesIn = u64("bytes_in")
r.BytesOut = u64("bytes_out")
r.CostMs = u32("cost_ms")
r.UA = str("ua")
r.Referer = str("referer")
r.LogType = str("log_type")
r.TraceId = str("trace_id")
r.FirewallPolicyId = u64("firewall_policy_id")
r.FirewallRuleGroupId = u64("firewall_rule_group_id")
r.FirewallRuleSetId = u64("firewall_rule_set_id")
r.FirewallRuleId = u64("firewall_rule_id")
r.RequestHeaders = str("request_headers")
r.RequestBody = str("request_body")
r.ResponseHeaders = str("response_headers")
r.ResponseBody = str("response_body")
return r
}
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog列表展示用
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
if r == nil {
return nil
}
a := &pb.HTTPAccessLog{
RequestId: r.TraceId,
ServerId: int64(r.ServerId),
NodeId: int64(r.NodeId),
Timestamp: r.Timestamp.Unix(),
Host: r.Host,
RawRemoteAddr: r.IP,
RemoteAddr: r.IP,
RequestMethod: r.Method,
RequestPath: r.Path,
Status: int32(r.Status),
RequestLength: int64(r.BytesIn),
BytesSent: int64(r.BytesOut),
RequestTime: float64(r.CostMs) / 1000,
UserAgent: r.UA,
Referer: r.Referer,
FirewallPolicyId: int64(r.FirewallPolicyId),
FirewallRuleGroupId: int64(r.FirewallRuleGroupId),
FirewallRuleSetId: int64(r.FirewallRuleSetId),
FirewallRuleId: int64(r.FirewallRuleId),
}
if r.TimeISO8601() != "" {
a.TimeISO8601 = r.TimeISO8601()
}
return a
}
// TimeISO8601 便于 RowToPB 使用
func (r *LogsIngestRow) TimeISO8601() string {
if r.Timestamp.IsZero() {
return ""
}
return r.Timestamp.UTC().Format("2006-01-02T15:04:05Z07:00")
}

View File

@@ -10,10 +10,20 @@ import (
var sharedAPIConfig *APIConfig = nil
// ClickHouseConfig 仅用于访问日志列表只读查询logs_ingest
type ClickHouseConfig struct {
Host string `yaml:"host" json:"host"`
Port int `yaml:"port" json:"port"`
User string `yaml:"user" json:"user"`
Password string `yaml:"password" json:"password"`
Database string `yaml:"database" json:"database"`
}
// APIConfig API节点配置
type APIConfig struct {
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
ClickHouse *ClickHouseConfig `yaml:"clickhouse,omitempty" json:"clickhouse,omitempty"`
numberId int64 // 数字ID
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "1.4.5" //1.3.9
Version = "1.4.6" //1.3.9
ProductName = "Edge API"
ProcessName = "edge-api"
@@ -17,5 +17,5 @@ const (
// 其他节点版本号,用来检测是否有需要升级的节点
NodeVersion = "1.4.5" //1.3.8.2
NodeVersion = "1.4.6" //1.3.8.2
)

View File

@@ -4,8 +4,8 @@
package teaconst
const (
DNSNodeVersion = "1.4.5" //1.3.8.2
UserNodeVersion = "1.4.5" //1.3.8.2
DNSNodeVersion = "1.4.6" //1.3.8.2
UserNodeVersion = "1.4.6" //1.3.8.2
ReportNodeVersion = "0.1.5"
DefaultMaxNodes int32 = 50

View File

@@ -107,7 +107,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
}
// CreatePolicy 创建策略
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool) (policyId int64, err error) {
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) {
var op = NewHTTPAccessLogPolicyOperator()
op.Name = name
op.Type = policyType
@@ -121,12 +121,15 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
op.IsOn = true
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.State = HTTPAccessLogPolicyStateEnabled
return this.SaveInt64(tx, op)
}
// UpdatePolicy 修改策略
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, isOn bool) error {
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -144,6 +147,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
var op = NewHTTPAccessLogPolicyOperator()
op.Id = policyId
op.Name = name
if policyType != "" {
op.Type = policyType
}
if len(optionsJSON) > 0 {
op.Options = optionsJSON
} else {
@@ -161,6 +167,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
op.IsPublic = isPublic
op.FirewallOnly = firewallOnly
op.DisableDefaultDB = disableDefaultDB
if len(writeTargetsJSON) > 0 {
op.WriteTargets = writeTargetsJSON
}
op.IsOn = isOn
return this.Save(tx, op)
}

View File

@@ -18,6 +18,7 @@ const (
HTTPAccessLogPolicyField_FirewallOnly dbs.FieldName = "firewallOnly" // 是否只记录防火墙相关
HTTPAccessLogPolicyField_Version dbs.FieldName = "version" // 版本号
HTTPAccessLogPolicyField_DisableDefaultDB dbs.FieldName = "disableDefaultDB" // 是否停止默认数据库存储
HTTPAccessLogPolicyField_WriteTargets dbs.FieldName = "writeTargets" // 写入目标 JSONfile/mysql/clickhouse
)
// HTTPAccessLogPolicy 访问日志策略
@@ -37,6 +38,7 @@ type HTTPAccessLogPolicy struct {
FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关
Version uint32 `field:"version"` // 版本号
DisableDefaultDB bool `field:"disableDefaultDB"` // 是否停止默认数据库存储
WriteTargets dbs.JSON `field:"writeTargets"` // 写入目标 JSON{"file":true,"mysql":true,"clickhouse":false}
}
type HTTPAccessLogPolicyOperator struct {
@@ -55,6 +57,7 @@ type HTTPAccessLogPolicyOperator struct {
FirewallOnly any // 是否只记录防火墙相关
Version any // 版本号
DisableDefaultDB any // 是否停止默认数据库存储
WriteTargets any // 写入目标 JSON
}
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

View File

@@ -1168,6 +1168,16 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
if config.GlobalServerConfig == nil {
config.GlobalServerConfig = nodeCluster.DecodeGlobalServerConfig()
}
// 注入公用访问日志策略的写入目标(供节点决定是否写文件、是否上报 API
if config.GlobalServerConfig != nil && clusterIndex == 0 {
publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx)
if publicPolicyId > 0 {
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
if publicPolicy != nil {
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
}
}
}
// 最大线程数、TCP连接数
if clusterIndex == 0 {
@@ -1972,6 +1982,25 @@ func (this *NodeDAO) FindEnabledNodesWithIds(tx *dbs.Tx, nodeIds []int64) (resul
return
}
// FindEnabledBasicNodesWithIds 根据一组ID查找节点基本信息id, name, clusterId用于批量填充日志列表的 Node/Cluster
func (this *NodeDAO) FindEnabledBasicNodesWithIds(tx *dbs.Tx, nodeIds []int64) (result []*Node, err error) {
if len(nodeIds) == 0 {
return nil, nil
}
idStrings := []string{}
for _, nodeId := range nodeIds {
idStrings = append(idStrings, numberutils.FormatInt64(nodeId))
}
_, err = this.Query(tx).
State(NodeStateEnabled).
Where("id IN ("+strings.Join(idStrings, ", ")+")").
Result("id", "name", "clusterId").
Slice(&result).
Reuse(false).
FindAll()
return
}
// DeleteNodeFromCluster 从集群中删除节点
func (this *NodeDAO) DeleteNodeFromCluster(tx *dbs.Tx, nodeId int64, clusterId int64) error {
one, err := this.Query(tx).

View File

@@ -65,3 +65,26 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
}
return config, nil
}
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
if err != nil {
return nil, err
}
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
if len(valueJSON) == 0 {
return out, nil
}
err = json.Unmarshal(valueJSON, out)
if err != nil {
return nil, err
}
if out.Port <= 0 {
out.Port = 8123
}
if out.Database == "" {
out.Database = "default"
}
return out, nil
}

View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
@@ -46,9 +47,8 @@ func (this *HTTPAccessLogService) CreateHTTPAccessLogs(ctx context.Context, req
return &pb.CreateHTTPAccessLogsResponse{}, nil
}
// ListHTTPAccessLogs 列出单页访问日志
// ListHTTPAccessLogs 列出单页访问日志(优先 ClickHouse否则 MySQLClickHouse 路径下节点/集群批量查询避免 N+1
func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *pb.ListHTTPAccessLogsRequest) (*pb.ListHTTPAccessLogsResponse, error) {
// 校验请求
_, userId, err := this.ValidateAdminAndUser(ctx, true)
if err != nil {
return nil, err
@@ -56,11 +56,8 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
var tx = this.NullTx()
// 检查服务ID
if userId > 0 {
req.UserId = userId
// 这里不用担心serverId <= 0 的情况因为如果userId>0则只会查询当前用户下的服务不会产生安全问题
if req.ServerId > 0 {
err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId)
if err != nil {
@@ -69,6 +66,17 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
}
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() && req.Day != "" {
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
if listErr != nil {
return nil, listErr
}
if resp != nil {
return resp, nil
}
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil {
return nil, err
@@ -82,8 +90,6 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
if err != nil {
return nil, err
}
// 节点 & 集群
pbNode, ok := pbNodeMap[a.NodeId]
if ok {
a.Node = pbNode
@@ -94,42 +100,131 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
if node != nil {
pbNode = &pb.Node{Id: int64(node.Id), Name: node.Name}
var clusterId = int64(node.ClusterId)
pbCluster, ok := pbClusterMap[clusterId]
if ok {
pbNode.NodeCluster = pbCluster
} else {
if !ok {
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, clusterId)
if err != nil {
return nil, err
}
if cluster != nil {
pbCluster = &pb.NodeCluster{
Id: int64(cluster.Id),
Name: cluster.Name,
}
pbNode.NodeCluster = pbCluster
pbCluster = &pb.NodeCluster{Id: int64(cluster.Id), Name: cluster.Name}
pbClusterMap[clusterId] = pbCluster
}
}
if pbCluster != nil {
pbNode.NodeCluster = pbCluster
}
pbNodeMap[a.NodeId] = pbNode
a.Node = pbNode
}
}
result = append(result, a)
}
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result, // TODO 仅仅为了兼容当用户节点版本大于0.0.8时可以删除
AccessLogs: result,
HasMore: hasMore,
RequestId: requestId,
}, nil
}
// listHTTPAccessLogsFromClickHouse 从 ClickHouse logs_ingest 查列表,并批量填充 Node/NodeCluster避免 N+1
func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.LogsIngestStore, req *pb.ListHTTPAccessLogsRequest, userId int64) (*pb.ListHTTPAccessLogsResponse, error) {
f := clickhouse.ListFilter{
Day: req.Day,
HourFrom: req.HourFrom,
HourTo: req.HourTo,
Size: req.Size,
Reverse: req.Reverse,
HasError: req.HasError,
HasFirewallPolicy: req.HasFirewallPolicy,
FirewallPolicyId: req.FirewallPolicyId,
NodeId: req.NodeId,
ClusterId: req.NodeClusterId,
LastRequestId: req.RequestId,
}
if req.ServerId > 0 {
f.ServerIds = []int64{req.ServerId}
} else if userId > 0 {
serverIds, err := models.SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
if err != nil {
return nil, err
}
if len(serverIds) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: nil, AccessLogs: nil, HasMore: false, RequestId: ""}, nil
}
f.ServerIds = serverIds
}
if req.NodeClusterId > 0 {
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, req.NodeClusterId)
if err != nil {
return nil, err
}
f.NodeIds = nodeIds
}
rows, nextCursor, err := store.List(ctx, f)
if err != nil {
return nil, err
}
if len(rows) == 0 {
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: []*pb.HTTPAccessLog{}, AccessLogs: []*pb.HTTPAccessLog{}, HasMore: false, RequestId: ""}, nil
}
result := make([]*pb.HTTPAccessLog, 0, len(rows))
nodeIdSet := make(map[int64]struct{})
for _, r := range rows {
result = append(result, clickhouse.RowToPB(r))
nodeIdSet[int64(r.NodeId)] = struct{}{}
}
nodeIds := make([]int64, 0, len(nodeIdSet))
for id := range nodeIdSet {
nodeIds = append(nodeIds, id)
}
nodes, err := models.SharedNodeDAO.FindEnabledBasicNodesWithIds(tx, nodeIds)
if err != nil {
return nil, err
}
clusterIds := make(map[int64]struct{})
for _, node := range nodes {
if node.ClusterId > 0 {
clusterIds[int64(node.ClusterId)] = struct{}{}
}
}
clusterIdList := make([]int64, 0, len(clusterIds))
for cid := range clusterIds {
clusterIdList = append(clusterIdList, cid)
}
clusters, _ := models.SharedNodeClusterDAO.FindEnabledNodeClustersWithIds(tx, clusterIdList)
clusterMap := make(map[int64]*pb.NodeCluster)
for _, c := range clusters {
clusterMap[int64(c.Id)] = &pb.NodeCluster{Id: int64(c.Id), Name: c.Name}
}
pbNodeMap := make(map[int64]*pb.Node)
for _, node := range nodes {
pbNode := &pb.Node{Id: int64(node.Id), Name: node.Name}
if c := clusterMap[int64(node.ClusterId)]; c != nil {
pbNode.NodeCluster = c
}
pbNodeMap[int64(node.Id)] = pbNode
}
for _, a := range result {
if n := pbNodeMap[a.NodeId]; n != nil {
a.Node = n
}
}
hasMore := nextCursor != ""
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result,
HasMore: hasMore,
RequestId: nextCursor,
}, nil
}
// FindHTTPAccessLog 查找单个日志
func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb.FindHTTPAccessLogRequest) (*pb.FindHTTPAccessLogResponse, error) {
// 校验请求

View File

@@ -11,7 +11,7 @@ import (
)
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return !accesslogs.SharedStorageManager.DisableDefaultDB()
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {

View File

@@ -53,6 +53,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
})
}
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
@@ -76,7 +77,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
}
// 创建
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB)
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
if err != nil {
return nil, err
}
@@ -101,7 +102,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
}
// 保存修改
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.IsOn)
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
if err != nil {
return nil, err
}
@@ -133,6 +134,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
IsPublic: policy.IsPublic,
FirewallOnly: policy.FirewallOnly == 1,
DisableDefaultDB: policy.DisableDefaultDB,
WriteTargetsJSON: policy.WriteTargets,
}}, nil
}

View File

@@ -2,6 +2,7 @@ package services
import (
"context"
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
@@ -26,6 +27,11 @@ func (this *SysSettingService) UpdateSysSetting(ctx context.Context, req *pb.Upd
return nil, err
}
// 若为 ClickHouse 配置,清空缓存使下次读取生效
if req.Code == "clickhouseConfig" {
clickhouse.ResetSharedConfig()
}
return this.Success()
}

View File

@@ -113017,7 +113017,7 @@
"name": "edgeHTTPAccessLogPolicies",
"engine": "InnoDB",
"charset": "utf8mb4_unicode_ci",
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n `writeTargets` json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
"fields": [
{
"name": "id",
@@ -113078,6 +113078,10 @@
{
"name": "disableDefaultDB",
"definition": "tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储'"
},
{
"name": "writeTargets",
"definition": "json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse'"
}
],
"indexes": [