主分支代码
This commit is contained in:
@@ -24,6 +24,7 @@ type StorageManager struct {
|
||||
|
||||
publicPolicyId int64
|
||||
disableDefaultDB bool
|
||||
writeTargets *serverconfigs.AccessLogWriteTargets // 公用策略的写入目标
|
||||
|
||||
locker sync.Mutex
|
||||
}
|
||||
@@ -79,12 +80,14 @@ func (this *StorageManager) Loop() error {
|
||||
|
||||
if int64(policy.Id) == publicPolicyId {
|
||||
this.disableDefaultDB = policy.DisableDefaultDB
|
||||
this.writeTargets = serverconfigs.ParseWriteTargetsFromPolicy(policy.WriteTargets, policy.Type, policy.DisableDefaultDB)
|
||||
foundPolicy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPolicy {
|
||||
this.disableDefaultDB = false
|
||||
this.writeTargets = nil
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
@@ -160,6 +163,27 @@ func (this *StorageManager) DisableDefaultDB() bool {
|
||||
return this.disableDefaultDB
|
||||
}
|
||||
|
||||
// WriteMySQL 公用策略是否写入 MySQL(以 writeTargets 为准,无则用 disableDefaultDB)
|
||||
func (this *StorageManager) WriteMySQL() bool {
|
||||
if this.writeTargets != nil {
|
||||
return this.writeTargets.MySQL
|
||||
}
|
||||
return !this.disableDefaultDB
|
||||
}
|
||||
|
||||
// WriteClickHouse 公用策略是否写入 ClickHouse(文件+Fluent Bit 或后续 API 直写)
|
||||
func (this *StorageManager) WriteClickHouse() bool {
|
||||
if this.writeTargets != nil {
|
||||
return this.writeTargets.ClickHouse
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// WriteTargets 返回公用策略的写入目标(供节点配置注入等)
|
||||
func (this *StorageManager) WriteTargets() *serverconfigs.AccessLogWriteTargets {
|
||||
return this.writeTargets
|
||||
}
|
||||
|
||||
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
|
||||
switch storageType {
|
||||
case serverconfigs.AccessLogStorageTypeFile:
|
||||
|
||||
134
EdgeAPI/internal/clickhouse/client.go
Normal file
134
EdgeAPI/internal/clickhouse/client.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client 通过 HTTP 接口执行只读查询(SELECT),返回 JSONEachRow 解析为 map 或结构体
|
||||
type Client struct {
|
||||
cfg *Config
|
||||
httpCli *http.Client
|
||||
}
|
||||
|
||||
// NewClient 使用共享配置创建客户端
|
||||
func NewClient() *Client {
|
||||
cfg := SharedConfig()
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
httpCli: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// IsConfigured 是否已配置
|
||||
func (c *Client) IsConfigured() bool {
|
||||
return c.cfg != nil && c.cfg.IsConfigured()
|
||||
}
|
||||
|
||||
// Query 执行 SELECT,将每行 JSON 解析到 dest 切片;dest 元素类型需为 *struct 或 map
|
||||
func (c *Client) Query(ctx context.Context, query string, dest interface{}) error {
|
||||
if !c.IsConfigured() {
|
||||
return fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
// 强制 JSONEachRow 便于解析
|
||||
q := strings.TrimSpace(query)
|
||||
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
|
||||
query = q + " FORMAT JSONEachRow"
|
||||
}
|
||||
u := c.buildURL(query)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.cfg.User != "" || c.cfg.Password != "" {
|
||||
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
|
||||
}
|
||||
resp, err := c.httpCli.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
return decodeRows(dec, dest)
|
||||
}
|
||||
|
||||
// QueryRow 执行仅返回一行的查询,将结果解析到 dest(*struct 或 *map)
|
||||
func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) error {
|
||||
if !c.IsConfigured() {
|
||||
return fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
q := strings.TrimSpace(query)
|
||||
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
|
||||
query = q + " FORMAT JSONEachRow"
|
||||
}
|
||||
u := c.buildURL(query)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.cfg.User != "" || c.cfg.Password != "" {
|
||||
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
|
||||
}
|
||||
resp, err := c.httpCli.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
return decodeOneRow(dec, dest)
|
||||
}
|
||||
|
||||
func (c *Client) buildURL(query string) string {
|
||||
rawURL := fmt.Sprintf("http://%s:%d/?query=%s&database=%s",
|
||||
c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
|
||||
return rawURL
|
||||
}
|
||||
|
||||
// decodeRows 将 JSONEachRow 流解析到 slice;元素类型须为 *struct 或 *map[string]interface{}
|
||||
func decodeRows(dec *json.Decoder, dest interface{}) error {
|
||||
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
|
||||
switch d := dest.(type) {
|
||||
case *[]map[string]interface{}:
|
||||
*d = (*d)[:0]
|
||||
for {
|
||||
var row map[string]interface{}
|
||||
if err := dec.Decode(&row); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
*d = append(*d, row)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{} or implement decoder)")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeOneRow(dec *json.Decoder, dest interface{}) error {
|
||||
switch d := dest.(type) {
|
||||
case *map[string]interface{}:
|
||||
if err := dec.Decode(d); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("clickhouse: unsupported dest type for QueryRow (use *map[string]interface{})")
|
||||
}
|
||||
}
|
||||
111
EdgeAPI/internal/clickhouse/config.go
Normal file
111
EdgeAPI/internal/clickhouse/config.go
Normal file
@@ -0,0 +1,111 @@
|
||||
// Package clickhouse 提供 ClickHouse 只读客户端,用于查询 logs_ingest(Fluent Bit 写入)。
|
||||
// 配置优先从后台页面(edgeSysSettings.clickhouseConfig)读取,其次 api.yaml,最后环境变量。
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
envHost = "CLICKHOUSE_HOST"
|
||||
envPort = "CLICKHOUSE_PORT"
|
||||
envUser = "CLICKHOUSE_USER"
|
||||
envPassword = "CLICKHOUSE_PASSWORD"
|
||||
envDatabase = "CLICKHOUSE_DATABASE"
|
||||
defaultPort = 8123
|
||||
defaultDB = "default"
|
||||
)
|
||||
|
||||
var (
|
||||
sharedConfig *Config
|
||||
configOnce sync.Once
|
||||
configLocker sync.Mutex
|
||||
)
|
||||
|
||||
// Config ClickHouse 连接配置(仅查询,不从代码写库)
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
}
|
||||
|
||||
// SharedConfig 返回全局配置(优先从后台 DB 读取,其次 api.yaml,最后环境变量)
|
||||
func SharedConfig() *Config {
|
||||
configLocker.Lock()
|
||||
defer configLocker.Unlock()
|
||||
if sharedConfig != nil {
|
||||
return sharedConfig
|
||||
}
|
||||
sharedConfig = loadConfig()
|
||||
return sharedConfig
|
||||
}
|
||||
|
||||
// ResetSharedConfig 清空缓存,下次 SharedConfig() 时重新从 DB/文件/环境变量加载(后台保存 ClickHouse 配置后调用)
|
||||
func ResetSharedConfig() {
|
||||
configLocker.Lock()
|
||||
defer configLocker.Unlock()
|
||||
sharedConfig = nil
|
||||
}
|
||||
|
||||
func loadConfig() *Config {
|
||||
cfg := &Config{Port: defaultPort, Database: defaultDB}
|
||||
// 1) 优先从后台页面配置(DB)读取
|
||||
if models.SharedSysSettingDAO != nil {
|
||||
if dbCfg, err := models.SharedSysSettingDAO.ReadClickHouseConfig(nil); err == nil && dbCfg != nil && dbCfg.Host != "" {
|
||||
cfg.Host = dbCfg.Host
|
||||
cfg.Port = dbCfg.Port
|
||||
cfg.User = dbCfg.User
|
||||
cfg.Password = dbCfg.Password
|
||||
cfg.Database = dbCfg.Database
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
}
|
||||
// 2) 其次 api.yaml
|
||||
apiConfig, err := configs.SharedAPIConfig()
|
||||
if err == nil && apiConfig != nil && apiConfig.ClickHouse != nil && apiConfig.ClickHouse.Host != "" {
|
||||
ch := apiConfig.ClickHouse
|
||||
cfg.Host = ch.Host
|
||||
cfg.Port = ch.Port
|
||||
cfg.User = ch.User
|
||||
cfg.Password = ch.Password
|
||||
cfg.Database = ch.Database
|
||||
if cfg.Port <= 0 {
|
||||
cfg.Port = defaultPort
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
// 3) 最后环境变量
|
||||
cfg.Host = os.Getenv(envHost)
|
||||
cfg.User = os.Getenv(envUser)
|
||||
cfg.Password = os.Getenv(envPassword)
|
||||
cfg.Database = os.Getenv(envDatabase)
|
||||
if cfg.Database == "" {
|
||||
cfg.Database = defaultDB
|
||||
}
|
||||
if p := os.Getenv(envPort); p != "" {
|
||||
if v, err := strconv.Atoi(p); err == nil {
|
||||
cfg.Port = v
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// IsConfigured 是否已配置(Host 非空即视为启用 ClickHouse 查询)
|
||||
func (c *Config) IsConfigured() bool {
|
||||
return c != nil && c.Host != ""
|
||||
}
|
||||
285
EdgeAPI/internal/clickhouse/logs_ingest_store.go
Normal file
285
EdgeAPI/internal/clickhouse/logs_ingest_store.go
Normal file
@@ -0,0 +1,285 @@
|
||||
// Package clickhouse 提供 logs_ingest 表的只读查询(列表分页),用于访问日志列表优先走 ClickHouse。
|
||||
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
|
||||
// LogsIngestRow 对应 ClickHouse logs_ingest 表的一行(用于 List 结果与 RowToPB)
|
||||
type LogsIngestRow struct {
|
||||
Timestamp time.Time
|
||||
NodeId uint64
|
||||
ClusterId uint64
|
||||
ServerId uint64
|
||||
Host string
|
||||
IP string
|
||||
Method string
|
||||
Path string
|
||||
Status uint16
|
||||
BytesIn uint64
|
||||
BytesOut uint64
|
||||
CostMs uint32
|
||||
UA string
|
||||
Referer string
|
||||
LogType string
|
||||
TraceId string
|
||||
FirewallPolicyId uint64
|
||||
FirewallRuleGroupId uint64
|
||||
FirewallRuleSetId uint64
|
||||
FirewallRuleId uint64
|
||||
RequestHeaders string
|
||||
RequestBody string
|
||||
ResponseHeaders string
|
||||
ResponseBody string
|
||||
}
|
||||
|
||||
// ListFilter 列表查询条件(与 ListHTTPAccessLogsRequest 对齐)
|
||||
type ListFilter struct {
|
||||
Day string
|
||||
HourFrom string
|
||||
HourTo string
|
||||
Size int64
|
||||
Reverse bool
|
||||
HasError bool
|
||||
HasFirewallPolicy bool
|
||||
FirewallPolicyId int64
|
||||
NodeId int64
|
||||
ClusterId int64
|
||||
LastRequestId string
|
||||
ServerIds []int64
|
||||
NodeIds []int64
|
||||
}
|
||||
|
||||
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
|
||||
type LogsIngestStore struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewLogsIngestStore 创建 store,内部使用共享 Client
|
||||
func NewLogsIngestStore() *LogsIngestStore {
|
||||
return &LogsIngestStore{client: NewClient()}
|
||||
}
|
||||
|
||||
// Client 返回底层 Client,供调用方判断 IsConfigured()
|
||||
func (s *LogsIngestStore) Client() *Client {
|
||||
return s.client
|
||||
}
|
||||
|
||||
// List 按条件分页查询 logs_ingest,返回行、下一页游标(trace_id)与错误
|
||||
func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsIngestRow, nextCursor string, err error) {
|
||||
if !s.client.IsConfigured() {
|
||||
return nil, "", fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
if f.Day == "" {
|
||||
return nil, "", fmt.Errorf("clickhouse: day required")
|
||||
}
|
||||
table := "logs_ingest"
|
||||
if s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
|
||||
table = quoteIdent(s.client.cfg.Database) + "." + quoteIdent("logs_ingest")
|
||||
} else {
|
||||
table = quoteIdent(table)
|
||||
}
|
||||
|
||||
conditions := []string{"toDate(timestamp) = '" + escapeString(f.Day) + "'"}
|
||||
if f.HourFrom != "" {
|
||||
if _, err := strconv.Atoi(f.HourFrom); err == nil {
|
||||
conditions = append(conditions, "toHour(timestamp) >= "+f.HourFrom)
|
||||
}
|
||||
}
|
||||
if f.HourTo != "" {
|
||||
if _, err := strconv.Atoi(f.HourTo); err == nil {
|
||||
conditions = append(conditions, "toHour(timestamp) <= "+f.HourTo)
|
||||
}
|
||||
}
|
||||
if len(f.ServerIds) > 0 {
|
||||
parts := make([]string, 0, len(f.ServerIds))
|
||||
for _, id := range f.ServerIds {
|
||||
parts = append(parts, strconv.FormatInt(id, 10))
|
||||
}
|
||||
conditions = append(conditions, "server_id IN ("+strings.Join(parts, ",")+")")
|
||||
}
|
||||
if len(f.NodeIds) > 0 {
|
||||
parts := make([]string, 0, len(f.NodeIds))
|
||||
for _, id := range f.NodeIds {
|
||||
parts = append(parts, strconv.FormatInt(id, 10))
|
||||
}
|
||||
conditions = append(conditions, "node_id IN ("+strings.Join(parts, ",")+")")
|
||||
}
|
||||
if f.NodeId > 0 {
|
||||
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NodeId, 10))
|
||||
}
|
||||
if f.ClusterId > 0 {
|
||||
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.ClusterId, 10))
|
||||
}
|
||||
if f.HasFirewallPolicy {
|
||||
conditions = append(conditions, "firewall_policy_id > 0")
|
||||
}
|
||||
if f.FirewallPolicyId > 0 {
|
||||
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
|
||||
}
|
||||
|
||||
where := strings.Join(conditions, " AND ")
|
||||
orderDir := "ASC"
|
||||
if f.Reverse {
|
||||
orderDir = "DESC"
|
||||
}
|
||||
limit := f.Size
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
|
||||
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||
table, where, orderBy, limit+1)
|
||||
|
||||
var rawRows []map[string]interface{}
|
||||
if err = s.client.Query(ctx, query, &rawRows); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
rows = make([]*LogsIngestRow, 0, len(rawRows))
|
||||
for _, m := range rawRows {
|
||||
r := mapToLogsIngestRow(m)
|
||||
if r != nil {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
if len(rows) > int(limit) {
|
||||
nextCursor = rows[limit].TraceId
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return rows, nextCursor, nil
|
||||
}
|
||||
|
||||
func quoteIdent(name string) string {
|
||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||
}
|
||||
|
||||
func escapeString(s string) string {
|
||||
return strings.ReplaceAll(s, "'", "''")
|
||||
}
|
||||
|
||||
func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
|
||||
r := &LogsIngestRow{}
|
||||
u64 := func(key string) uint64 {
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return uint64(x)
|
||||
case string:
|
||||
n, _ := strconv.ParseUint(x, 10, 64)
|
||||
return n
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return uint64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
u32 := func(key string) uint32 {
|
||||
return uint32(u64(key))
|
||||
}
|
||||
str := func(key string) string {
|
||||
if v, ok := m[key]; ok && v != nil {
|
||||
if s, ok := v.(string); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
ts := func(key string) time.Time {
|
||||
v, ok := m[key]
|
||||
if ok && v != nil {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
t, _ := time.Parse("2006-01-02 15:04:05", x)
|
||||
return t
|
||||
case float64:
|
||||
return time.Unix(int64(x), 0)
|
||||
case json.Number:
|
||||
n, _ := x.Int64()
|
||||
return time.Unix(n, 0)
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
r.Timestamp = ts("timestamp")
|
||||
r.NodeId = u64("node_id")
|
||||
r.ClusterId = u64("cluster_id")
|
||||
r.ServerId = u64("server_id")
|
||||
r.Host = str("host")
|
||||
r.IP = str("ip")
|
||||
r.Method = str("method")
|
||||
r.Path = str("path")
|
||||
r.Status = uint16(u64("status"))
|
||||
r.BytesIn = u64("bytes_in")
|
||||
r.BytesOut = u64("bytes_out")
|
||||
r.CostMs = u32("cost_ms")
|
||||
r.UA = str("ua")
|
||||
r.Referer = str("referer")
|
||||
r.LogType = str("log_type")
|
||||
r.TraceId = str("trace_id")
|
||||
r.FirewallPolicyId = u64("firewall_policy_id")
|
||||
r.FirewallRuleGroupId = u64("firewall_rule_group_id")
|
||||
r.FirewallRuleSetId = u64("firewall_rule_set_id")
|
||||
r.FirewallRuleId = u64("firewall_rule_id")
|
||||
r.RequestHeaders = str("request_headers")
|
||||
r.RequestBody = str("request_body")
|
||||
r.ResponseHeaders = str("response_headers")
|
||||
r.ResponseBody = str("response_body")
|
||||
return r
|
||||
}
|
||||
|
||||
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用)
|
||||
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
a := &pb.HTTPAccessLog{
|
||||
RequestId: r.TraceId,
|
||||
ServerId: int64(r.ServerId),
|
||||
NodeId: int64(r.NodeId),
|
||||
Timestamp: r.Timestamp.Unix(),
|
||||
Host: r.Host,
|
||||
RawRemoteAddr: r.IP,
|
||||
RemoteAddr: r.IP,
|
||||
RequestMethod: r.Method,
|
||||
RequestPath: r.Path,
|
||||
Status: int32(r.Status),
|
||||
RequestLength: int64(r.BytesIn),
|
||||
BytesSent: int64(r.BytesOut),
|
||||
RequestTime: float64(r.CostMs) / 1000,
|
||||
UserAgent: r.UA,
|
||||
Referer: r.Referer,
|
||||
FirewallPolicyId: int64(r.FirewallPolicyId),
|
||||
FirewallRuleGroupId: int64(r.FirewallRuleGroupId),
|
||||
FirewallRuleSetId: int64(r.FirewallRuleSetId),
|
||||
FirewallRuleId: int64(r.FirewallRuleId),
|
||||
}
|
||||
if r.TimeISO8601() != "" {
|
||||
a.TimeISO8601 = r.TimeISO8601()
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// TimeISO8601 便于 RowToPB 使用
|
||||
func (r *LogsIngestRow) TimeISO8601() string {
|
||||
if r.Timestamp.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return r.Timestamp.UTC().Format("2006-01-02T15:04:05Z07:00")
|
||||
}
|
||||
@@ -10,10 +10,20 @@ import (
|
||||
|
||||
var sharedAPIConfig *APIConfig = nil
|
||||
|
||||
// ClickHouseConfig 仅用于访问日志列表只读查询(logs_ingest)
|
||||
type ClickHouseConfig struct {
|
||||
Host string `yaml:"host" json:"host"`
|
||||
Port int `yaml:"port" json:"port"`
|
||||
User string `yaml:"user" json:"user"`
|
||||
Password string `yaml:"password" json:"password"`
|
||||
Database string `yaml:"database" json:"database"`
|
||||
}
|
||||
|
||||
// APIConfig API节点配置
|
||||
type APIConfig struct {
|
||||
NodeId string `yaml:"nodeId" json:"nodeId"`
|
||||
Secret string `yaml:"secret" json:"secret"`
|
||||
NodeId string `yaml:"nodeId" json:"nodeId"`
|
||||
Secret string `yaml:"secret" json:"secret"`
|
||||
ClickHouse *ClickHouseConfig `yaml:"clickhouse,omitempty" json:"clickhouse,omitempty"`
|
||||
|
||||
numberId int64 // 数字ID
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.5" //1.3.9
|
||||
Version = "1.4.6" //1.3.9
|
||||
|
||||
ProductName = "Edge API"
|
||||
ProcessName = "edge-api"
|
||||
@@ -17,5 +17,5 @@ const (
|
||||
|
||||
// 其他节点版本号,用来检测是否有需要升级的节点
|
||||
|
||||
NodeVersion = "1.4.5" //1.3.8.2
|
||||
NodeVersion = "1.4.6" //1.3.8.2
|
||||
)
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
DNSNodeVersion = "1.4.5" //1.3.8.2
|
||||
UserNodeVersion = "1.4.5" //1.3.8.2
|
||||
DNSNodeVersion = "1.4.6" //1.3.8.2
|
||||
UserNodeVersion = "1.4.6" //1.3.8.2
|
||||
ReportNodeVersion = "0.1.5"
|
||||
|
||||
DefaultMaxNodes int32 = 50
|
||||
|
||||
@@ -107,7 +107,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
|
||||
}
|
||||
|
||||
// CreatePolicy 创建策略
|
||||
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool) (policyId int64, err error) {
|
||||
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte) (policyId int64, err error) {
|
||||
var op = NewHTTPAccessLogPolicyOperator()
|
||||
op.Name = name
|
||||
op.Type = policyType
|
||||
@@ -121,12 +121,15 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
|
||||
op.IsOn = true
|
||||
op.FirewallOnly = firewallOnly
|
||||
op.DisableDefaultDB = disableDefaultDB
|
||||
if len(writeTargetsJSON) > 0 {
|
||||
op.WriteTargets = writeTargetsJSON
|
||||
}
|
||||
op.State = HTTPAccessLogPolicyStateEnabled
|
||||
return this.SaveInt64(tx, op)
|
||||
}
|
||||
|
||||
// UpdatePolicy 修改策略
|
||||
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, isOn bool) error {
|
||||
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, disableDefaultDB bool, writeTargetsJSON []byte, isOn bool) error {
|
||||
if policyId <= 0 {
|
||||
return errors.New("invalid policyId")
|
||||
}
|
||||
@@ -144,6 +147,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
|
||||
var op = NewHTTPAccessLogPolicyOperator()
|
||||
op.Id = policyId
|
||||
op.Name = name
|
||||
if policyType != "" {
|
||||
op.Type = policyType
|
||||
}
|
||||
if len(optionsJSON) > 0 {
|
||||
op.Options = optionsJSON
|
||||
} else {
|
||||
@@ -161,6 +167,9 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
|
||||
op.IsPublic = isPublic
|
||||
op.FirewallOnly = firewallOnly
|
||||
op.DisableDefaultDB = disableDefaultDB
|
||||
if len(writeTargetsJSON) > 0 {
|
||||
op.WriteTargets = writeTargetsJSON
|
||||
}
|
||||
op.IsOn = isOn
|
||||
return this.Save(tx, op)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
HTTPAccessLogPolicyField_FirewallOnly dbs.FieldName = "firewallOnly" // 是否只记录防火墙相关
|
||||
HTTPAccessLogPolicyField_Version dbs.FieldName = "version" // 版本号
|
||||
HTTPAccessLogPolicyField_DisableDefaultDB dbs.FieldName = "disableDefaultDB" // 是否停止默认数据库存储
|
||||
HTTPAccessLogPolicyField_WriteTargets dbs.FieldName = "writeTargets" // 写入目标 JSON:file/mysql/clickhouse
|
||||
)
|
||||
|
||||
// HTTPAccessLogPolicy 访问日志策略
|
||||
@@ -37,6 +38,7 @@ type HTTPAccessLogPolicy struct {
|
||||
FirewallOnly uint8 `field:"firewallOnly"` // 是否只记录防火墙相关
|
||||
Version uint32 `field:"version"` // 版本号
|
||||
DisableDefaultDB bool `field:"disableDefaultDB"` // 是否停止默认数据库存储
|
||||
WriteTargets dbs.JSON `field:"writeTargets"` // 写入目标 JSON:{"file":true,"mysql":true,"clickhouse":false}
|
||||
}
|
||||
|
||||
type HTTPAccessLogPolicyOperator struct {
|
||||
@@ -55,6 +57,7 @@ type HTTPAccessLogPolicyOperator struct {
|
||||
FirewallOnly any // 是否只记录防火墙相关
|
||||
Version any // 版本号
|
||||
DisableDefaultDB any // 是否停止默认数据库存储
|
||||
WriteTargets any // 写入目标 JSON
|
||||
}
|
||||
|
||||
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {
|
||||
|
||||
@@ -1168,6 +1168,16 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, dataMap *shared
|
||||
if config.GlobalServerConfig == nil {
|
||||
config.GlobalServerConfig = nodeCluster.DecodeGlobalServerConfig()
|
||||
}
|
||||
// 注入公用访问日志策略的写入目标(供节点决定是否写文件、是否上报 API)
|
||||
if config.GlobalServerConfig != nil && clusterIndex == 0 {
|
||||
publicPolicyId, _ := SharedHTTPAccessLogPolicyDAO.FindCurrentPublicPolicyId(tx)
|
||||
if publicPolicyId > 0 {
|
||||
publicPolicy, _ := SharedHTTPAccessLogPolicyDAO.FindEnabledHTTPAccessLogPolicy(tx, publicPolicyId)
|
||||
if publicPolicy != nil {
|
||||
config.GlobalServerConfig.HTTPAccessLog.WriteTargets = serverconfigs.ParseWriteTargetsFromPolicy(publicPolicy.WriteTargets, publicPolicy.Type, publicPolicy.DisableDefaultDB)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 最大线程数、TCP连接数
|
||||
if clusterIndex == 0 {
|
||||
@@ -1972,6 +1982,25 @@ func (this *NodeDAO) FindEnabledNodesWithIds(tx *dbs.Tx, nodeIds []int64) (resul
|
||||
return
|
||||
}
|
||||
|
||||
// FindEnabledBasicNodesWithIds 根据一组ID查找节点基本信息(id, name, clusterId),用于批量填充日志列表的 Node/Cluster
|
||||
func (this *NodeDAO) FindEnabledBasicNodesWithIds(tx *dbs.Tx, nodeIds []int64) (result []*Node, err error) {
|
||||
if len(nodeIds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
idStrings := []string{}
|
||||
for _, nodeId := range nodeIds {
|
||||
idStrings = append(idStrings, numberutils.FormatInt64(nodeId))
|
||||
}
|
||||
_, err = this.Query(tx).
|
||||
State(NodeStateEnabled).
|
||||
Where("id IN ("+strings.Join(idStrings, ", ")+")").
|
||||
Result("id", "name", "clusterId").
|
||||
Slice(&result).
|
||||
Reuse(false).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteNodeFromCluster 从集群中删除节点
|
||||
func (this *NodeDAO) DeleteNodeFromCluster(tx *dbs.Tx, nodeId int64, clusterId int64) error {
|
||||
one, err := this.Query(tx).
|
||||
|
||||
@@ -65,3 +65,26 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
|
||||
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
||||
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
|
||||
if len(valueJSON) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
err = json.Unmarshal(valueJSON, out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out.Port <= 0 {
|
||||
out.Port = 8123
|
||||
}
|
||||
if out.Database == "" {
|
||||
out.Database = "default"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/errors"
|
||||
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
|
||||
@@ -46,9 +47,8 @@ func (this *HTTPAccessLogService) CreateHTTPAccessLogs(ctx context.Context, req
|
||||
return &pb.CreateHTTPAccessLogsResponse{}, nil
|
||||
}
|
||||
|
||||
// ListHTTPAccessLogs 列出单页访问日志
|
||||
// ListHTTPAccessLogs 列出单页访问日志(优先 ClickHouse,否则 MySQL;ClickHouse 路径下节点/集群批量查询避免 N+1)
|
||||
func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *pb.ListHTTPAccessLogsRequest) (*pb.ListHTTPAccessLogsResponse, error) {
|
||||
// 校验请求
|
||||
_, userId, err := this.ValidateAdminAndUser(ctx, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -56,11 +56,8 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
// 检查服务ID
|
||||
if userId > 0 {
|
||||
req.UserId = userId
|
||||
|
||||
// 这里不用担心serverId <= 0 的情况,因为如果userId>0,则只会查询当前用户下的服务,不会产生安全问题
|
||||
if req.ServerId > 0 {
|
||||
err = models.SharedServerDAO.CheckUserServer(tx, userId, req.ServerId)
|
||||
if err != nil {
|
||||
@@ -69,6 +66,17 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
}
|
||||
}
|
||||
|
||||
store := clickhouse.NewLogsIngestStore()
|
||||
if store.Client().IsConfigured() && req.Day != "" {
|
||||
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
|
||||
if listErr != nil {
|
||||
return nil, listErr
|
||||
}
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -82,8 +90,6 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 节点 & 集群
|
||||
pbNode, ok := pbNodeMap[a.NodeId]
|
||||
if ok {
|
||||
a.Node = pbNode
|
||||
@@ -94,42 +100,131 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
|
||||
}
|
||||
if node != nil {
|
||||
pbNode = &pb.Node{Id: int64(node.Id), Name: node.Name}
|
||||
|
||||
var clusterId = int64(node.ClusterId)
|
||||
pbCluster, ok := pbClusterMap[clusterId]
|
||||
if ok {
|
||||
pbNode.NodeCluster = pbCluster
|
||||
} else {
|
||||
if !ok {
|
||||
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, clusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cluster != nil {
|
||||
pbCluster = &pb.NodeCluster{
|
||||
Id: int64(cluster.Id),
|
||||
Name: cluster.Name,
|
||||
}
|
||||
pbNode.NodeCluster = pbCluster
|
||||
pbCluster = &pb.NodeCluster{Id: int64(cluster.Id), Name: cluster.Name}
|
||||
pbClusterMap[clusterId] = pbCluster
|
||||
}
|
||||
}
|
||||
|
||||
if pbCluster != nil {
|
||||
pbNode.NodeCluster = pbCluster
|
||||
}
|
||||
pbNodeMap[a.NodeId] = pbNode
|
||||
a.Node = pbNode
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, a)
|
||||
}
|
||||
|
||||
return &pb.ListHTTPAccessLogsResponse{
|
||||
HttpAccessLogs: result,
|
||||
AccessLogs: result, // TODO 仅仅为了兼容,当用户节点版本大于0.0.8时可以删除
|
||||
AccessLogs: result,
|
||||
HasMore: hasMore,
|
||||
RequestId: requestId,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// listHTTPAccessLogsFromClickHouse 从 ClickHouse logs_ingest 查列表,并批量填充 Node/NodeCluster(避免 N+1)
|
||||
func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.LogsIngestStore, req *pb.ListHTTPAccessLogsRequest, userId int64) (*pb.ListHTTPAccessLogsResponse, error) {
|
||||
f := clickhouse.ListFilter{
|
||||
Day: req.Day,
|
||||
HourFrom: req.HourFrom,
|
||||
HourTo: req.HourTo,
|
||||
Size: req.Size,
|
||||
Reverse: req.Reverse,
|
||||
HasError: req.HasError,
|
||||
HasFirewallPolicy: req.HasFirewallPolicy,
|
||||
FirewallPolicyId: req.FirewallPolicyId,
|
||||
NodeId: req.NodeId,
|
||||
ClusterId: req.NodeClusterId,
|
||||
LastRequestId: req.RequestId,
|
||||
}
|
||||
if req.ServerId > 0 {
|
||||
f.ServerIds = []int64{req.ServerId}
|
||||
} else if userId > 0 {
|
||||
serverIds, err := models.SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(serverIds) == 0 {
|
||||
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: nil, AccessLogs: nil, HasMore: false, RequestId: ""}, nil
|
||||
}
|
||||
f.ServerIds = serverIds
|
||||
}
|
||||
if req.NodeClusterId > 0 {
|
||||
nodeIds, err := models.SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, req.NodeClusterId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.NodeIds = nodeIds
|
||||
}
|
||||
|
||||
rows, nextCursor, err := store.List(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return &pb.ListHTTPAccessLogsResponse{HttpAccessLogs: []*pb.HTTPAccessLog{}, AccessLogs: []*pb.HTTPAccessLog{}, HasMore: false, RequestId: ""}, nil
|
||||
}
|
||||
|
||||
result := make([]*pb.HTTPAccessLog, 0, len(rows))
|
||||
nodeIdSet := make(map[int64]struct{})
|
||||
for _, r := range rows {
|
||||
result = append(result, clickhouse.RowToPB(r))
|
||||
nodeIdSet[int64(r.NodeId)] = struct{}{}
|
||||
}
|
||||
nodeIds := make([]int64, 0, len(nodeIdSet))
|
||||
for id := range nodeIdSet {
|
||||
nodeIds = append(nodeIds, id)
|
||||
}
|
||||
nodes, err := models.SharedNodeDAO.FindEnabledBasicNodesWithIds(tx, nodeIds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clusterIds := make(map[int64]struct{})
|
||||
for _, node := range nodes {
|
||||
if node.ClusterId > 0 {
|
||||
clusterIds[int64(node.ClusterId)] = struct{}{}
|
||||
}
|
||||
}
|
||||
clusterIdList := make([]int64, 0, len(clusterIds))
|
||||
for cid := range clusterIds {
|
||||
clusterIdList = append(clusterIdList, cid)
|
||||
}
|
||||
clusters, _ := models.SharedNodeClusterDAO.FindEnabledNodeClustersWithIds(tx, clusterIdList)
|
||||
clusterMap := make(map[int64]*pb.NodeCluster)
|
||||
for _, c := range clusters {
|
||||
clusterMap[int64(c.Id)] = &pb.NodeCluster{Id: int64(c.Id), Name: c.Name}
|
||||
}
|
||||
pbNodeMap := make(map[int64]*pb.Node)
|
||||
for _, node := range nodes {
|
||||
pbNode := &pb.Node{Id: int64(node.Id), Name: node.Name}
|
||||
if c := clusterMap[int64(node.ClusterId)]; c != nil {
|
||||
pbNode.NodeCluster = c
|
||||
}
|
||||
pbNodeMap[int64(node.Id)] = pbNode
|
||||
}
|
||||
for _, a := range result {
|
||||
if n := pbNodeMap[a.NodeId]; n != nil {
|
||||
a.Node = n
|
||||
}
|
||||
}
|
||||
|
||||
hasMore := nextCursor != ""
|
||||
return &pb.ListHTTPAccessLogsResponse{
|
||||
HttpAccessLogs: result,
|
||||
AccessLogs: result,
|
||||
HasMore: hasMore,
|
||||
RequestId: nextCursor,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FindHTTPAccessLog 查找单个日志
|
||||
func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb.FindHTTPAccessLogRequest) (*pb.FindHTTPAccessLogResponse, error) {
|
||||
// 校验请求
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
|
||||
return !accesslogs.SharedStorageManager.DisableDefaultDB()
|
||||
return accesslogs.SharedStorageManager.WriteMySQL()
|
||||
}
|
||||
|
||||
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
|
||||
|
||||
@@ -53,6 +53,7 @@ func (this *HTTPAccessLogPolicyService) ListHTTPAccessLogPolicies(ctx context.Co
|
||||
IsPublic: policy.IsPublic,
|
||||
FirewallOnly: policy.FirewallOnly == 1,
|
||||
DisableDefaultDB: policy.DisableDefaultDB,
|
||||
WriteTargetsJSON: policy.WriteTargets,
|
||||
})
|
||||
}
|
||||
return &pb.ListHTTPAccessLogPoliciesResponse{HttpAccessLogPolicies: pbPolicies}, nil
|
||||
@@ -76,7 +77,7 @@ func (this *HTTPAccessLogPolicyService) CreateHTTPAccessLogPolicy(ctx context.Co
|
||||
}
|
||||
|
||||
// 创建
|
||||
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB)
|
||||
policyId, err := models.SharedHTTPAccessLogPolicyDAO.CreatePolicy(tx, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -101,7 +102,7 @@ func (this *HTTPAccessLogPolicyService) UpdateHTTPAccessLogPolicy(ctx context.Co
|
||||
}
|
||||
|
||||
// 保存修改
|
||||
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.IsOn)
|
||||
err = models.SharedHTTPAccessLogPolicyDAO.UpdatePolicy(tx, req.HttpAccessLogPolicyId, req.Name, req.Type, req.OptionsJSON, req.CondsJSON, req.IsPublic, req.FirewallOnly, req.DisableDefaultDB, req.WriteTargetsJSON, req.IsOn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -133,6 +134,7 @@ func (this *HTTPAccessLogPolicyService) FindHTTPAccessLogPolicy(ctx context.Cont
|
||||
IsPublic: policy.IsPublic,
|
||||
FirewallOnly: policy.FirewallOnly == 1,
|
||||
DisableDefaultDB: policy.DisableDefaultDB,
|
||||
WriteTargetsJSON: policy.WriteTargets,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
)
|
||||
@@ -26,6 +27,11 @@ func (this *SysSettingService) UpdateSysSetting(ctx context.Context, req *pb.Upd
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 若为 ClickHouse 配置,清空缓存使下次读取生效
|
||||
if req.Code == "clickhouseConfig" {
|
||||
clickhouse.ResetSharedConfig()
|
||||
}
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
|
||||
@@ -113017,7 +113017,7 @@
|
||||
"name": "edgeHTTPAccessLogPolicies",
|
||||
"engine": "InnoDB",
|
||||
"charset": "utf8mb4_unicode_ci",
|
||||
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
|
||||
"definition": "CREATE TABLE `edgeHTTPAccessLogPolicies` (\n `id` int unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `templateId` int unsigned DEFAULT '0' COMMENT '模版ID',\n `adminId` int unsigned DEFAULT '0' COMMENT '管理员ID',\n `userId` int unsigned DEFAULT '0' COMMENT '用户ID',\n `state` tinyint unsigned DEFAULT '1' COMMENT '状态',\n `createdAt` bigint unsigned DEFAULT '0' COMMENT '创建时间',\n `name` varchar(255) DEFAULT NULL COMMENT '名称',\n `isOn` tinyint unsigned DEFAULT '1' COMMENT '是否启用',\n `type` varchar(255) DEFAULT NULL COMMENT '存储类型',\n `options` json DEFAULT NULL COMMENT '存储选项',\n `conds` json DEFAULT NULL COMMENT '请求条件',\n `isPublic` tinyint unsigned DEFAULT '0' COMMENT '是否为公用',\n `firewallOnly` tinyint unsigned DEFAULT '0' COMMENT '是否只记录防火墙相关',\n `version` int unsigned DEFAULT '0' COMMENT '版本号',\n `disableDefaultDB` tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储',\n `writeTargets` json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse',\n PRIMARY KEY (`id`),\n KEY `isPublic` (`isPublic`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='访问日志策略'",
|
||||
"fields": [
|
||||
{
|
||||
"name": "id",
|
||||
@@ -113078,6 +113078,10 @@
|
||||
{
|
||||
"name": "disableDefaultDB",
|
||||
"definition": "tinyint unsigned DEFAULT '0' COMMENT '是否停止默认数据库存储'"
|
||||
},
|
||||
{
|
||||
"name": "writeTargets",
|
||||
"definition": "json DEFAULT NULL COMMENT '写入目标: file/mysql/clickhouse'"
|
||||
}
|
||||
],
|
||||
"indexes": [
|
||||
|
||||
Reference in New Issue
Block a user