robin/waf-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

管理端全部功能跑通

Browse Source
This commit is contained in:
robin
2026-02-27 10:35:22 +08:00
parent 4d275c921d
commit 150799f41d
263 changed files with 22664 additions and 4053 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
EdgeAPI/internal/clickhouse/client.go
View File

@@ -1,6 +1,7 @@
package clickhouse
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
@@ -12,13 +13,11 @@ import (
"time"
)
// Client 通过 HTTP 接口执行只读查询（SELECT），返回 JSONEachRow 解析为 map 或结构体
type Client struct {
cfg *Config
httpCli *http.Client
}
// NewClient 使用共享配置创建客户端
func NewClient() *Client {
cfg := SharedConfig()
transport := &http.Transport{}
@@ -28,6 +27,7 @@ func NewClient() *Client {
ServerName: cfg.TLSServerName,
}
}
return &Client{
cfg: cfg,
httpCli: &http.Client{
@@ -37,21 +37,20 @@ func NewClient() *Client {
}
}
// IsConfigured 是否已配置
func (c *Client) IsConfigured() bool {
return c.cfg != nil && c.cfg.IsConfigured()
}
// Query 执行 SELECT，将每行 JSON 解析到 dest 切片；dest 元素类型需为 *struct 或 map
func (c *Client) Query(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
// 强制 JSONEachRow 便于解析
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
@@ -60,28 +59,32 @@ func (c *Client) Query(ctx context.Context, query string, dest interface{}) erro
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeRows(dec, dest)
}
// QueryRow 执行仅返回一行的查询，将结果解析到 dest（*struct 或 *map）
func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
q := strings.TrimSpace(query)
if !strings.HasSuffix(strings.ToUpper(q), "FORMAT JSONEACHROW") {
query = q + " FORMAT JSONEachRow"
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
@@ -90,32 +93,109 @@ func (c *Client) QueryRow(ctx context.Context, query string, dest interface{}) e
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
dec := json.NewDecoder(resp.Body)
return decodeOneRow(dec, dest)
}
func (c *Client) Execute(ctx context.Context, query string) error {
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
u := c.buildURL(strings.TrimSpace(query))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil)
if err != nil {
return err
}
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
return nil
}
func (c *Client) InsertJSONEachRow(ctx context.Context, insertSQL string, rows []map[string]interface{}) error {
if len(rows) == 0 {
return nil
}
if !c.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
query := strings.TrimSpace(insertSQL)
if !strings.HasSuffix(strings.ToUpper(query), "FORMAT JSONEACHROW") {
query += " FORMAT JSONEachRow"
}
var payload bytes.Buffer
for _, row := range rows {
if row == nil {
continue
}
data, err := json.Marshal(row)
if err != nil {
return err
}
payload.Write(data)
payload.WriteByte('\n')
}
u := c.buildURL(query)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &payload)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if c.cfg.User != "" || c.cfg.Password != "" {
req.SetBasicAuth(c.cfg.User, c.cfg.Password)
}
resp, err := c.httpCli.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("clickhouse HTTP %d: %s", resp.StatusCode, string(body))
}
return nil
}
func (c *Client) buildURL(query string) string {
scheme := "http"
if c.cfg != nil && strings.EqualFold(c.cfg.Scheme, "https") {
scheme = "https"
}
rawURL := fmt.Sprintf("%s://%s:%d/?query=%s&database=%s",
return fmt.Sprintf("%s://%s:%d/?query=%s&database=%s",
scheme, c.cfg.Host, c.cfg.Port, url.QueryEscape(query), url.QueryEscape(c.cfg.Database))
return rawURL
}
// decodeRows 将 JSONEachRow 流解析到 slice；元素类型须为 *struct 或 *[]map[string]interface{}
func decodeRows(dec *json.Decoder, dest interface{}) error {
// dest 应为 *[]*SomeStruct 或 *[]map[string]interface{}
switch d := dest.(type) {
case *[]map[string]interface{}:
*d = (*d)[:0]
@@ -130,7 +210,7 @@ func decodeRows(dec *json.Decoder, dest interface{}) error {
*d = append(*d, row)
}
default:
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{} or implement decoder)")
return fmt.Errorf("clickhouse: unsupported dest type for Query (use *[]map[string]interface{})")
}
}

18
EdgeAPI/internal/clickhouse/config.go
View File

@@ -12,15 +12,15 @@ import (
)
const (
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
envScheme = "CLICKHOUSE_SCHEME"
defaultPort = 8443
defaultDB = "default"
defaultScheme = "https"
envHost = "CLICKHOUSE_HOST"
envPort = "CLICKHOUSE_PORT"
envUser = "CLICKHOUSE_USER"
envPassword = "CLICKHOUSE_PASSWORD"
envDatabase = "CLICKHOUSE_DATABASE"
envScheme = "CLICKHOUSE_SCHEME"
defaultPort = 8443
defaultDB = "default"
defaultScheme = "https"
)
var (

279
EdgeAPI/internal/clickhouse/httpdns_access_logs_store.go Normal file
View File

@@ -0,0 +1,279 @@
package clickhouse
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
)
const httpDNSAccessLogsTable = "httpdns_access_logs_ingest"
type HTTPDNSAccessLogRow struct {
RequestId string
ClusterId int64
NodeId int64
AppId string
AppName string
Domain string
QType string
ClientIP string
ClientRegion string
Carrier string
SDKVersion string
OS string
ResultIPs string
Status string
ErrorCode string
CostMs int32
CreatedAt int64
Day string
Summary string
}
type HTTPDNSAccessLogListFilter struct {
Day string
ClusterId int64
NodeId int64
AppId string
Domain string
Status string
Keyword string
Offset int64
Size int64
}
type HTTPDNSAccessLogsStore struct {
client *Client
}
func NewHTTPDNSAccessLogsStore() *HTTPDNSAccessLogsStore {
return &HTTPDNSAccessLogsStore{client: NewClient()}
}
func (s *HTTPDNSAccessLogsStore) Client() *Client {
return s.client
}
func (s *HTTPDNSAccessLogsStore) Insert(ctx context.Context, logs []*pb.HTTPDNSAccessLog) error {
if len(logs) == 0 {
return nil
}
if !s.client.IsConfigured() {
return fmt.Errorf("clickhouse: not configured")
}
rows := make([]map[string]interface{}, 0, len(logs))
for _, item := range logs {
if item == nil {
continue
}
rows = append(rows, map[string]interface{}{
"request_id": item.GetRequestId(),
"cluster_id": item.GetClusterId(),
"node_id": item.GetNodeId(),
"app_id": item.GetAppId(),
"app_name": item.GetAppName(),
"domain": item.GetDomain(),
"qtype": item.GetQtype(),
"client_ip": item.GetClientIP(),
"client_region": item.GetClientRegion(),
"carrier": item.GetCarrier(),
"sdk_version": item.GetSdkVersion(),
"os": item.GetOs(),
"result_ips": item.GetResultIPs(),
"status": item.GetStatus(),
"error_code": item.GetErrorCode(),
"cost_ms": item.GetCostMs(),
"created_at": item.GetCreatedAt(),
"day": item.GetDay(),
"summary": item.GetSummary(),
})
}
query := fmt.Sprintf("INSERT INTO %s (request_id, cluster_id, node_id, app_id, app_name, domain, qtype, client_ip, client_region, carrier, sdk_version, os, result_ips, status, error_code, cost_ms, created_at, day, summary)",
s.tableName())
return s.client.InsertJSONEachRow(ctx, query, rows)
}
func (s *HTTPDNSAccessLogsStore) Count(ctx context.Context, f HTTPDNSAccessLogListFilter) (int64, error) {
if !s.client.IsConfigured() {
return 0, fmt.Errorf("clickhouse: not configured")
}
conditions := s.buildConditions(f)
query := fmt.Sprintf("SELECT count() AS count FROM %s", s.tableName())
if len(conditions) > 0 {
query += " WHERE " + strings.Join(conditions, " AND ")
}
row := map[string]interface{}{}
if err := s.client.QueryRow(ctx, query, &row); err != nil {
return 0, err
}
return toInt64(row["count"]), nil
}
func (s *HTTPDNSAccessLogsStore) List(ctx context.Context, f HTTPDNSAccessLogListFilter) ([]*HTTPDNSAccessLogRow, error) {
if !s.client.IsConfigured() {
return nil, fmt.Errorf("clickhouse: not configured")
}
size := f.Size
if size <= 0 {
size = 20
}
if size > 1000 {
size = 1000
}
offset := f.Offset
if offset < 0 {
offset = 0
}
conditions := s.buildConditions(f)
query := fmt.Sprintf("SELECT request_id, cluster_id, node_id, app_id, app_name, domain, qtype, client_ip, client_region, carrier, sdk_version, os, result_ips, status, error_code, cost_ms, created_at, day, summary FROM %s",
s.tableName())
if len(conditions) > 0 {
query += " WHERE " + strings.Join(conditions, " AND ")
}
query += " ORDER BY created_at DESC, request_id DESC"
query += fmt.Sprintf(" LIMIT %d OFFSET %d", size, offset)
rawRows := []map[string]interface{}{}
if err := s.client.Query(ctx, query, &rawRows); err != nil {
return nil, err
}
result := make([]*HTTPDNSAccessLogRow, 0, len(rawRows))
for _, row := range rawRows {
result = append(result, &HTTPDNSAccessLogRow{
RequestId: toString(row["request_id"]),
ClusterId: toInt64(row["cluster_id"]),
NodeId: toInt64(row["node_id"]),
AppId: toString(row["app_id"]),
AppName: toString(row["app_name"]),
Domain: toString(row["domain"]),
QType: toString(row["qtype"]),
ClientIP: toString(row["client_ip"]),
ClientRegion: toString(row["client_region"]),
Carrier: toString(row["carrier"]),
SDKVersion: toString(row["sdk_version"]),
OS: toString(row["os"]),
ResultIPs: toString(row["result_ips"]),
Status: toString(row["status"]),
ErrorCode: toString(row["error_code"]),
CostMs: int32(toInt64(row["cost_ms"])),
CreatedAt: toInt64(row["created_at"]),
Day: toString(row["day"]),
Summary: toString(row["summary"]),
})
}
return result, nil
}
func HTTPDNSRowToPB(row *HTTPDNSAccessLogRow) *pb.HTTPDNSAccessLog {
if row == nil {
return nil
}
return &pb.HTTPDNSAccessLog{
RequestId: row.RequestId,
ClusterId: row.ClusterId,
NodeId: row.NodeId,
AppId: row.AppId,
AppName: row.AppName,
Domain: row.Domain,
Qtype: row.QType,
ClientIP: row.ClientIP,
ClientRegion: row.ClientRegion,
Carrier: row.Carrier,
SdkVersion: row.SDKVersion,
Os: row.OS,
ResultIPs: row.ResultIPs,
Status: row.Status,
ErrorCode: row.ErrorCode,
CostMs: row.CostMs,
CreatedAt: row.CreatedAt,
Day: row.Day,
Summary: row.Summary,
}
}
func (s *HTTPDNSAccessLogsStore) buildConditions(f HTTPDNSAccessLogListFilter) []string {
conditions := []string{}
if day := strings.TrimSpace(f.Day); day != "" {
conditions = append(conditions, "day = '"+escapeString(day)+"'")
}
if f.ClusterId > 0 {
conditions = append(conditions, "cluster_id = "+strconv.FormatInt(f.ClusterId, 10))
}
if f.NodeId > 0 {
conditions = append(conditions, "node_id = "+strconv.FormatInt(f.NodeId, 10))
}
if appID := strings.TrimSpace(f.AppId); appID != "" {
conditions = append(conditions, "app_id = '"+escapeString(appID)+"'")
}
if domain := strings.TrimSpace(f.Domain); domain != "" {
conditions = append(conditions, "domain = '"+escapeString(domain)+"'")
}
if status := strings.TrimSpace(f.Status); status != "" {
conditions = append(conditions, "status = '"+escapeString(status)+"'")
}
if keyword := strings.TrimSpace(f.Keyword); keyword != "" {
kw := escapeString(keyword)
conditions = append(conditions, "(summary LIKE '%"+kw+"%' OR app_name LIKE '%"+kw+"%' OR client_ip LIKE '%"+kw+"%' OR result_ips LIKE '%"+kw+"%')")
}
return conditions
}
func (s *HTTPDNSAccessLogsStore) tableName() string {
if s.client != nil && s.client.cfg != nil && s.client.cfg.Database != "" && s.client.cfg.Database != "default" {
return quoteIdent(s.client.cfg.Database) + "." + quoteIdent(httpDNSAccessLogsTable)
}
return quoteIdent(httpDNSAccessLogsTable)
}
func toString(value interface{}) string {
if value == nil {
return ""
}
switch v := value.(type) {
case string:
return v
case json.Number:
return v.String()
default:
return fmt.Sprintf("%v", v)
}
}
func toInt64(value interface{}) int64 {
if value == nil {
return 0
}
switch v := value.(type) {
case int:
return int64(v)
case int32:
return int64(v)
case int64:
return v
case uint32:
return int64(v)
case uint64:
return int64(v)
case float64:
return int64(v)
case json.Number:
n, _ := v.Int64()
return n
case string:
n, _ := strconv.ParseInt(v, 10, 64)
return n
default:
return 0
}
}

4
EdgeAPI/internal/clickhouse/logs_ingest_store.go
View File

@@ -351,8 +351,8 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
RemoteAddr: r.IP,
RequestMethod: r.Method,
RequestPath: r.Path,
RequestURI: r.Path, // 前端使用 requestURI 显示完整路径
Scheme: "http", // 默认 http，日志中未存储实际值
RequestURI: r.Path, // 前端使用 requestURI 显示完整路径
Scheme: "http", // 默认 http，日志中未存储实际值
Proto: "HTTP/1.1", // 默认值，日志中未存储实际值
Status: int32(r.Status),
RequestLength: int64(r.BytesIn),