日常查询由mysql改为clickhouse

EdgeAPI/.db.yaml

@@ -1,5 +0,0 @@
-user: root
-password: 123456
-host: 127.0.0.1:3307
-database: db_edge
-boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]

EdgeAPI/.gitignore

@@ -28,3 +28,13 @@ logs/
 # 临时文件
 *.tmp
 .DS_Store
+
+# Runtime Data
+data/
+configs/api.yaml
+configs/db.yaml
+db.yaml
+db.yaml.link.bak
+
+# Build Artifacts
+bin/

EdgeAPI/db.yaml

@@ -1,6 +1,6 @@
 user: root
 password: 123456
 host: 127.0.0.1
-pord: 3307
+pord: 3308
 database: db_edge
 boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]

EdgeAPI/internal/clickhouse/logs_ingest_store.go

@@ -56,6 +56,10 @@ type ListFilter struct {
 	LastRequestId     string
 	ServerIds         []int64
 	NodeIds           []int64
+	// 搜索条件
+	Keyword string
+	Ip      string
+	Domain  string
 }
 
 // LogsIngestStore 封装对 logs_ingest 的只读列表查询
@@ -126,10 +130,35 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
 		conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
 	}
 
+	// 搜索条件
+	if f.Keyword != "" {
+		keyword := escapeString(f.Keyword)
+		// 在 host, path, ip, ua 中模糊搜索
+		conditions = append(conditions, fmt.Sprintf("(host LIKE '%%%s%%' OR path LIKE '%%%s%%' OR ip LIKE '%%%s%%' OR ua LIKE '%%%s%%')", keyword, keyword, keyword, keyword))
+	}
+	if f.Ip != "" {
+		conditions = append(conditions, "ip = '"+escapeString(f.Ip)+"'")
+	}
+	if f.Domain != "" {
+		conditions = append(conditions, "host LIKE '%"+escapeString(f.Domain)+"%'")
+	}
+
+	// 游标分页：使用 trace_id 作为游标
+	// Reverse=false：历史向后翻页，查询更早的数据
+	// Reverse=true：实时增量拉新，查询更新的数据
+	if f.LastRequestId != "" {
+		if f.Reverse {
+			conditions = append(conditions, "trace_id > '"+escapeString(f.LastRequestId)+"'")
+		} else {
+			conditions = append(conditions, "trace_id < '"+escapeString(f.LastRequestId)+"'")
+		}
+	}
+
 	where := strings.Join(conditions, " AND ")
-	orderDir := "ASC"
+	// 默认按时间倒序（最新的在前面），与前端默认行为一致
+	orderDir := "DESC"
 	if f.Reverse {
-		orderDir = "DESC"
+		orderDir = "ASC"
 	}
 	limit := f.Size
 	if limit <= 0 {
@@ -138,7 +167,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
 	if limit > 1000 {
 		limit = 1000
 	}
-	orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
+	orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
 
 	query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
 		table, where, orderBy, limit+1)
@@ -155,13 +184,51 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
 			rows = append(rows, r)
 		}
 	}
+	if !f.Reverse {
+		if len(rows) > int(limit) {
+			nextCursor = rows[limit].TraceId
+			rows = rows[:limit]
+		}
+		return rows, nextCursor, nil
+	}
+
 	if len(rows) > int(limit) {
-		nextCursor = rows[limit].TraceId
 		rows = rows[:limit]
 	}
+	if len(rows) > 0 {
+		nextCursor = rows[len(rows)-1].TraceId
+	}
+
+	// 实时模式统一返回为“最新在前”，与前端显示和 MySQL 语义一致。
+	for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 {
+		rows[left], rows[right] = rows[right], rows[left]
+	}
 	return rows, nextCursor, nil
 }
 
+// FindByTraceId 按 trace_id 查询单条日志详情
+func (s *LogsIngestStore) FindByTraceId(ctx context.Context, traceId string) (*LogsIngestRow, error) {
+	if !s.client.IsConfigured() {
+		return nil, fmt.Errorf("clickhouse: not configured")
+	}
+	if traceId == "" {
+		return nil, nil
+	}
+
+	table := quoteIdent("logs_ingest")
+	query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE trace_id = '%s' LIMIT 1",
+		table, escapeString(traceId))
+
+	var rawRows []map[string]interface{}
+	if err := s.client.Query(ctx, query, &rawRows); err != nil {
+		return nil, err
+	}
+	if len(rawRows) == 0 {
+		return nil, nil
+	}
+	return mapToLogsIngestRow(rawRows[0]), nil
+}
+
 func quoteIdent(name string) string {
 	return "`" + strings.ReplaceAll(name, "`", "``") + "`"
 }
@@ -244,7 +311,7 @@ func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
 	return r
 }
 
-// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog（列表展示用）
+// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog（列表展示用+详情展示）
 func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
 	if r == nil {
 		return nil
@@ -259,6 +326,9 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
 		RemoteAddr:          r.IP,
 		RequestMethod:       r.Method,
 		RequestPath:         r.Path,
+		RequestURI:          r.Path, // 前端使用 requestURI 显示完整路径
+		Scheme:              "http", // 默认 http，日志中未存储实际值
+		Proto:               "HTTP/1.1", // 默认值，日志中未存储实际值
 		Status:              int32(r.Status),
 		RequestLength:       int64(r.BytesIn),
 		BytesSent:           int64(r.BytesOut),
@@ -273,6 +343,39 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
 	if r.TimeISO8601() != "" {
 		a.TimeISO8601 = r.TimeISO8601()
 	}
+	// TimeLocal: 用户友好的时间格式 (e.g., "2026-02-07 23:17:12")
+	if !r.Timestamp.IsZero() {
+		a.TimeLocal = r.Timestamp.Format("2006-01-02 15:04:05")
+	}
+
+	// 解析请求头 (JSON -> map[string]*pb.Strings)
+	// ClickHouse 中存储的是 map[string]string 格式
+	if r.RequestHeaders != "" {
+		var headers map[string]string
+		if err := json.Unmarshal([]byte(r.RequestHeaders), &headers); err == nil {
+			a.Header = make(map[string]*pb.Strings)
+			for k, v := range headers {
+				a.Header[k] = &pb.Strings{Values: []string{v}}
+			}
+		}
+	}
+
+	// 解析响应头 (JSON -> map[string]*pb.Strings)
+	if r.ResponseHeaders != "" {
+		var headers map[string]string
+		if err := json.Unmarshal([]byte(r.ResponseHeaders), &headers); err == nil {
+			a.SentHeader = make(map[string]*pb.Strings)
+			for k, v := range headers {
+				a.SentHeader[k] = &pb.Strings{Values: []string{v}}
+			}
+		}
+	}
+
+	// 请求体
+	if r.RequestBody != "" {
+		a.RequestBody = []byte(r.RequestBody)
+	}
+
 	return a
 }
 

EdgeAPI/internal/db/models/sys_setting_dao.go

@@ -261,3 +261,23 @@ func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs
 	}
 	return config, nil
 }
+
+// ReadClickHouseConfig 读取ClickHouse配置
+func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
+	valueJSON, err := this.ReadSetting(tx, "clickhouseConfig")
+	if err != nil {
+		return nil, err
+	}
+	if len(valueJSON) == 0 {
+		return &systemconfigs.ClickHouseSetting{
+			Port:     8123,
+			Database: "default",
+		}, nil
+	}
+	var config = &systemconfigs.ClickHouseSetting{}
+	err = json.Unmarshal(valueJSON, config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}

EdgeAPI/internal/db/models/sys_setting_dao_plus.go

@@ -66,25 +66,3 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
 	return config, nil
 }
 
-// ReadClickHouseConfig 读取 ClickHouse 连接配置（后台页面配置，用于访问日志 logs_ingest 查询）
-func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
-	valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
-	if err != nil {
-		return nil, err
-	}
-	out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
-	if len(valueJSON) == 0 {
-		return out, nil
-	}
-	err = json.Unmarshal(valueJSON, out)
-	if err != nil {
-		return nil, err
-	}
-	if out.Port <= 0 {
-		out.Port = 8123
-	}
-	if out.Database == "" {
-		out.Database = "default"
-	}
-	return out, nil
-}

EdgeAPI/internal/db/utils/disk.go

@@ -6,7 +6,6 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/dbs"
-	"golang.org/x/sys/unix"
 	"time"
 )
 
@@ -60,14 +59,7 @@ func CheckHasFreeSpace() bool {
 		}
 		LocalDatabaseDataDir = dir
 
-		var stat unix.Statfs_t
-		err = unix.Statfs(dir, &stat)
-		if err != nil {
-			return true
-		}
-
-		var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
-		return availableSpace > minFreeSpaceGB
+		return checkHasFreeSpace(dir)
 	}
 	return true
 }

EdgeAPI/internal/db/utils/disk_unix.go

@@ -0,0 +1,18 @@
+//go:build !windows
+
+package dbutils
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func checkHasFreeSpace(dir string) bool {
+	var stat unix.Statfs_t
+	err := unix.Statfs(dir, &stat)
+	if err != nil {
+		return true
+	}
+
+	var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
+	return availableSpace > minFreeSpaceGB
+}

EdgeAPI/internal/db/utils/disk_windows.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package dbutils
+
+func checkHasFreeSpace(dir string) bool {
+	return true
+}

EdgeAPI/internal/nodes/node_status_executor_windows.go

@@ -4,8 +4,9 @@
 package nodes
 
 import (
-	"context"
-	"github.com/shirou/gopsutil/v3/cpu"
+	//"context"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	//"github.com/shirou/gopsutil/v3/cpu"
 	"github.com/shirou/gopsutil/v3/mem"
 	"math"
 	"sync"
@@ -21,7 +22,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
 var windowsLoadLocker = &sync.Mutex{}
 
 // 更新内存
-func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
+func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
 	stat, err := mem.VirtualMemory()
 	if err != nil {
 		status.Error = err.Error()
@@ -32,14 +33,14 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
 }
 
 // 更新负载
-func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
+func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
 	timestamp := time.Now().Unix()
 
 	currentLoad := 0
-	info, err := cpu.ProcInfo()
-	if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
-		currentLoad = int(info[0].ProcessorQueueLength)
-	}
+	//info, err := cpu.ProcInfo()
+	//if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
+	//	currentLoad = int(info[0].ProcessorQueueLength)
+	//}
 
 	// 删除15分钟之前的数据
 	windowsLoadLocker.Lock()
@@ -93,9 +94,9 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
 	windowsLoadLocker.Unlock()
 
 	// 在老Windows上不显示错误
-	if err == context.DeadlineExceeded {
-		err = nil
-	}
+	//if err == context.DeadlineExceeded {
+	//	err = nil
+	//}
 	status.Load1m = load1
 	status.Load5m = load5
 	status.Load15m = load15

EdgeAPI/internal/rpc/services/service_http_access_log.go

@@ -144,6 +144,9 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
 		NodeId:            req.NodeId,
 		ClusterId:         req.NodeClusterId,
 		LastRequestId:     req.RequestId,
+		Keyword:           req.Keyword,
+		Ip:                req.Ip,
+		Domain:            req.Domain,
 	}
 	if req.ServerId > 0 {
 		f.ServerIds = []int64{req.ServerId}
@@ -216,7 +219,10 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
 		}
 	}
 
-	hasMore := nextCursor != ""
+	hasMore := false
+	if !req.Reverse {
+		hasMore = nextCursor != ""
+	}
 	return &pb.ListHTTPAccessLogsResponse{
 		HttpAccessLogs: result,
 		AccessLogs:     result,
@@ -233,6 +239,28 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
 		return nil, err
 	}
 
+	// 优先从 ClickHouse 查询
+	store := clickhouse.NewLogsIngestStore()
+	if store.Client().IsConfigured() {
+		row, err := store.FindByTraceId(ctx, req.RequestId)
+		if err != nil {
+			return nil, err
+		}
+		if row != nil {
+			// 检查权限
+			if userId > 0 {
+				var tx = this.NullTx()
+				err = models.SharedServerDAO.CheckUserServer(tx, userId, int64(row.ServerId))
+				if err != nil {
+					return nil, err
+				}
+			}
+			a := clickhouse.RowToPB(row)
+			return &pb.FindHTTPAccessLogResponse{HttpAccessLog: a}, nil
+		}
+	}
+
+	// 如果 ClickHouse 未配置或未找到，则回退到 MySQL
 	var tx = this.NullTx()
 
 	accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)

EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go

@@ -11,7 +11,7 @@ import (
 )
 
 func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
-	return accesslogs.SharedStorageManager.WriteMySQL()
+	return false
 }
 
 func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {