From b7388d83b0f7150842d622167687d18cfea4ad0f Mon Sep 17 00:00:00 2001 From: robin Date: Sun, 8 Feb 2026 02:00:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=97=A5=E5=B8=B8=E6=9F=A5=E8=AF=A2=E7=94=B1my?= =?UTF-8?q?sql=E6=94=B9=E4=B8=BAclickhouse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- EdgeAPI/.db.yaml | 5 - EdgeAPI/.gitignore | 10 + EdgeAPI/db.yaml | 2 +- .../internal/clickhouse/logs_ingest_store.go | 113 ++++++++++- EdgeAPI/internal/db/models/sys_setting_dao.go | 20 ++ .../db/models/sys_setting_dao_plus.go | 22 --- EdgeAPI/internal/db/utils/disk.go | 10 +- EdgeAPI/internal/db/utils/disk_unix.go | 18 ++ EdgeAPI/internal/db/utils/disk_windows.go | 7 + .../nodes/node_status_executor_windows.go | 23 +-- .../rpc/services/service_http_access_log.go | 30 ++- .../service_http_access_log_ext_plus.go | 2 +- EdgeAdmin/.gitignore | 12 ++ .../@default/servers/server/log/index.js | 66 +++++-- EdgeCommon/go.mod | 1 + EdgeCommon/go.sum | 2 + .../pkg/systemconfigs/clickhouse_setting.go | 1 - EdgeNode/.gitignore | 16 ++ EdgeNode/internal/accesslogs/ingest_log.go | 2 +- EdgeNode/internal/apps/app_cmd.go | 6 +- EdgeNode/internal/apps/app_utils_unix.go | 15 ++ EdgeNode/internal/caches/manager.go | 40 ---- EdgeNode/internal/caches/manager_unix.go | 48 +++++ EdgeNode/internal/caches/storage_file.go | 3 +- .../internal/caches/storage_utils_unix.go | 9 + EdgeNode/internal/nodes/http_writer.go | 166 ---------------- .../internal/nodes/http_writer_ext_unix.go | 181 ++++++++++++++++++ .../{node_panic.go => node_panic_unix.go} | 3 +- .../nodes/node_status_executor_windows.go | 25 +-- .../utils/fs/{locker.go => locker_unix.go} | 1 + EdgeNode/internal/utils/fs/stat.go | 46 ----- EdgeNode/internal/utils/fs/stat_unix.go | 54 ++++++ EdgeNode/internal/utils/mmap/flags_others.go | 2 +- .../mmap/{mmap_ext.go => mmap_ext_unix.go} | 2 +- .../internal/utils/{net.go => net_unix.go} | 6 +- .../internal/waf/injectionutils/utils_sqli.go | 2 + .../waf/injectionutils/utils_sqli_nocgo.go | 18 ++ .../internal/waf/injectionutils/utils_xss.go | 2 + .../waf/injectionutils/utils_xss_nocgo.go | 17 ++ deploy/fluent-bit/.gitignore | 2 + deploy/fluent-bit/logs.db | Bin 0 -> 8192 bytes deploy/fluent-bit/logs.db-shm | Bin 0 -> 32768 bytes deploy/fluent-bit/logs.db-wal | Bin 0 -> 218392 bytes 43 files changed, 657 insertions(+), 353 deletions(-) delete mode 100644 EdgeAPI/.db.yaml create mode 100644 EdgeAPI/internal/db/utils/disk_unix.go create mode 100644 EdgeAPI/internal/db/utils/disk_windows.go create mode 100644 EdgeNode/internal/apps/app_utils_unix.go create mode 100644 EdgeNode/internal/caches/manager_unix.go create mode 100644 EdgeNode/internal/caches/storage_utils_unix.go create mode 100644 EdgeNode/internal/nodes/http_writer_ext_unix.go rename EdgeNode/internal/nodes/{node_panic.go => node_panic_unix.go} (96%) rename EdgeNode/internal/utils/fs/{locker.go => locker_unix.go} (98%) create mode 100644 EdgeNode/internal/utils/fs/stat_unix.go rename EdgeNode/internal/utils/mmap/{mmap_ext.go => mmap_ext_unix.go} (98%) rename EdgeNode/internal/utils/{net.go => net_unix.go} (77%) create mode 100644 EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go create mode 100644 EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go create mode 100644 deploy/fluent-bit/.gitignore create mode 100644 deploy/fluent-bit/logs.db create mode 100644 deploy/fluent-bit/logs.db-shm create mode 100644 deploy/fluent-bit/logs.db-wal diff --git a/EdgeAPI/.db.yaml b/EdgeAPI/.db.yaml deleted file mode 100644 index 0197ea3..0000000 --- a/EdgeAPI/.db.yaml +++ /dev/null @@ -1,5 +0,0 @@ -user: root -password: 123456 -host: 127.0.0.1:3307 -database: db_edge -boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ] \ No newline at end of file diff --git a/EdgeAPI/.gitignore b/EdgeAPI/.gitignore index df96ead..09d0e54 100644 --- a/EdgeAPI/.gitignore +++ b/EdgeAPI/.gitignore @@ -28,3 +28,13 @@ logs/ # 临时文件 *.tmp .DS_Store + +# Runtime Data +data/ +configs/api.yaml +configs/db.yaml +db.yaml +db.yaml.link.bak + +# Build Artifacts +bin/ diff --git a/EdgeAPI/db.yaml b/EdgeAPI/db.yaml index 6785333..dc29168 100644 --- a/EdgeAPI/db.yaml +++ b/EdgeAPI/db.yaml @@ -1,6 +1,6 @@ user: root password: 123456 host: 127.0.0.1 -pord: 3307 +pord: 3308 database: db_edge boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ] \ No newline at end of file diff --git a/EdgeAPI/internal/clickhouse/logs_ingest_store.go b/EdgeAPI/internal/clickhouse/logs_ingest_store.go index eec17c4..047bd1d 100644 --- a/EdgeAPI/internal/clickhouse/logs_ingest_store.go +++ b/EdgeAPI/internal/clickhouse/logs_ingest_store.go @@ -56,6 +56,10 @@ type ListFilter struct { LastRequestId string ServerIds []int64 NodeIds []int64 + // 搜索条件 + Keyword string + Ip string + Domain string } // LogsIngestStore 封装对 logs_ingest 的只读列表查询 @@ -126,10 +130,35 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10)) } + // 搜索条件 + if f.Keyword != "" { + keyword := escapeString(f.Keyword) + // 在 host, path, ip, ua 中模糊搜索 + conditions = append(conditions, fmt.Sprintf("(host LIKE '%%%s%%' OR path LIKE '%%%s%%' OR ip LIKE '%%%s%%' OR ua LIKE '%%%s%%')", keyword, keyword, keyword, keyword)) + } + if f.Ip != "" { + conditions = append(conditions, "ip = '"+escapeString(f.Ip)+"'") + } + if f.Domain != "" { + conditions = append(conditions, "host LIKE '%"+escapeString(f.Domain)+"%'") + } + + // 游标分页：使用 trace_id 作为游标 + // Reverse=false：历史向后翻页，查询更早的数据 + // Reverse=true：实时增量拉新，查询更新的数据 + if f.LastRequestId != "" { + if f.Reverse { + conditions = append(conditions, "trace_id > '"+escapeString(f.LastRequestId)+"'") + } else { + conditions = append(conditions, "trace_id < '"+escapeString(f.LastRequestId)+"'") + } + } + where := strings.Join(conditions, " AND ") - orderDir := "ASC" + // 默认按时间倒序（最新的在前面），与前端默认行为一致 + orderDir := "DESC" if f.Reverse { - orderDir = "DESC" + orderDir = "ASC" } limit := f.Size if limit <= 0 { @@ -138,7 +167,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI if limit > 1000 { limit = 1000 } - orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir) + orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir) query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d", table, where, orderBy, limit+1) @@ -155,13 +184,51 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI rows = append(rows, r) } } + if !f.Reverse { + if len(rows) > int(limit) { + nextCursor = rows[limit].TraceId + rows = rows[:limit] + } + return rows, nextCursor, nil + } + if len(rows) > int(limit) { - nextCursor = rows[limit].TraceId rows = rows[:limit] } + if len(rows) > 0 { + nextCursor = rows[len(rows)-1].TraceId + } + + // 实时模式统一返回为“最新在前”，与前端显示和 MySQL 语义一致。 + for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 { + rows[left], rows[right] = rows[right], rows[left] + } return rows, nextCursor, nil } +// FindByTraceId 按 trace_id 查询单条日志详情 +func (s *LogsIngestStore) FindByTraceId(ctx context.Context, traceId string) (*LogsIngestRow, error) { + if !s.client.IsConfigured() { + return nil, fmt.Errorf("clickhouse: not configured") + } + if traceId == "" { + return nil, nil + } + + table := quoteIdent("logs_ingest") + query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE trace_id = '%s' LIMIT 1", + table, escapeString(traceId)) + + var rawRows []map[string]interface{} + if err := s.client.Query(ctx, query, &rawRows); err != nil { + return nil, err + } + if len(rawRows) == 0 { + return nil, nil + } + return mapToLogsIngestRow(rawRows[0]), nil +} + func quoteIdent(name string) string { return "`" + strings.ReplaceAll(name, "`", "``") + "`" } @@ -244,7 +311,7 @@ func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow { return r } -// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog（列表展示用） +// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog（列表展示用+详情展示） func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog { if r == nil { return nil @@ -259,6 +326,9 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog { RemoteAddr: r.IP, RequestMethod: r.Method, RequestPath: r.Path, + RequestURI: r.Path, // 前端使用 requestURI 显示完整路径 + Scheme: "http", // 默认 http，日志中未存储实际值 + Proto: "HTTP/1.1", // 默认值，日志中未存储实际值 Status: int32(r.Status), RequestLength: int64(r.BytesIn), BytesSent: int64(r.BytesOut), @@ -273,6 +343,39 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog { if r.TimeISO8601() != "" { a.TimeISO8601 = r.TimeISO8601() } + // TimeLocal: 用户友好的时间格式 (e.g., "2026-02-07 23:17:12") + if !r.Timestamp.IsZero() { + a.TimeLocal = r.Timestamp.Format("2006-01-02 15:04:05") + } + + // 解析请求头 (JSON -> map[string]*pb.Strings) + // ClickHouse 中存储的是 map[string]string 格式 + if r.RequestHeaders != "" { + var headers map[string]string + if err := json.Unmarshal([]byte(r.RequestHeaders), &headers); err == nil { + a.Header = make(map[string]*pb.Strings) + for k, v := range headers { + a.Header[k] = &pb.Strings{Values: []string{v}} + } + } + } + + // 解析响应头 (JSON -> map[string]*pb.Strings) + if r.ResponseHeaders != "" { + var headers map[string]string + if err := json.Unmarshal([]byte(r.ResponseHeaders), &headers); err == nil { + a.SentHeader = make(map[string]*pb.Strings) + for k, v := range headers { + a.SentHeader[k] = &pb.Strings{Values: []string{v}} + } + } + } + + // 请求体 + if r.RequestBody != "" { + a.RequestBody = []byte(r.RequestBody) + } + return a } diff --git a/EdgeAPI/internal/db/models/sys_setting_dao.go b/EdgeAPI/internal/db/models/sys_setting_dao.go index c686be6..8795cca 100644 --- a/EdgeAPI/internal/db/models/sys_setting_dao.go +++ b/EdgeAPI/internal/db/models/sys_setting_dao.go @@ -261,3 +261,23 @@ func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs } return config, nil } + +// ReadClickHouseConfig 读取ClickHouse配置 +func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) { + valueJSON, err := this.ReadSetting(tx, "clickhouseConfig") + if err != nil { + return nil, err + } + if len(valueJSON) == 0 { + return &systemconfigs.ClickHouseSetting{ + Port: 8123, + Database: "default", + }, nil + } + var config = &systemconfigs.ClickHouseSetting{} + err = json.Unmarshal(valueJSON, config) + if err != nil { + return nil, err + } + return config, nil +} diff --git a/EdgeAPI/internal/db/models/sys_setting_dao_plus.go b/EdgeAPI/internal/db/models/sys_setting_dao_plus.go index 034d268..d38c61f 100644 --- a/EdgeAPI/internal/db/models/sys_setting_dao_plus.go +++ b/EdgeAPI/internal/db/models/sys_setting_dao_plus.go @@ -66,25 +66,3 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe return config, nil } -// ReadClickHouseConfig 读取 ClickHouse 连接配置（后台页面配置，用于访问日志 logs_ingest 查询） -func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) { - valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig) - if err != nil { - return nil, err - } - out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"} - if len(valueJSON) == 0 { - return out, nil - } - err = json.Unmarshal(valueJSON, out) - if err != nil { - return nil, err - } - if out.Port <= 0 { - out.Port = 8123 - } - if out.Database == "" { - out.Database = "default" - } - return out, nil -} diff --git a/EdgeAPI/internal/db/utils/disk.go b/EdgeAPI/internal/db/utils/disk.go index 90ae664..c6ee1ab 100644 --- a/EdgeAPI/internal/db/utils/disk.go +++ b/EdgeAPI/internal/db/utils/disk.go @@ -6,7 +6,6 @@ import ( "github.com/TeaOSLab/EdgeAPI/internal/goman" "github.com/go-sql-driver/mysql" "github.com/iwind/TeaGo/dbs" - "golang.org/x/sys/unix" "time" ) @@ -60,14 +59,7 @@ func CheckHasFreeSpace() bool { } LocalDatabaseDataDir = dir - var stat unix.Statfs_t - err = unix.Statfs(dir, &stat) - if err != nil { - return true - } - - var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB - return availableSpace > minFreeSpaceGB + return checkHasFreeSpace(dir) } return true } diff --git a/EdgeAPI/internal/db/utils/disk_unix.go b/EdgeAPI/internal/db/utils/disk_unix.go new file mode 100644 index 0000000..2ee4cec --- /dev/null +++ b/EdgeAPI/internal/db/utils/disk_unix.go @@ -0,0 +1,18 @@ +//go:build !windows + +package dbutils + +import ( + "golang.org/x/sys/unix" +) + +func checkHasFreeSpace(dir string) bool { + var stat unix.Statfs_t + err := unix.Statfs(dir, &stat) + if err != nil { + return true + } + + var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB + return availableSpace > minFreeSpaceGB +} diff --git a/EdgeAPI/internal/db/utils/disk_windows.go b/EdgeAPI/internal/db/utils/disk_windows.go new file mode 100644 index 0000000..16bbbf0 --- /dev/null +++ b/EdgeAPI/internal/db/utils/disk_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package dbutils + +func checkHasFreeSpace(dir string) bool { + return true +} diff --git a/EdgeAPI/internal/nodes/node_status_executor_windows.go b/EdgeAPI/internal/nodes/node_status_executor_windows.go index 51c61d5..de8041f 100644 --- a/EdgeAPI/internal/nodes/node_status_executor_windows.go +++ b/EdgeAPI/internal/nodes/node_status_executor_windows.go @@ -4,8 +4,9 @@ package nodes import ( - "context" - "github.com/shirou/gopsutil/v3/cpu" + //"context" + "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" + //"github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/mem" "math" "sync" @@ -21,7 +22,7 @@ var windowsLoadValues = []*WindowsLoadValue{} var windowsLoadLocker = &sync.Mutex{} // 更新内存 -func (this *NodeStatusExecutor) updateMem(status *NodeStatus) { +func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) { stat, err := mem.VirtualMemory() if err != nil { status.Error = err.Error() @@ -32,14 +33,14 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) { } // 更新负载 -func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) { +func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) { timestamp := time.Now().Unix() currentLoad := 0 - info, err := cpu.ProcInfo() - if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 { - currentLoad = int(info[0].ProcessorQueueLength) - } + //info, err := cpu.ProcInfo() + //if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 { + // currentLoad = int(info[0].ProcessorQueueLength) + //} // 删除15分钟之前的数据 windowsLoadLocker.Lock() @@ -93,9 +94,9 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) { windowsLoadLocker.Unlock() // 在老Windows上不显示错误 - if err == context.DeadlineExceeded { - err = nil - } + //if err == context.DeadlineExceeded { + // err = nil + //} status.Load1m = load1 status.Load5m = load5 status.Load15m = load15 diff --git a/EdgeAPI/internal/rpc/services/service_http_access_log.go b/EdgeAPI/internal/rpc/services/service_http_access_log.go index baa4d68..7c5af2b 100644 --- a/EdgeAPI/internal/rpc/services/service_http_access_log.go +++ b/EdgeAPI/internal/rpc/services/service_http_access_log.go @@ -144,6 +144,9 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C NodeId: req.NodeId, ClusterId: req.NodeClusterId, LastRequestId: req.RequestId, + Keyword: req.Keyword, + Ip: req.Ip, + Domain: req.Domain, } if req.ServerId > 0 { f.ServerIds = []int64{req.ServerId} @@ -216,7 +219,10 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C } } - hasMore := nextCursor != "" + hasMore := false + if !req.Reverse { + hasMore = nextCursor != "" + } return &pb.ListHTTPAccessLogsResponse{ HttpAccessLogs: result, AccessLogs: result, @@ -233,6 +239,28 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb return nil, err } + // 优先从 ClickHouse 查询 + store := clickhouse.NewLogsIngestStore() + if store.Client().IsConfigured() { + row, err := store.FindByTraceId(ctx, req.RequestId) + if err != nil { + return nil, err + } + if row != nil { + // 检查权限 + if userId > 0 { + var tx = this.NullTx() + err = models.SharedServerDAO.CheckUserServer(tx, userId, int64(row.ServerId)) + if err != nil { + return nil, err + } + } + a := clickhouse.RowToPB(row) + return &pb.FindHTTPAccessLogResponse{HttpAccessLog: a}, nil + } + } + + // 如果 ClickHouse 未配置或未找到，则回退到 MySQL var tx = this.NullTx() accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId) diff --git a/EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go b/EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go index 1938ca2..1f6ade6 100644 --- a/EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go +++ b/EdgeAPI/internal/rpc/services/service_http_access_log_ext_plus.go @@ -11,7 +11,7 @@ import ( ) func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool { - return accesslogs.SharedStorageManager.WriteMySQL() + return false } func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error { diff --git a/EdgeAdmin/.gitignore b/EdgeAdmin/.gitignore index e69de29..07593d8 100644 --- a/EdgeAdmin/.gitignore +++ b/EdgeAdmin/.gitignore @@ -0,0 +1,12 @@ +部署启动手册.md + +# Runtime Data +data/ +logs/ + +# Local Configs +configs/api_admin.yaml +configs/server.yaml + +# Build Artifacts +bin/ \ No newline at end of file diff --git a/EdgeAdmin/web/views/@default/servers/server/log/index.js b/EdgeAdmin/web/views/@default/servers/server/log/index.js index 336511e..41cc22d 100644 --- a/EdgeAdmin/web/views/@default/servers/server/log/index.js +++ b/EdgeAdmin/web/views/@default/servers/server/log/index.js @@ -7,6 +7,47 @@ Tea.context(function () { this.accessLogs = [] this.isLoaded = false + this.mergeAccessLogs = function (newAccessLogs, regions, wafInfos) { + let mergedLogs = newAccessLogs.concat(this.accessLogs) + let result = [] + let seenRequestIds = {} + + let that = this + mergedLogs.forEach(function (accessLog) { + if (accessLog == null) { + return + } + + let requestId = accessLog.requestId || "" + if (requestId.length > 0) { + if (seenRequestIds[requestId] === true) { + return + } + seenRequestIds[requestId] = true + } + + that.formatTime(accessLog) + + let region = regions[accessLog.remoteAddr] + if (typeof (region) == "string") { + accessLog.region = region + } else if (typeof accessLog.region != "string") { + accessLog.region = "" + } + + let wafInfo = wafInfos[accessLog.firewallRuleSetId] + if (accessLog.firewallRuleSetId > 0 && typeof (wafInfo) == "object") { + accessLog.wafInfo = wafInfo + } else if (typeof accessLog.wafInfo == "undefined") { + accessLog.wafInfo = null + } + + result.push(accessLog) + }) + + return result + } + this.load = function () { // 如果有弹窗时，暂时不更新 if (teaweb.hasPopup()) { @@ -27,30 +68,17 @@ Tea.context(function () { nodeId: this.nodeId }) .success(function (resp) { - this.accessLogs = resp.data.accessLogs.concat(this.accessLogs) - - // 添加区域信息 - let that = this - this.accessLogs.forEach(function (accessLog) { - that.formatTime(accessLog) - if (typeof (resp.data.regions[accessLog.remoteAddr]) == "string") { - accessLog.region = resp.data.regions[accessLog.remoteAddr] - } else { - accessLog.region = "" - } - if (accessLog.firewallRuleSetId > 0 && typeof (resp.data.wafInfos[accessLog.firewallRuleSetId]) == "object") { - accessLog.wafInfo = resp.data.wafInfos[accessLog.firewallRuleSetId] - } else { - accessLog.wafInfo = null - } - }) + let newAccessLogs = resp.data.accessLogs || [] + this.accessLogs = this.mergeAccessLogs(newAccessLogs, resp.data.regions || {}, resp.data.wafInfos || {}) let max = 100 if (this.accessLogs.length > max) { this.accessLogs = this.accessLogs.slice(0, max) } this.hasMore = resp.data.hasMore - this.requestId = resp.data.requestId + if (typeof resp.data.requestId == "string" && resp.data.requestId.length > 0) { + this.requestId = resp.data.requestId + } }) .done(function () { if (!this.isLoaded) { @@ -79,4 +107,4 @@ Tea.context(function () { } } } -}) \ No newline at end of file +}) diff --git a/EdgeCommon/go.mod b/EdgeCommon/go.mod index a81dff5..c368785 100644 --- a/EdgeCommon/go.mod +++ b/EdgeCommon/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/kr/text v0.2.0 // indirect + github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 // indirect github.com/oschwald/maxminddb-golang v1.13.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect diff --git a/EdgeCommon/go.sum b/EdgeCommon/go.sum index 81aeb7a..c0addca 100644 --- a/EdgeCommon/go.sum +++ b/EdgeCommon/go.sum @@ -17,6 +17,8 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 h1:ouxvoYN6WL482nK1zS1IplyniAut3G4D+0N5JS0YTuw= +github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879/go.mod h1:+mNMTBuDMdEGhWzoQgc6kBdqeaQpWh5ba8zqmp2MxCU= github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= diff --git a/EdgeCommon/pkg/systemconfigs/clickhouse_setting.go b/EdgeCommon/pkg/systemconfigs/clickhouse_setting.go index f6cd09d..e488e44 100644 --- a/EdgeCommon/pkg/systemconfigs/clickhouse_setting.go +++ b/EdgeCommon/pkg/systemconfigs/clickhouse_setting.go @@ -1,4 +1,3 @@ -//go:build plus package systemconfigs diff --git a/EdgeNode/.gitignore b/EdgeNode/.gitignore index e69de29..e3ed1cd 100644 --- a/EdgeNode/.gitignore +++ b/EdgeNode/.gitignore @@ -0,0 +1,16 @@ +# Windows local development +*_windows.go +configs/api_node.yaml + +# IDE +.idea/ +.vscode/ + +# Build binaries +bin/ + +# Runtime Data +data/ +configs/node.json +logs/ +opt/ diff --git a/EdgeNode/internal/accesslogs/ingest_log.go b/EdgeNode/internal/accesslogs/ingest_log.go index 6f64e6a..483f1cf 100644 --- a/EdgeNode/internal/accesslogs/ingest_log.go +++ b/EdgeNode/internal/accesslogs/ingest_log.go @@ -102,7 +102,7 @@ func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog, Host: l.GetHost(), IP: l.GetRawRemoteAddr(), Method: l.GetRequestMethod(), - Path: l.GetRequestPath(), + Path: l.GetRequestURI(), // 使用 RequestURI 以包含查询参数 Status: l.GetStatus(), BytesIn: l.GetRequestLength(), BytesOut: l.GetBytesSent(), diff --git a/EdgeNode/internal/apps/app_cmd.go b/EdgeNode/internal/apps/app_cmd.go index cad4bd1..9f3fb94 100644 --- a/EdgeNode/internal/apps/app_cmd.go +++ b/EdgeNode/internal/apps/app_cmd.go @@ -15,7 +15,6 @@ import ( "runtime" "strconv" "strings" - "syscall" "time" ) @@ -197,10 +196,7 @@ func (this *AppCmd) runStart() { _ = os.Setenv("EdgeBackground", "on") var cmd = exec.Command(this.exe()) - cmd.SysProcAttr = &syscall.SysProcAttr{ - Foreground: false, - Setsid: true, - } + configureSysProcAttr(cmd) err := cmd.Start() if err != nil { diff --git a/EdgeNode/internal/apps/app_utils_unix.go b/EdgeNode/internal/apps/app_utils_unix.go new file mode 100644 index 0000000..8911d18 --- /dev/null +++ b/EdgeNode/internal/apps/app_utils_unix.go @@ -0,0 +1,15 @@ +//go:build !windows + +package apps + +import ( + "os/exec" + "syscall" +) + +func configureSysProcAttr(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Foreground: false, + Setsid: true, + } +} diff --git a/EdgeNode/internal/caches/manager.go b/EdgeNode/internal/caches/manager.go index c13457f..9197680 100644 --- a/EdgeNode/internal/caches/manager.go +++ b/EdgeNode/internal/caches/manager.go @@ -1,7 +1,6 @@ package caches import ( - "fmt" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs" "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared" teaconst "github.com/TeaOSLab/EdgeNode/internal/const" @@ -10,7 +9,6 @@ import ( memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem" "github.com/iwind/TeaGo/lists" "github.com/iwind/TeaGo/types" - "golang.org/x/sys/unix" "strconv" "sync" ) @@ -191,44 +189,6 @@ func (this *Manager) StorageMap() map[int64]StorageInterface { return this.storageMap } -// TotalDiskSize 消耗的磁盘尺寸 -func (this *Manager) TotalDiskSize() int64 { - this.locker.RLock() - defer this.locker.RUnlock() - - var total = int64(0) - var sidMap = map[string]bool{} // partition sid => bool - for _, storage := range this.storageMap { - // 这里不能直接用 storage.TotalDiskSize() 相加，因为多个缓存策略缓存目录可能处在同一个分区目录下 - fileStorage, ok := storage.(*FileStorage) - if ok { - var options = fileStorage.options // copy - if options != nil { - var dir = options.Dir // copy - if len(dir) == 0 { - continue - } - var stat = &unix.Statfs_t{} - err := unix.Statfs(dir, stat) - if err != nil { - continue - } - var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1]) - if sidMap[sid] { - continue - } - sidMap[sid] = true - total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin - } - } - } - - if total < 0 { - total = 0 - } - - return total -} // TotalMemorySize 消耗的内存尺寸 func (this *Manager) TotalMemorySize() int64 { diff --git a/EdgeNode/internal/caches/manager_unix.go b/EdgeNode/internal/caches/manager_unix.go new file mode 100644 index 0000000..c3ce62c --- /dev/null +++ b/EdgeNode/internal/caches/manager_unix.go @@ -0,0 +1,48 @@ +// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !windows + +package caches + +import ( + "fmt" + "golang.org/x/sys/unix" +) + +// TotalDiskSize 消耗的磁盘尺寸 +func (this *Manager) TotalDiskSize() int64 { + this.locker.RLock() + defer this.locker.RUnlock() + + var total = int64(0) + var sidMap = map[string]bool{} // partition sid => bool + for _, storage := range this.storageMap { + // 这里不能直接用 storage.TotalDiskSize() 相加，因为多个缓存策略缓存目录可能处在同一个分区目录下 + fileStorage, ok := storage.(*FileStorage) + if ok { + var options = fileStorage.options // copy + if options != nil { + var dir = options.Dir // copy + if len(dir) == 0 { + continue + } + var stat = &unix.Statfs_t{} + err := unix.Statfs(dir, stat) + if err != nil { + continue + } + var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1]) + if sidMap[sid] { + continue + } + sidMap[sid] = true + total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin + } + } + } + + if total < 0 { + total = 0 + } + + return total +} diff --git a/EdgeNode/internal/caches/storage_file.go b/EdgeNode/internal/caches/storage_file.go index ede7439..8053592 100644 --- a/EdgeNode/internal/caches/storage_file.go +++ b/EdgeNode/internal/caches/storage_file.go @@ -35,7 +35,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "time" ) @@ -736,7 +735,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea // 尝试锁定，如果锁定失败，则直接返回 fsutils.WriterLimiter.Ack() - err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + err = tryLockFile(int(writer.Fd())) fsutils.WriterLimiter.Release() if err != nil { removeOnFailure = false diff --git a/EdgeNode/internal/caches/storage_utils_unix.go b/EdgeNode/internal/caches/storage_utils_unix.go new file mode 100644 index 0000000..6d27504 --- /dev/null +++ b/EdgeNode/internal/caches/storage_utils_unix.go @@ -0,0 +1,9 @@ +//go:build !windows + +package caches + +import "syscall" + +func tryLockFile(fd int) error { + return syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB) +} diff --git a/EdgeNode/internal/nodes/http_writer.go b/EdgeNode/internal/nodes/http_writer.go index a899ece..d045f99 100644 --- a/EdgeNode/internal/nodes/http_writer.go +++ b/EdgeNode/internal/nodes/http_writer.go @@ -19,11 +19,8 @@ import ( "github.com/TeaOSLab/EdgeNode/internal/utils/writers" _ "github.com/biessek/golang-ico" "github.com/iwind/TeaGo/types" - "github.com/iwind/gowebp" _ "golang.org/x/image/bmp" _ "golang.org/x/image/webp" - "image" - "image/gif" _ "image/jpeg" _ "image/png" "io" @@ -1045,169 +1042,6 @@ func (this *HTTPWriter) calculateStaleLife() int { return staleLife } -// 结束WebP -func (this *HTTPWriter) finishWebP() { - // 处理WebP - if this.webpIsEncoding { - atomic.AddInt32(&webPThreads, 1) - defer func() { - atomic.AddInt32(&webPThreads, -1) - }() - - var webpCacheWriter caches.Writer - - // 准备WebP Cache - if this.cacheReader != nil || this.cacheWriter != nil { - var cacheKey = "" - var expiredAt int64 = 0 - - if this.cacheReader != nil { - cacheKey = this.req.cacheKey + caches.SuffixWebP - expiredAt = this.cacheReader.ExpiresAt() - } else if this.cacheWriter != nil { - cacheKey = this.cacheWriter.Key() + caches.SuffixWebP - expiredAt = this.cacheWriter.ExpiredAt() - } - - webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false) - if webpCacheWriter != nil { - // 写入Header - for k, v := range this.Header() { - if this.shouldIgnoreHeader(k) { - continue - } - - // 这里是原始的数据，不需要内容编码 - if k == "Content-Encoding" || k == "Transfer-Encoding" { - continue - } - for _, v1 := range v { - _, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n")) - if err != nil { - remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error()) - _ = webpCacheWriter.Discard() - webpCacheWriter = nil - break - } - } - } - - if webpCacheWriter != nil { - var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter) - teeWriter.OnFail(func(err error) { - if webpCacheWriter != nil { - _ = webpCacheWriter.Discard() - } - webpCacheWriter = nil - }) - this.writer = teeWriter - } - } - } - - var reader = readers.NewBytesCounterReader(this.rawReader) - - var imageData image.Image - var gifImage *gif.GIF - var isGif = strings.Contains(this.webpOriginContentType, "image/gif") - var err error - if isGif { - gifImage, err = gif.DecodeAll(reader) - if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) { - webPIgnoreURLSet.Push(this.req.URL()) - return - } - } else { - imageData, _, err = image.Decode(reader) - if imageData != nil { - var bound = imageData.Bounds() - if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension { - webPIgnoreURLSet.Push(this.req.URL()) - return - } - } - } - - if err != nil { - // 发生了错误终止处理 - webPIgnoreURLSet.Push(this.req.URL()) - return - } - - var f = types.Float32(this.webpQuality) - if f <= 0 || f > 100 { - if this.size > (8<<20) || this.size <= 0 { - f = 30 - } else if this.size > (1 << 20) { - f = 50 - } else if this.size > (128 << 10) { - f = 60 - } else { - f = 75 - } - } - - if imageData != nil { - err = gowebp.Encode(this.writer, imageData, &gowebp.Options{ - Lossless: false, - Quality: f, - Exact: true, - }) - } else if gifImage != nil { - var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount) - - anim.WebPAnimEncoderOptions.SetKmin(9) - anim.WebPAnimEncoderOptions.SetKmax(17) - var webpConfig = gowebp.NewWebpConfig() - //webpConfig.SetLossless(1) - webpConfig.SetQuality(f) - - var timeline = 0 - var lastErr error - for i, img := range gifImage.Image { - err = anim.AddFrame(img, timeline, webpConfig) - if err != nil { - // 有错误直接跳过 - lastErr = err - err = nil - } - timeline += gifImage.Delay[i] * 10 - } - if lastErr != nil { - remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error()) - } - err = anim.AddFrame(nil, timeline, webpConfig) - - if err == nil { - err = anim.Encode(this.writer) - } - - anim.ReleaseMemory() - } - - if err != nil && !this.req.canIgnore(err) { - remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error()) - } - - if err == nil && webpCacheWriter != nil { - err = webpCacheWriter.Close() - if err != nil { - _ = webpCacheWriter.Discard() - } else { - this.cacheStorage.AddToList(&caches.Item{ - Type: webpCacheWriter.ItemType(), - Key: webpCacheWriter.Key(), - ExpiresAt: webpCacheWriter.ExpiredAt(), - StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()), - HeaderSize: webpCacheWriter.HeaderSize(), - BodySize: webpCacheWriter.BodySize(), - Host: this.req.ReqHost, - ServerId: this.req.ReqServer.Id, - }) - } - } - } -} // 结束缓存相关处理 func (this *HTTPWriter) finishCache() { diff --git a/EdgeNode/internal/nodes/http_writer_ext_unix.go b/EdgeNode/internal/nodes/http_writer_ext_unix.go new file mode 100644 index 0000000..d7f2c22 --- /dev/null +++ b/EdgeNode/internal/nodes/http_writer_ext_unix.go @@ -0,0 +1,181 @@ +// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. +//go:build !windows + +package nodes + +import ( + "github.com/TeaOSLab/EdgeNode/internal/caches" + "github.com/TeaOSLab/EdgeNode/internal/remotelogs" + "github.com/TeaOSLab/EdgeNode/internal/utils/readers" + "github.com/TeaOSLab/EdgeNode/internal/utils/writers" + "github.com/iwind/TeaGo/types" + "github.com/iwind/gowebp" + "image" + "image/gif" + "strings" + "sync/atomic" +) + +// 结束WebP +func (this *HTTPWriter) finishWebP() { + // 处理WebP + if this.webpIsEncoding { + atomic.AddInt32(&webPThreads, 1) + defer func() { + atomic.AddInt32(&webPThreads, -1) + }() + + var webpCacheWriter caches.Writer + + // 准备WebP Cache + if this.cacheReader != nil || this.cacheWriter != nil { + var cacheKey = "" + var expiredAt int64 = 0 + + if this.cacheReader != nil { + cacheKey = this.req.cacheKey + caches.SuffixWebP + expiredAt = this.cacheReader.ExpiresAt() + } else if this.cacheWriter != nil { + cacheKey = this.cacheWriter.Key() + caches.SuffixWebP + expiredAt = this.cacheWriter.ExpiredAt() + } + + webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false) + if webpCacheWriter != nil { + // 写入Header + for k, v := range this.Header() { + if this.shouldIgnoreHeader(k) { + continue + } + + // 这里是原始的数据，不需要内容编码 + if k == "Content-Encoding" || k == "Transfer-Encoding" { + continue + } + for _, v1 := range v { + _, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n")) + if err != nil { + remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error()) + _ = webpCacheWriter.Discard() + webpCacheWriter = nil + break + } + } + } + + if webpCacheWriter != nil { + var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter) + teeWriter.OnFail(func(err error) { + if webpCacheWriter != nil { + _ = webpCacheWriter.Discard() + } + webpCacheWriter = nil + }) + this.writer = teeWriter + } + } + } + + var reader = readers.NewBytesCounterReader(this.rawReader) + + var imageData image.Image + var gifImage *gif.GIF + var isGif = strings.Contains(this.webpOriginContentType, "image/gif") + var err error + if isGif { + gifImage, err = gif.DecodeAll(reader) + if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) { + webPIgnoreURLSet.Push(this.req.URL()) + return + } + } else { + imageData, _, err = image.Decode(reader) + if imageData != nil { + var bound = imageData.Bounds() + if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension { + webPIgnoreURLSet.Push(this.req.URL()) + return + } + } + } + + if err != nil { + // 发生了错误终止处理 + webPIgnoreURLSet.Push(this.req.URL()) + return + } + + var f = types.Float32(this.webpQuality) + if f <= 0 || f > 100 { + if this.size > (8<<20) || this.size <= 0 { + f = 30 + } else if this.size > (1 << 20) { + f = 50 + } else if this.size > (128 << 10) { + f = 60 + } else { + f = 75 + } + } + + if imageData != nil { + err = gowebp.Encode(this.writer, imageData, &gowebp.Options{ + Lossless: false, + Quality: f, + Exact: true, + }) + } else if gifImage != nil { + var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount) + + anim.WebPAnimEncoderOptions.SetKmin(9) + anim.WebPAnimEncoderOptions.SetKmax(17) + var webpConfig = gowebp.NewWebpConfig() + //webpConfig.SetLossless(1) + webpConfig.SetQuality(f) + + var timeline = 0 + var lastErr error + for i, img := range gifImage.Image { + err = anim.AddFrame(img, timeline, webpConfig) + if err != nil { + // 有错误直接跳过 + lastErr = err + err = nil + } + timeline += gifImage.Delay[i] * 10 + } + if lastErr != nil { + remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error()) + } + err = anim.AddFrame(nil, timeline, webpConfig) + + if err == nil { + err = anim.Encode(this.writer) + } + + anim.ReleaseMemory() + } + + if err != nil && !this.req.canIgnore(err) { + remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error()) + } + + if err == nil && webpCacheWriter != nil { + err = webpCacheWriter.Close() + if err != nil { + _ = webpCacheWriter.Discard() + } else { + this.cacheStorage.AddToList(&caches.Item{ + Type: webpCacheWriter.ItemType(), + Key: webpCacheWriter.Key(), + ExpiresAt: webpCacheWriter.ExpiredAt(), + StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()), + HeaderSize: webpCacheWriter.HeaderSize(), + BodySize: webpCacheWriter.BodySize(), + Host: this.req.ReqHost, + ServerId: this.req.ReqServer.Id, + }) + } + } + } +} diff --git a/EdgeNode/internal/nodes/node_panic.go b/EdgeNode/internal/nodes/node_panic_unix.go similarity index 96% rename from EdgeNode/internal/nodes/node_panic.go rename to EdgeNode/internal/nodes/node_panic_unix.go index 69de93b..87688be 100644 --- a/EdgeNode/internal/nodes/node_panic.go +++ b/EdgeNode/internal/nodes/node_panic_unix.go @@ -1,6 +1,5 @@ // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved. -//go:build !arm64 -// +build !arm64 +//go:build !arm64 && !windows package nodes diff --git a/EdgeNode/internal/nodes/node_status_executor_windows.go b/EdgeNode/internal/nodes/node_status_executor_windows.go index 51c61d5..a37f754 100644 --- a/EdgeNode/internal/nodes/node_status_executor_windows.go +++ b/EdgeNode/internal/nodes/node_status_executor_windows.go @@ -4,8 +4,7 @@ package nodes import ( - "context" - "github.com/shirou/gopsutil/v3/cpu" + "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/shirou/gopsutil/v3/mem" "math" "sync" @@ -21,7 +20,7 @@ var windowsLoadValues = []*WindowsLoadValue{} var windowsLoadLocker = &sync.Mutex{} // 更新内存 -func (this *NodeStatusExecutor) updateMem(status *NodeStatus) { +func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) { stat, err := mem.VirtualMemory() if err != nil { status.Error = err.Error() @@ -32,14 +31,16 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) { } // 更新负载 -func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) { +func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) { timestamp := time.Now().Unix() currentLoad := 0 - info, err := cpu.ProcInfo() - if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 { - currentLoad = int(info[0].ProcessorQueueLength) - } + /* + info, err := cpu.ProcInfo() + if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 { + currentLoad = int(info[0].ProcessorQueueLength) + } + */ // 删除15分钟之前的数据 windowsLoadLocker.Lock() @@ -93,9 +94,11 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) { windowsLoadLocker.Unlock() // 在老Windows上不显示错误 - if err == context.DeadlineExceeded { - err = nil - } + /* + if err == context.DeadlineExceeded { + err = nil + } + */ status.Load1m = load1 status.Load5m = load5 status.Load15m = load15 diff --git a/EdgeNode/internal/utils/fs/locker.go b/EdgeNode/internal/utils/fs/locker_unix.go similarity index 98% rename from EdgeNode/internal/utils/fs/locker.go rename to EdgeNode/internal/utils/fs/locker_unix.go index ae00f8c..21e4f02 100644 --- a/EdgeNode/internal/utils/fs/locker.go +++ b/EdgeNode/internal/utils/fs/locker_unix.go @@ -1,4 +1,5 @@ // Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !windows package fsutils diff --git a/EdgeNode/internal/utils/fs/stat.go b/EdgeNode/internal/utils/fs/stat.go index 0cf59ee..3eed657 100644 --- a/EdgeNode/internal/utils/fs/stat.go +++ b/EdgeNode/internal/utils/fs/stat.go @@ -4,20 +4,9 @@ package fsutils import ( "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" - "golang.org/x/sys/unix" "sync" ) -// StatDevice device contains the path -func StatDevice(path string) (*StatResult, error) { - var stat = &unix.Statfs_t{} - err := unix.Statfs(path, stat) - if err != nil { - return nil, err - } - return NewStatResult(stat), nil -} - var locker = &sync.RWMutex{} var cacheMap = map[string]*StatResult{} // path => StatResult @@ -44,38 +33,3 @@ func StatDeviceCache(path string) (*StatResult, error) { cacheMap[path] = stat return stat, nil } - -type StatResult struct { - rawStat *unix.Statfs_t - blockSize uint64 - - updatedAt int64 -} - -func NewStatResult(rawStat *unix.Statfs_t) *StatResult { - var blockSize = rawStat.Bsize - if blockSize < 0 { - blockSize = 0 - } - - return &StatResult{ - rawStat: rawStat, - blockSize: uint64(blockSize), - updatedAt: fasttime.Now().Unix(), - } -} - -func (this *StatResult) FreeSize() uint64 { - return this.rawStat.Bfree * this.blockSize -} - -func (this *StatResult) TotalSize() uint64 { - return this.rawStat.Blocks * this.blockSize -} - -func (this *StatResult) UsedSize() uint64 { - if this.rawStat.Bfree <= this.rawStat.Blocks { - return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize - } - return 0 -} diff --git a/EdgeNode/internal/utils/fs/stat_unix.go b/EdgeNode/internal/utils/fs/stat_unix.go new file mode 100644 index 0000000..0841d7f --- /dev/null +++ b/EdgeNode/internal/utils/fs/stat_unix.go @@ -0,0 +1,54 @@ +// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !windows + +package fsutils + +import ( + "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" + "golang.org/x/sys/unix" +) + +// StatDevice device contains the path +func StatDevice(path string) (*StatResult, error) { + var stat = &unix.Statfs_t{} + err := unix.Statfs(path, stat) + if err != nil { + return nil, err + } + return NewStatResult(stat), nil +} + +type StatResult struct { + rawStat *unix.Statfs_t + blockSize uint64 + + updatedAt int64 +} + +func NewStatResult(rawStat *unix.Statfs_t) *StatResult { + var blockSize = rawStat.Bsize + if blockSize < 0 { + blockSize = 0 + } + + return &StatResult{ + rawStat: rawStat, + blockSize: uint64(blockSize), + updatedAt: fasttime.Now().Unix(), + } +} + +func (this *StatResult) FreeSize() uint64 { + return this.rawStat.Bfree * this.blockSize +} + +func (this *StatResult) TotalSize() uint64 { + return this.rawStat.Blocks * this.blockSize +} + +func (this *StatResult) UsedSize() uint64 { + if this.rawStat.Bfree <= this.rawStat.Blocks { + return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize + } + return 0 +} diff --git a/EdgeNode/internal/utils/mmap/flags_others.go b/EdgeNode/internal/utils/mmap/flags_others.go index a140756..081f6e1 100644 --- a/EdgeNode/internal/utils/mmap/flags_others.go +++ b/EdgeNode/internal/utils/mmap/flags_others.go @@ -1,5 +1,5 @@ // Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . -//go:build !linux +//go:build !linux && !windows package mmap diff --git a/EdgeNode/internal/utils/mmap/mmap_ext.go b/EdgeNode/internal/utils/mmap/mmap_ext_unix.go similarity index 98% rename from EdgeNode/internal/utils/mmap/mmap_ext.go rename to EdgeNode/internal/utils/mmap/mmap_ext_unix.go index ab96af9..dee18f3 100644 --- a/EdgeNode/internal/utils/mmap/mmap_ext.go +++ b/EdgeNode/internal/utils/mmap/mmap_ext_unix.go @@ -1,5 +1,5 @@ // Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . -//go:build plus +//go:build plus && !windows package mmap diff --git a/EdgeNode/internal/utils/net.go b/EdgeNode/internal/utils/net_unix.go similarity index 77% rename from EdgeNode/internal/utils/net.go rename to EdgeNode/internal/utils/net_unix.go index 0d1fd0e..16dd53b 100644 --- a/EdgeNode/internal/utils/net.go +++ b/EdgeNode/internal/utils/net_unix.go @@ -1,5 +1,5 @@ -//go:build !freebsd -// +build !freebsd +//go:build !freebsd && !windows +// +build !freebsd,!windows package utils @@ -15,7 +15,7 @@ func ListenReuseAddr(network string, addr string) (net.Listener, error) { config := &net.ListenConfig{ Control: func(network, address string, c syscall.RawConn) error { return c.Control(func(fd uintptr) { - err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1) + err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1) if err != nil { logs.Println("[LISTEN]" + err.Error()) } diff --git a/EdgeNode/internal/waf/injectionutils/utils_sqli.go b/EdgeNode/internal/waf/injectionutils/utils_sqli.go index a8acdc5..ebacd4e 100644 --- a/EdgeNode/internal/waf/injectionutils/utils_sqli.go +++ b/EdgeNode/internal/waf/injectionutils/utils_sqli.go @@ -8,6 +8,8 @@ package injectionutils #include #include */ +//go:build cgo + import "C" import ( "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" diff --git a/EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go b/EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go new file mode 100644 index 0000000..f6210b6 --- /dev/null +++ b/EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go @@ -0,0 +1,18 @@ +// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !cgo + +package injectionutils + +import ( + "github.com/TeaOSLab/EdgeNode/internal/waf/utils" +) + +// DetectSQLInjectionCache detect sql injection in string with cache +func DetectSQLInjectionCache(input string, isStrict bool, cacheLife utils.CacheLife) bool { + return false +} + +// DetectSQLInjection detect sql injection in string +func DetectSQLInjection(input string, isStrict bool) bool { + return false +} diff --git a/EdgeNode/internal/waf/injectionutils/utils_xss.go b/EdgeNode/internal/waf/injectionutils/utils_xss.go index 531ec36..3d3d0c2 100644 --- a/EdgeNode/internal/waf/injectionutils/utils_xss.go +++ b/EdgeNode/internal/waf/injectionutils/utils_xss.go @@ -8,6 +8,8 @@ package injectionutils #include #include */ +//go:build cgo + import "C" import ( "github.com/TeaOSLab/EdgeNode/internal/utils/fasttime" diff --git a/EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go b/EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go new file mode 100644 index 0000000..16a3daa --- /dev/null +++ b/EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go @@ -0,0 +1,17 @@ +// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn . +//go:build !cgo + +package injectionutils + +import ( + "github.com/TeaOSLab/EdgeNode/internal/waf/utils" +) + +func DetectXSSCache(input string, isStrict bool, cacheLife utils.CacheLife) bool { + return false +} + +// DetectXSS detect XSS in string +func DetectXSS(input string, isStrict bool) bool { + return false +} diff --git a/deploy/fluent-bit/.gitignore b/deploy/fluent-bit/.gitignore new file mode 100644 index 0000000..7401f92 --- /dev/null +++ b/deploy/fluent-bit/.gitignore @@ -0,0 +1,2 @@ +fluent-bit-windows.conf +clickhouse-upstream-windows.conf diff --git a/deploy/fluent-bit/logs.db b/deploy/fluent-bit/logs.db new file mode 100644 index 0000000000000000000000000000000000000000..e06183cb885e70bf577e7498358a1946719bbda1 GIT binary patch literal 8192 zcmeI%F-yZh6bJB2T1=M`H>cv;s-Q(w1Unh^l+xI0nuvu!Ih#uf#0HW>XI-4cuOuq? zY5V|g-i1KZr7kXocjPX)clpERmnmd$Wk-TTn51`1z#ds8x=xM&2qAjW01BW03ZMWApa2S>z>)|&b~R(I+0>pq!ER$7 zO-F)7@hFU9p3P146Ng$Z1=l*ZDaK=doz8{j453ShTL4pb$JMe+ zZ(Qj0W%q1b){`*IxBw_pvJy>`K;1g4j8o184@#9Z5#01BW03ZMWApa2T|r$AdXh*w#u)@a+mXQ>}26Q2hYo)6pV zyW3~uG0(D=T+zPY@?YZB_4nxc8ulv&>DLx!A6Te@Me0MP4s?SI7N>ZcCTS`6sdDnE N552{?*-p#b`~~$&iWvX^ literal 0 HcmV?d00001 diff --git a/deploy/fluent-bit/logs.db-shm b/deploy/fluent-bit/logs.db-shm new file mode 100644 index 0000000000000000000000000000000000000000..0850e154714402fa10ebc8cbbdfd674dc7fedd07 GIT binary patch literal 32768 zcmeI)IZlHy5C+hXeTNXXuU5li8t5oF0Q`wigd$PUqBqib zY|HY;)`x1Qw?EaX z-qd+D<(-R~GPy;4GDUrg9xJ-8}MVZb7E3 z@VIB^OrF|t@PDV?={R+Mr-IDY$3K;=uCA&+=G?lDbM0Su_=Mx1`}UO{`!d@Pey0ok ztL1?`iwAT&tP8ZuxCjs+K!5-N0t5&UAV45(fv_&{!=cCi!u>t%0=>tUeO$9KZ@ey$ z_Vj)C1PBlyK!5-N0t5&UAkZ!l)&)-65J4BXH2=WDzdm#8XkEZh009C72oNAZfB*pk z1PCNA5Y`0-H%8C}MwS;H_9ZfB*pk1PBlyK!5-N0?7-6b%7i6Bj^JAAGu-s3&ju5*9DSaKL7#*2oNAZ zfB*pk1PBla>jF2Q96=Z8y6~aPUi`!DM|A=7CqRGz0RjXF5FkK+0De*y#u z5FkK+009C72oOkKAgl{4I}$+`7`|-fj@l`GC+Y&puO9#b0t5&UAV7cs0RjXFgmr<2 zvIx4s&f&Q|4qn*%OJ9_MYR2N8o z{Qw9MAV7cs0RjXF5FkJxtP5<5;e7-%YhRl)V{E@px`6o;AV7cs0RjXF5FkK+K=J}% zU7+Rt2<{iieZKk4g4PG`&;^oTKL7#*2oNAZfB*pk1PBla>H@iWEfI8qF4Nl{E#C0^ zjk@1PBlyK!5-N0tAv52S&G-Je~sNf$_d{Qw9M zAV7cs0RjXF5FkJxtP9+BD1t6jE`R z5p;pkzk7D>H)|*4=>o~G9{>RY1PBlyK!5-N0t5(zb%DB*Bj^HWoY}Ov+rxi6s0)}s z0RjXF5FkK+009C72qZ5M)&+k1X#`y$TeteHf!A*<(FKxUKL7#*2oNAZfB*pk1PBla z>jEnVM$iS`?RCnm^@W#SpbMBk0RjXF5FkK+009C72qZ5M)&*YO6+stRv~pN^L+kXz zx^sE>QoMPYw>=I`Si3Ao=wJAV7cs0RjXF z5FkK+0D-VBu)QLJE^y+w{A(`z^P~~FfcX<3K!5-N0t5&UAV7dX@&aL9;K1V%bb(!i zOSd%){JLBhNPhhQ2oNAZfB*pk1PBlyKp?0K*ABIp7+&Bbj?mt;QE1(IJs00IOE5FkK+009C7 z2oMPC0<&(9pbKO_IW+IozkTttE@1uy2oNAZfB*pk1PBlyki0-x7r6712)aQ1(UNmF z&uA{!1(IJs00IOE5FkK+009C72oMPC0uOv1K^JJO+SR4>zBPMw0rMw7fB*pk1PBly zK!5;&BWAo=kFAV7cs z0RjXF5FkK+0D(+c7g(`5f-X>e)zb3*Uv*vRegXR@K!5-N0t5&UAV7csf#e0ky1;u) z5p;o1|6X|S?i=S$)diAYKL7#*2oNAZfB*pk1PBla>jGQGM9>A6ubMdO$UWmrbOG}x zK!5-N0t5&UAV7csf#e0ky1+jxBj^G@Uex`bvCA)9tqUZ-egFgr5FkK+009C72oN9; N)&=%n@ICAT{{c8;!BGGJ literal 0 HcmV?d00001