日常查询由mysql改为clickhouse

This commit is contained in:
robin
2026-02-08 02:00:51 +08:00
parent bc223fd1aa
commit b7388d83b0
43 changed files with 657 additions and 353 deletions

View File

@@ -1,5 +0,0 @@
user: root
password: 123456
host: 127.0.0.1:3307
database: db_edge
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]

10
EdgeAPI/.gitignore vendored
View File

@@ -28,3 +28,13 @@ logs/
# 临时文件
*.tmp
.DS_Store
# Runtime Data
data/
configs/api.yaml
configs/db.yaml
db.yaml
db.yaml.link.bak
# Build Artifacts
bin/

View File

@@ -1,6 +1,6 @@
user: root
password: 123456
host: 127.0.0.1
pord: 3307
pord: 3308
database: db_edge
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]

View File

@@ -56,6 +56,10 @@ type ListFilter struct {
LastRequestId string
ServerIds []int64
NodeIds []int64
// 搜索条件
Keyword string
Ip string
Domain string
}
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
@@ -126,10 +130,35 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
}
where := strings.Join(conditions, " AND ")
orderDir := "ASC"
// 搜索条件
if f.Keyword != "" {
keyword := escapeString(f.Keyword)
// 在 host, path, ip, ua 中模糊搜索
conditions = append(conditions, fmt.Sprintf("(host LIKE '%%%s%%' OR path LIKE '%%%s%%' OR ip LIKE '%%%s%%' OR ua LIKE '%%%s%%')", keyword, keyword, keyword, keyword))
}
if f.Ip != "" {
conditions = append(conditions, "ip = '"+escapeString(f.Ip)+"'")
}
if f.Domain != "" {
conditions = append(conditions, "host LIKE '%"+escapeString(f.Domain)+"%'")
}
// 游标分页:使用 trace_id 作为游标
// Reverse=false历史向后翻页查询更早的数据
// Reverse=true实时增量拉新查询更新的数据
if f.LastRequestId != "" {
if f.Reverse {
orderDir = "DESC"
conditions = append(conditions, "trace_id > '"+escapeString(f.LastRequestId)+"'")
} else {
conditions = append(conditions, "trace_id < '"+escapeString(f.LastRequestId)+"'")
}
}
where := strings.Join(conditions, " AND ")
// 默认按时间倒序(最新的在前面),与前端默认行为一致
orderDir := "DESC"
if f.Reverse {
orderDir = "ASC"
}
limit := f.Size
if limit <= 0 {
@@ -138,7 +167,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
if limit > 1000 {
limit = 1000
}
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
table, where, orderBy, limit+1)
@@ -155,6 +184,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
rows = append(rows, r)
}
}
if !f.Reverse {
if len(rows) > int(limit) {
nextCursor = rows[limit].TraceId
rows = rows[:limit]
@@ -162,6 +192,43 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
return rows, nextCursor, nil
}
if len(rows) > int(limit) {
rows = rows[:limit]
}
if len(rows) > 0 {
nextCursor = rows[len(rows)-1].TraceId
}
// 实时模式统一返回为“最新在前”,与前端显示和 MySQL 语义一致。
for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 {
rows[left], rows[right] = rows[right], rows[left]
}
return rows, nextCursor, nil
}
// FindByTraceId 按 trace_id 查询单条日志详情
func (s *LogsIngestStore) FindByTraceId(ctx context.Context, traceId string) (*LogsIngestRow, error) {
if !s.client.IsConfigured() {
return nil, fmt.Errorf("clickhouse: not configured")
}
if traceId == "" {
return nil, nil
}
table := quoteIdent("logs_ingest")
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE trace_id = '%s' LIMIT 1",
table, escapeString(traceId))
var rawRows []map[string]interface{}
if err := s.client.Query(ctx, query, &rawRows); err != nil {
return nil, err
}
if len(rawRows) == 0 {
return nil, nil
}
return mapToLogsIngestRow(rawRows[0]), nil
}
func quoteIdent(name string) string {
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
}
@@ -244,7 +311,7 @@ func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
return r
}
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog列表展示用
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog列表展示用+详情展示
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
if r == nil {
return nil
@@ -259,6 +326,9 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
RemoteAddr: r.IP,
RequestMethod: r.Method,
RequestPath: r.Path,
RequestURI: r.Path, // 前端使用 requestURI 显示完整路径
Scheme: "http", // 默认 http日志中未存储实际值
Proto: "HTTP/1.1", // 默认值,日志中未存储实际值
Status: int32(r.Status),
RequestLength: int64(r.BytesIn),
BytesSent: int64(r.BytesOut),
@@ -273,6 +343,39 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
if r.TimeISO8601() != "" {
a.TimeISO8601 = r.TimeISO8601()
}
// TimeLocal: 用户友好的时间格式 (e.g., "2026-02-07 23:17:12")
if !r.Timestamp.IsZero() {
a.TimeLocal = r.Timestamp.Format("2006-01-02 15:04:05")
}
// 解析请求头 (JSON -> map[string]*pb.Strings)
// ClickHouse 中存储的是 map[string]string 格式
if r.RequestHeaders != "" {
var headers map[string]string
if err := json.Unmarshal([]byte(r.RequestHeaders), &headers); err == nil {
a.Header = make(map[string]*pb.Strings)
for k, v := range headers {
a.Header[k] = &pb.Strings{Values: []string{v}}
}
}
}
// 解析响应头 (JSON -> map[string]*pb.Strings)
if r.ResponseHeaders != "" {
var headers map[string]string
if err := json.Unmarshal([]byte(r.ResponseHeaders), &headers); err == nil {
a.SentHeader = make(map[string]*pb.Strings)
for k, v := range headers {
a.SentHeader[k] = &pb.Strings{Values: []string{v}}
}
}
}
// 请求体
if r.RequestBody != "" {
a.RequestBody = []byte(r.RequestBody)
}
return a
}

View File

@@ -261,3 +261,23 @@ func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs
}
return config, nil
}
// ReadClickHouseConfig 读取ClickHouse配置
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
valueJSON, err := this.ReadSetting(tx, "clickhouseConfig")
if err != nil {
return nil, err
}
if len(valueJSON) == 0 {
return &systemconfigs.ClickHouseSetting{
Port: 8123,
Database: "default",
}, nil
}
var config = &systemconfigs.ClickHouseSetting{}
err = json.Unmarshal(valueJSON, config)
if err != nil {
return nil, err
}
return config, nil
}

View File

@@ -66,25 +66,3 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
return config, nil
}
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
if err != nil {
return nil, err
}
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
if len(valueJSON) == 0 {
return out, nil
}
err = json.Unmarshal(valueJSON, out)
if err != nil {
return nil, err
}
if out.Port <= 0 {
out.Port = 8123
}
if out.Database == "" {
out.Database = "default"
}
return out, nil
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"golang.org/x/sys/unix"
"time"
)
@@ -60,14 +59,7 @@ func CheckHasFreeSpace() bool {
}
LocalDatabaseDataDir = dir
var stat unix.Statfs_t
err = unix.Statfs(dir, &stat)
if err != nil {
return true
}
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
return availableSpace > minFreeSpaceGB
return checkHasFreeSpace(dir)
}
return true
}

View File

@@ -0,0 +1,18 @@
//go:build !windows
package dbutils
import (
"golang.org/x/sys/unix"
)
func checkHasFreeSpace(dir string) bool {
var stat unix.Statfs_t
err := unix.Statfs(dir, &stat)
if err != nil {
return true
}
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
return availableSpace > minFreeSpaceGB
}

View File

@@ -0,0 +1,7 @@
//go:build windows
package dbutils
func checkHasFreeSpace(dir string) bool {
return true
}

View File

@@ -4,8 +4,9 @@
package nodes
import (
"context"
"github.com/shirou/gopsutil/v3/cpu"
//"context"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
//"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/mem"
"math"
"sync"
@@ -21,7 +22,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
var windowsLoadLocker = &sync.Mutex{}
// 更新内存
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
stat, err := mem.VirtualMemory()
if err != nil {
status.Error = err.Error()
@@ -32,14 +33,14 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
}
// 更新负载
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
timestamp := time.Now().Unix()
currentLoad := 0
info, err := cpu.ProcInfo()
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
currentLoad = int(info[0].ProcessorQueueLength)
}
//info, err := cpu.ProcInfo()
//if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
// currentLoad = int(info[0].ProcessorQueueLength)
//}
// 删除15分钟之前的数据
windowsLoadLocker.Lock()
@@ -93,9 +94,9 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
windowsLoadLocker.Unlock()
// 在老Windows上不显示错误
if err == context.DeadlineExceeded {
err = nil
}
//if err == context.DeadlineExceeded {
// err = nil
//}
status.Load1m = load1
status.Load5m = load5
status.Load15m = load15

View File

@@ -144,6 +144,9 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
NodeId: req.NodeId,
ClusterId: req.NodeClusterId,
LastRequestId: req.RequestId,
Keyword: req.Keyword,
Ip: req.Ip,
Domain: req.Domain,
}
if req.ServerId > 0 {
f.ServerIds = []int64{req.ServerId}
@@ -216,7 +219,10 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
}
}
hasMore := nextCursor != ""
hasMore := false
if !req.Reverse {
hasMore = nextCursor != ""
}
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: result,
AccessLogs: result,
@@ -233,6 +239,28 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
return nil, err
}
// 优先从 ClickHouse 查询
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() {
row, err := store.FindByTraceId(ctx, req.RequestId)
if err != nil {
return nil, err
}
if row != nil {
// 检查权限
if userId > 0 {
var tx = this.NullTx()
err = models.SharedServerDAO.CheckUserServer(tx, userId, int64(row.ServerId))
if err != nil {
return nil, err
}
}
a := clickhouse.RowToPB(row)
return &pb.FindHTTPAccessLogResponse{HttpAccessLog: a}, nil
}
}
// 如果 ClickHouse 未配置或未找到,则回退到 MySQL
var tx = this.NullTx()
accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)

View File

@@ -11,7 +11,7 @@ import (
)
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
return false
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {

12
EdgeAdmin/.gitignore vendored
View File

@@ -0,0 +1,12 @@
部署启动手册.md
# Runtime Data
data/
logs/
# Local Configs
configs/api_admin.yaml
configs/server.yaml
# Build Artifacts
bin/

View File

@@ -7,6 +7,47 @@ Tea.context(function () {
this.accessLogs = []
this.isLoaded = false
this.mergeAccessLogs = function (newAccessLogs, regions, wafInfos) {
let mergedLogs = newAccessLogs.concat(this.accessLogs)
let result = []
let seenRequestIds = {}
let that = this
mergedLogs.forEach(function (accessLog) {
if (accessLog == null) {
return
}
let requestId = accessLog.requestId || ""
if (requestId.length > 0) {
if (seenRequestIds[requestId] === true) {
return
}
seenRequestIds[requestId] = true
}
that.formatTime(accessLog)
let region = regions[accessLog.remoteAddr]
if (typeof (region) == "string") {
accessLog.region = region
} else if (typeof accessLog.region != "string") {
accessLog.region = ""
}
let wafInfo = wafInfos[accessLog.firewallRuleSetId]
if (accessLog.firewallRuleSetId > 0 && typeof (wafInfo) == "object") {
accessLog.wafInfo = wafInfo
} else if (typeof accessLog.wafInfo == "undefined") {
accessLog.wafInfo = null
}
result.push(accessLog)
})
return result
}
this.load = function () {
// 如果有弹窗时,暂时不更新
if (teaweb.hasPopup()) {
@@ -27,30 +68,17 @@ Tea.context(function () {
nodeId: this.nodeId
})
.success(function (resp) {
this.accessLogs = resp.data.accessLogs.concat(this.accessLogs)
// 添加区域信息
let that = this
this.accessLogs.forEach(function (accessLog) {
that.formatTime(accessLog)
if (typeof (resp.data.regions[accessLog.remoteAddr]) == "string") {
accessLog.region = resp.data.regions[accessLog.remoteAddr]
} else {
accessLog.region = ""
}
if (accessLog.firewallRuleSetId > 0 && typeof (resp.data.wafInfos[accessLog.firewallRuleSetId]) == "object") {
accessLog.wafInfo = resp.data.wafInfos[accessLog.firewallRuleSetId]
} else {
accessLog.wafInfo = null
}
})
let newAccessLogs = resp.data.accessLogs || []
this.accessLogs = this.mergeAccessLogs(newAccessLogs, resp.data.regions || {}, resp.data.wafInfos || {})
let max = 100
if (this.accessLogs.length > max) {
this.accessLogs = this.accessLogs.slice(0, max)
}
this.hasMore = resp.data.hasMore
if (typeof resp.data.requestId == "string" && resp.data.requestId.length > 0) {
this.requestId = resp.data.requestId
}
})
.done(function () {
if (!this.isLoaded) {

View File

@@ -13,6 +13,7 @@ require (
require (
github.com/kr/text v0.2.0 // indirect
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 // indirect
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect

View File

@@ -17,6 +17,8 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 h1:ouxvoYN6WL482nK1zS1IplyniAut3G4D+0N5JS0YTuw=
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879/go.mod h1:+mNMTBuDMdEGhWzoQgc6kBdqeaQpWh5ba8zqmp2MxCU=
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=

View File

@@ -1,4 +1,3 @@
//go:build plus
package systemconfigs

16
EdgeNode/.gitignore vendored
View File

@@ -0,0 +1,16 @@
# Windows local development
*_windows.go
configs/api_node.yaml
# IDE
.idea/
.vscode/
# Build binaries
bin/
# Runtime Data
data/
configs/node.json
logs/
opt/

View File

@@ -102,7 +102,7 @@ func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog,
Host: l.GetHost(),
IP: l.GetRawRemoteAddr(),
Method: l.GetRequestMethod(),
Path: l.GetRequestPath(),
Path: l.GetRequestURI(), // 使用 RequestURI 以包含查询参数
Status: l.GetStatus(),
BytesIn: l.GetRequestLength(),
BytesOut: l.GetBytesSent(),

View File

@@ -15,7 +15,6 @@ import (
"runtime"
"strconv"
"strings"
"syscall"
"time"
)
@@ -197,10 +196,7 @@ func (this *AppCmd) runStart() {
_ = os.Setenv("EdgeBackground", "on")
var cmd = exec.Command(this.exe())
cmd.SysProcAttr = &syscall.SysProcAttr{
Foreground: false,
Setsid: true,
}
configureSysProcAttr(cmd)
err := cmd.Start()
if err != nil {

View File

@@ -0,0 +1,15 @@
//go:build !windows
package apps
import (
"os/exec"
"syscall"
)
func configureSysProcAttr(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{
Foreground: false,
Setsid: true,
}
}

View File

@@ -1,7 +1,6 @@
package caches
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -10,7 +9,6 @@ import (
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"golang.org/x/sys/unix"
"strconv"
"sync"
)
@@ -191,44 +189,6 @@ func (this *Manager) StorageMap() map[int64]StorageInterface {
return this.storageMap
}
// TotalDiskSize 消耗的磁盘尺寸
func (this *Manager) TotalDiskSize() int64 {
this.locker.RLock()
defer this.locker.RUnlock()
var total = int64(0)
var sidMap = map[string]bool{} // partition sid => bool
for _, storage := range this.storageMap {
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
fileStorage, ok := storage.(*FileStorage)
if ok {
var options = fileStorage.options // copy
if options != nil {
var dir = options.Dir // copy
if len(dir) == 0 {
continue
}
var stat = &unix.Statfs_t{}
err := unix.Statfs(dir, stat)
if err != nil {
continue
}
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
if sidMap[sid] {
continue
}
sidMap[sid] = true
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
}
}
}
if total < 0 {
total = 0
}
return total
}
// TotalMemorySize 消耗的内存尺寸
func (this *Manager) TotalMemorySize() int64 {

View File

@@ -0,0 +1,48 @@
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !windows
package caches
import (
"fmt"
"golang.org/x/sys/unix"
)
// TotalDiskSize 消耗的磁盘尺寸
func (this *Manager) TotalDiskSize() int64 {
this.locker.RLock()
defer this.locker.RUnlock()
var total = int64(0)
var sidMap = map[string]bool{} // partition sid => bool
for _, storage := range this.storageMap {
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
fileStorage, ok := storage.(*FileStorage)
if ok {
var options = fileStorage.options // copy
if options != nil {
var dir = options.Dir // copy
if len(dir) == 0 {
continue
}
var stat = &unix.Statfs_t{}
err := unix.Statfs(dir, stat)
if err != nil {
continue
}
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
if sidMap[sid] {
continue
}
sidMap[sid] = true
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
}
}
}
if total < 0 {
total = 0
}
return total
}

View File

@@ -35,7 +35,6 @@ import (
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
)
@@ -736,7 +735,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
// 尝试锁定,如果锁定失败,则直接返回
fsutils.WriterLimiter.Ack()
err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
err = tryLockFile(int(writer.Fd()))
fsutils.WriterLimiter.Release()
if err != nil {
removeOnFailure = false

View File

@@ -0,0 +1,9 @@
//go:build !windows
package caches
import "syscall"
func tryLockFile(fd int) error {
return syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB)
}

View File

@@ -19,11 +19,8 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
_ "github.com/biessek/golang-ico"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gowebp"
_ "golang.org/x/image/bmp"
_ "golang.org/x/image/webp"
"image"
"image/gif"
_ "image/jpeg"
_ "image/png"
"io"
@@ -1045,169 +1042,6 @@ func (this *HTTPWriter) calculateStaleLife() int {
return staleLife
}
// 结束WebP
func (this *HTTPWriter) finishWebP() {
// 处理WebP
if this.webpIsEncoding {
atomic.AddInt32(&webPThreads, 1)
defer func() {
atomic.AddInt32(&webPThreads, -1)
}()
var webpCacheWriter caches.Writer
// 准备WebP Cache
if this.cacheReader != nil || this.cacheWriter != nil {
var cacheKey = ""
var expiredAt int64 = 0
if this.cacheReader != nil {
cacheKey = this.req.cacheKey + caches.SuffixWebP
expiredAt = this.cacheReader.ExpiresAt()
} else if this.cacheWriter != nil {
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
expiredAt = this.cacheWriter.ExpiredAt()
}
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
if webpCacheWriter != nil {
// 写入Header
for k, v := range this.Header() {
if this.shouldIgnoreHeader(k) {
continue
}
// 这里是原始的数据,不需要内容编码
if k == "Content-Encoding" || k == "Transfer-Encoding" {
continue
}
for _, v1 := range v {
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
_ = webpCacheWriter.Discard()
webpCacheWriter = nil
break
}
}
}
if webpCacheWriter != nil {
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
teeWriter.OnFail(func(err error) {
if webpCacheWriter != nil {
_ = webpCacheWriter.Discard()
}
webpCacheWriter = nil
})
this.writer = teeWriter
}
}
}
var reader = readers.NewBytesCounterReader(this.rawReader)
var imageData image.Image
var gifImage *gif.GIF
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
var err error
if isGif {
gifImage, err = gif.DecodeAll(reader)
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
webPIgnoreURLSet.Push(this.req.URL())
return
}
} else {
imageData, _, err = image.Decode(reader)
if imageData != nil {
var bound = imageData.Bounds()
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
webPIgnoreURLSet.Push(this.req.URL())
return
}
}
}
if err != nil {
// 发生了错误终止处理
webPIgnoreURLSet.Push(this.req.URL())
return
}
var f = types.Float32(this.webpQuality)
if f <= 0 || f > 100 {
if this.size > (8<<20) || this.size <= 0 {
f = 30
} else if this.size > (1 << 20) {
f = 50
} else if this.size > (128 << 10) {
f = 60
} else {
f = 75
}
}
if imageData != nil {
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
Lossless: false,
Quality: f,
Exact: true,
})
} else if gifImage != nil {
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
anim.WebPAnimEncoderOptions.SetKmin(9)
anim.WebPAnimEncoderOptions.SetKmax(17)
var webpConfig = gowebp.NewWebpConfig()
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
var timeline = 0
var lastErr error
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
if err != nil {
// 有错误直接跳过
lastErr = err
err = nil
}
timeline += gifImage.Delay[i] * 10
}
if lastErr != nil {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
}
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
anim.ReleaseMemory()
}
if err != nil && !this.req.canIgnore(err) {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
}
if err == nil && webpCacheWriter != nil {
err = webpCacheWriter.Close()
if err != nil {
_ = webpCacheWriter.Discard()
} else {
this.cacheStorage.AddToList(&caches.Item{
Type: webpCacheWriter.ItemType(),
Key: webpCacheWriter.Key(),
ExpiresAt: webpCacheWriter.ExpiredAt(),
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
HeaderSize: webpCacheWriter.HeaderSize(),
BodySize: webpCacheWriter.BodySize(),
Host: this.req.ReqHost,
ServerId: this.req.ReqServer.Id,
})
}
}
}
}
// 结束缓存相关处理
func (this *HTTPWriter) finishCache() {

View File

@@ -0,0 +1,181 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !windows
package nodes
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gowebp"
"image"
"image/gif"
"strings"
"sync/atomic"
)
// 结束WebP
func (this *HTTPWriter) finishWebP() {
// 处理WebP
if this.webpIsEncoding {
atomic.AddInt32(&webPThreads, 1)
defer func() {
atomic.AddInt32(&webPThreads, -1)
}()
var webpCacheWriter caches.Writer
// 准备WebP Cache
if this.cacheReader != nil || this.cacheWriter != nil {
var cacheKey = ""
var expiredAt int64 = 0
if this.cacheReader != nil {
cacheKey = this.req.cacheKey + caches.SuffixWebP
expiredAt = this.cacheReader.ExpiresAt()
} else if this.cacheWriter != nil {
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
expiredAt = this.cacheWriter.ExpiredAt()
}
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
if webpCacheWriter != nil {
// 写入Header
for k, v := range this.Header() {
if this.shouldIgnoreHeader(k) {
continue
}
// 这里是原始的数据,不需要内容编码
if k == "Content-Encoding" || k == "Transfer-Encoding" {
continue
}
for _, v1 := range v {
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
_ = webpCacheWriter.Discard()
webpCacheWriter = nil
break
}
}
}
if webpCacheWriter != nil {
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
teeWriter.OnFail(func(err error) {
if webpCacheWriter != nil {
_ = webpCacheWriter.Discard()
}
webpCacheWriter = nil
})
this.writer = teeWriter
}
}
}
var reader = readers.NewBytesCounterReader(this.rawReader)
var imageData image.Image
var gifImage *gif.GIF
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
var err error
if isGif {
gifImage, err = gif.DecodeAll(reader)
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
webPIgnoreURLSet.Push(this.req.URL())
return
}
} else {
imageData, _, err = image.Decode(reader)
if imageData != nil {
var bound = imageData.Bounds()
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
webPIgnoreURLSet.Push(this.req.URL())
return
}
}
}
if err != nil {
// 发生了错误终止处理
webPIgnoreURLSet.Push(this.req.URL())
return
}
var f = types.Float32(this.webpQuality)
if f <= 0 || f > 100 {
if this.size > (8<<20) || this.size <= 0 {
f = 30
} else if this.size > (1 << 20) {
f = 50
} else if this.size > (128 << 10) {
f = 60
} else {
f = 75
}
}
if imageData != nil {
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
Lossless: false,
Quality: f,
Exact: true,
})
} else if gifImage != nil {
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
anim.WebPAnimEncoderOptions.SetKmin(9)
anim.WebPAnimEncoderOptions.SetKmax(17)
var webpConfig = gowebp.NewWebpConfig()
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
var timeline = 0
var lastErr error
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
if err != nil {
// 有错误直接跳过
lastErr = err
err = nil
}
timeline += gifImage.Delay[i] * 10
}
if lastErr != nil {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
}
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
anim.ReleaseMemory()
}
if err != nil && !this.req.canIgnore(err) {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
}
if err == nil && webpCacheWriter != nil {
err = webpCacheWriter.Close()
if err != nil {
_ = webpCacheWriter.Discard()
} else {
this.cacheStorage.AddToList(&caches.Item{
Type: webpCacheWriter.ItemType(),
Key: webpCacheWriter.Key(),
ExpiresAt: webpCacheWriter.ExpiredAt(),
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
HeaderSize: webpCacheWriter.HeaderSize(),
BodySize: webpCacheWriter.BodySize(),
Host: this.req.ReqHost,
ServerId: this.req.ReqServer.Id,
})
}
}
}
}

View File

@@ -1,6 +1,5 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !arm64
// +build !arm64
//go:build !arm64 && !windows
package nodes

View File

@@ -4,8 +4,7 @@
package nodes
import (
"context"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/shirou/gopsutil/v3/mem"
"math"
"sync"
@@ -21,7 +20,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
var windowsLoadLocker = &sync.Mutex{}
// 更新内存
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
stat, err := mem.VirtualMemory()
if err != nil {
status.Error = err.Error()
@@ -32,14 +31,16 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
}
// 更新负载
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
timestamp := time.Now().Unix()
currentLoad := 0
/*
info, err := cpu.ProcInfo()
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
currentLoad = int(info[0].ProcessorQueueLength)
}
*/
// 删除15分钟之前的数据
windowsLoadLocker.Lock()
@@ -93,9 +94,11 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
windowsLoadLocker.Unlock()
// 在老Windows上不显示错误
/*
if err == context.DeadlineExceeded {
err = nil
}
*/
status.Load1m = load1
status.Load5m = load5
status.Load15m = load15

View File

@@ -1,4 +1,5 @@
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !windows
package fsutils

View File

@@ -4,20 +4,9 @@ package fsutils
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"golang.org/x/sys/unix"
"sync"
)
// StatDevice device contains the path
func StatDevice(path string) (*StatResult, error) {
var stat = &unix.Statfs_t{}
err := unix.Statfs(path, stat)
if err != nil {
return nil, err
}
return NewStatResult(stat), nil
}
var locker = &sync.RWMutex{}
var cacheMap = map[string]*StatResult{} // path => StatResult
@@ -44,38 +33,3 @@ func StatDeviceCache(path string) (*StatResult, error) {
cacheMap[path] = stat
return stat, nil
}
type StatResult struct {
rawStat *unix.Statfs_t
blockSize uint64
updatedAt int64
}
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
var blockSize = rawStat.Bsize
if blockSize < 0 {
blockSize = 0
}
return &StatResult{
rawStat: rawStat,
blockSize: uint64(blockSize),
updatedAt: fasttime.Now().Unix(),
}
}
func (this *StatResult) FreeSize() uint64 {
return this.rawStat.Bfree * this.blockSize
}
func (this *StatResult) TotalSize() uint64 {
return this.rawStat.Blocks * this.blockSize
}
func (this *StatResult) UsedSize() uint64 {
if this.rawStat.Bfree <= this.rawStat.Blocks {
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
}
return 0
}

View File

@@ -0,0 +1,54 @@
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !windows
package fsutils
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
"golang.org/x/sys/unix"
)
// StatDevice device contains the path
func StatDevice(path string) (*StatResult, error) {
var stat = &unix.Statfs_t{}
err := unix.Statfs(path, stat)
if err != nil {
return nil, err
}
return NewStatResult(stat), nil
}
type StatResult struct {
rawStat *unix.Statfs_t
blockSize uint64
updatedAt int64
}
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
var blockSize = rawStat.Bsize
if blockSize < 0 {
blockSize = 0
}
return &StatResult{
rawStat: rawStat,
blockSize: uint64(blockSize),
updatedAt: fasttime.Now().Unix(),
}
}
func (this *StatResult) FreeSize() uint64 {
return this.rawStat.Bfree * this.blockSize
}
func (this *StatResult) TotalSize() uint64 {
return this.rawStat.Blocks * this.blockSize
}
func (this *StatResult) UsedSize() uint64 {
if this.rawStat.Bfree <= this.rawStat.Blocks {
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
}
return 0
}

View File

@@ -1,5 +1,5 @@
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !linux
//go:build !linux && !windows
package mmap

View File

@@ -1,5 +1,5 @@
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build plus
//go:build plus && !windows
package mmap

View File

@@ -1,5 +1,5 @@
//go:build !freebsd
// +build !freebsd
//go:build !freebsd && !windows
// +build !freebsd,!windows
package utils
@@ -15,7 +15,7 @@ func ListenReuseAddr(network string, addr string) (net.Listener, error) {
config := &net.ListenConfig{
Control: func(network, address string, c syscall.RawConn) error {
return c.Control(func(fd uintptr) {
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1)
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)
if err != nil {
logs.Println("[LISTEN]" + err.Error())
}

View File

@@ -8,6 +8,8 @@ package injectionutils
#include <libinjection.h>
#include <stdlib.h>
*/
//go:build cgo
import "C"
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"

View File

@@ -0,0 +1,18 @@
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !cgo
package injectionutils
import (
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
)
// DetectSQLInjectionCache detect sql injection in string with cache
func DetectSQLInjectionCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
return false
}
// DetectSQLInjection detect sql injection in string
func DetectSQLInjection(input string, isStrict bool) bool {
return false
}

View File

@@ -8,6 +8,8 @@ package injectionutils
#include <libinjection.h>
#include <stdlib.h>
*/
//go:build cgo
import "C"
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"

View File

@@ -0,0 +1,17 @@
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
//go:build !cgo
package injectionutils
import (
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
)
func DetectXSSCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
return false
}
// DetectXSS detect XSS in string
func DetectXSS(input string, isStrict bool) bool {
return false
}

2
deploy/fluent-bit/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
fluent-bit-windows.conf
clickhouse-upstream-windows.conf

BIN
deploy/fluent-bit/logs.db Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.