日常查询由mysql改为clickhouse
This commit is contained in:
@@ -1,5 +0,0 @@
|
||||
user: root
|
||||
password: 123456
|
||||
host: 127.0.0.1:3307
|
||||
database: db_edge
|
||||
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]
|
||||
10
EdgeAPI/.gitignore
vendored
10
EdgeAPI/.gitignore
vendored
@@ -28,3 +28,13 @@ logs/
|
||||
# 临时文件
|
||||
*.tmp
|
||||
.DS_Store
|
||||
|
||||
# Runtime Data
|
||||
data/
|
||||
configs/api.yaml
|
||||
configs/db.yaml
|
||||
db.yaml
|
||||
db.yaml.link.bak
|
||||
|
||||
# Build Artifacts
|
||||
bin/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
user: root
|
||||
password: 123456
|
||||
host: 127.0.0.1
|
||||
pord: 3307
|
||||
pord: 3308
|
||||
database: db_edge
|
||||
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]
|
||||
@@ -56,6 +56,10 @@ type ListFilter struct {
|
||||
LastRequestId string
|
||||
ServerIds []int64
|
||||
NodeIds []int64
|
||||
// 搜索条件
|
||||
Keyword string
|
||||
Ip string
|
||||
Domain string
|
||||
}
|
||||
|
||||
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
|
||||
@@ -126,10 +130,35 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
||||
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
|
||||
}
|
||||
|
||||
// 搜索条件
|
||||
if f.Keyword != "" {
|
||||
keyword := escapeString(f.Keyword)
|
||||
// 在 host, path, ip, ua 中模糊搜索
|
||||
conditions = append(conditions, fmt.Sprintf("(host LIKE '%%%s%%' OR path LIKE '%%%s%%' OR ip LIKE '%%%s%%' OR ua LIKE '%%%s%%')", keyword, keyword, keyword, keyword))
|
||||
}
|
||||
if f.Ip != "" {
|
||||
conditions = append(conditions, "ip = '"+escapeString(f.Ip)+"'")
|
||||
}
|
||||
if f.Domain != "" {
|
||||
conditions = append(conditions, "host LIKE '%"+escapeString(f.Domain)+"%'")
|
||||
}
|
||||
|
||||
// 游标分页:使用 trace_id 作为游标
|
||||
// Reverse=false:历史向后翻页,查询更早的数据
|
||||
// Reverse=true:实时增量拉新,查询更新的数据
|
||||
if f.LastRequestId != "" {
|
||||
if f.Reverse {
|
||||
conditions = append(conditions, "trace_id > '"+escapeString(f.LastRequestId)+"'")
|
||||
} else {
|
||||
conditions = append(conditions, "trace_id < '"+escapeString(f.LastRequestId)+"'")
|
||||
}
|
||||
}
|
||||
|
||||
where := strings.Join(conditions, " AND ")
|
||||
orderDir := "ASC"
|
||||
// 默认按时间倒序(最新的在前面),与前端默认行为一致
|
||||
orderDir := "DESC"
|
||||
if f.Reverse {
|
||||
orderDir = "DESC"
|
||||
orderDir = "ASC"
|
||||
}
|
||||
limit := f.Size
|
||||
if limit <= 0 {
|
||||
@@ -138,7 +167,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
||||
if limit > 1000 {
|
||||
limit = 1000
|
||||
}
|
||||
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
|
||||
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
|
||||
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||
table, where, orderBy, limit+1)
|
||||
@@ -155,13 +184,51 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
if !f.Reverse {
|
||||
if len(rows) > int(limit) {
|
||||
nextCursor = rows[limit].TraceId
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return rows, nextCursor, nil
|
||||
}
|
||||
|
||||
if len(rows) > int(limit) {
|
||||
nextCursor = rows[limit].TraceId
|
||||
rows = rows[:limit]
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
nextCursor = rows[len(rows)-1].TraceId
|
||||
}
|
||||
|
||||
// 实时模式统一返回为“最新在前”,与前端显示和 MySQL 语义一致。
|
||||
for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 {
|
||||
rows[left], rows[right] = rows[right], rows[left]
|
||||
}
|
||||
return rows, nextCursor, nil
|
||||
}
|
||||
|
||||
// FindByTraceId 按 trace_id 查询单条日志详情
|
||||
func (s *LogsIngestStore) FindByTraceId(ctx context.Context, traceId string) (*LogsIngestRow, error) {
|
||||
if !s.client.IsConfigured() {
|
||||
return nil, fmt.Errorf("clickhouse: not configured")
|
||||
}
|
||||
if traceId == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
table := quoteIdent("logs_ingest")
|
||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE trace_id = '%s' LIMIT 1",
|
||||
table, escapeString(traceId))
|
||||
|
||||
var rawRows []map[string]interface{}
|
||||
if err := s.client.Query(ctx, query, &rawRows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rawRows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return mapToLogsIngestRow(rawRows[0]), nil
|
||||
}
|
||||
|
||||
func quoteIdent(name string) string {
|
||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||
}
|
||||
@@ -244,7 +311,7 @@ func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
|
||||
return r
|
||||
}
|
||||
|
||||
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用)
|
||||
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用+详情展示)
|
||||
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||
if r == nil {
|
||||
return nil
|
||||
@@ -259,6 +326,9 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||
RemoteAddr: r.IP,
|
||||
RequestMethod: r.Method,
|
||||
RequestPath: r.Path,
|
||||
RequestURI: r.Path, // 前端使用 requestURI 显示完整路径
|
||||
Scheme: "http", // 默认 http,日志中未存储实际值
|
||||
Proto: "HTTP/1.1", // 默认值,日志中未存储实际值
|
||||
Status: int32(r.Status),
|
||||
RequestLength: int64(r.BytesIn),
|
||||
BytesSent: int64(r.BytesOut),
|
||||
@@ -273,6 +343,39 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||
if r.TimeISO8601() != "" {
|
||||
a.TimeISO8601 = r.TimeISO8601()
|
||||
}
|
||||
// TimeLocal: 用户友好的时间格式 (e.g., "2026-02-07 23:17:12")
|
||||
if !r.Timestamp.IsZero() {
|
||||
a.TimeLocal = r.Timestamp.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
// 解析请求头 (JSON -> map[string]*pb.Strings)
|
||||
// ClickHouse 中存储的是 map[string]string 格式
|
||||
if r.RequestHeaders != "" {
|
||||
var headers map[string]string
|
||||
if err := json.Unmarshal([]byte(r.RequestHeaders), &headers); err == nil {
|
||||
a.Header = make(map[string]*pb.Strings)
|
||||
for k, v := range headers {
|
||||
a.Header[k] = &pb.Strings{Values: []string{v}}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 解析响应头 (JSON -> map[string]*pb.Strings)
|
||||
if r.ResponseHeaders != "" {
|
||||
var headers map[string]string
|
||||
if err := json.Unmarshal([]byte(r.ResponseHeaders), &headers); err == nil {
|
||||
a.SentHeader = make(map[string]*pb.Strings)
|
||||
for k, v := range headers {
|
||||
a.SentHeader[k] = &pb.Strings{Values: []string{v}}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 请求体
|
||||
if r.RequestBody != "" {
|
||||
a.RequestBody = []byte(r.RequestBody)
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
|
||||
@@ -261,3 +261,23 @@ func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ReadClickHouseConfig 读取ClickHouse配置
|
||||
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
||||
valueJSON, err := this.ReadSetting(tx, "clickhouseConfig")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(valueJSON) == 0 {
|
||||
return &systemconfigs.ClickHouseSetting{
|
||||
Port: 8123,
|
||||
Database: "default",
|
||||
}, nil
|
||||
}
|
||||
var config = &systemconfigs.ClickHouseSetting{}
|
||||
err = json.Unmarshal(valueJSON, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
@@ -66,25 +66,3 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
|
||||
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
||||
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
|
||||
if len(valueJSON) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
err = json.Unmarshal(valueJSON, out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out.Port <= 0 {
|
||||
out.Port = 8123
|
||||
}
|
||||
if out.Database == "" {
|
||||
out.Database = "default"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"golang.org/x/sys/unix"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -60,14 +59,7 @@ func CheckHasFreeSpace() bool {
|
||||
}
|
||||
LocalDatabaseDataDir = dir
|
||||
|
||||
var stat unix.Statfs_t
|
||||
err = unix.Statfs(dir, &stat)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
|
||||
return availableSpace > minFreeSpaceGB
|
||||
return checkHasFreeSpace(dir)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
18
EdgeAPI/internal/db/utils/disk_unix.go
Normal file
18
EdgeAPI/internal/db/utils/disk_unix.go
Normal file
@@ -0,0 +1,18 @@
|
||||
//go:build !windows
|
||||
|
||||
package dbutils
|
||||
|
||||
import (
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func checkHasFreeSpace(dir string) bool {
|
||||
var stat unix.Statfs_t
|
||||
err := unix.Statfs(dir, &stat)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
|
||||
return availableSpace > minFreeSpaceGB
|
||||
}
|
||||
7
EdgeAPI/internal/db/utils/disk_windows.go
Normal file
7
EdgeAPI/internal/db/utils/disk_windows.go
Normal file
@@ -0,0 +1,7 @@
|
||||
//go:build windows
|
||||
|
||||
package dbutils
|
||||
|
||||
func checkHasFreeSpace(dir string) bool {
|
||||
return true
|
||||
}
|
||||
@@ -4,8 +4,9 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
//"context"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
//"github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/shirou/gopsutil/v3/mem"
|
||||
"math"
|
||||
"sync"
|
||||
@@ -21,7 +22,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
|
||||
var windowsLoadLocker = &sync.Mutex{}
|
||||
|
||||
// 更新内存
|
||||
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
||||
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
|
||||
stat, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
status.Error = err.Error()
|
||||
@@ -32,14 +33,14 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
||||
}
|
||||
|
||||
// 更新负载
|
||||
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
||||
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
|
||||
timestamp := time.Now().Unix()
|
||||
|
||||
currentLoad := 0
|
||||
info, err := cpu.ProcInfo()
|
||||
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||
currentLoad = int(info[0].ProcessorQueueLength)
|
||||
}
|
||||
//info, err := cpu.ProcInfo()
|
||||
//if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||
// currentLoad = int(info[0].ProcessorQueueLength)
|
||||
//}
|
||||
|
||||
// 删除15分钟之前的数据
|
||||
windowsLoadLocker.Lock()
|
||||
@@ -93,9 +94,9 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
||||
windowsLoadLocker.Unlock()
|
||||
|
||||
// 在老Windows上不显示错误
|
||||
if err == context.DeadlineExceeded {
|
||||
err = nil
|
||||
}
|
||||
//if err == context.DeadlineExceeded {
|
||||
// err = nil
|
||||
//}
|
||||
status.Load1m = load1
|
||||
status.Load5m = load5
|
||||
status.Load15m = load15
|
||||
|
||||
@@ -144,6 +144,9 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
|
||||
NodeId: req.NodeId,
|
||||
ClusterId: req.NodeClusterId,
|
||||
LastRequestId: req.RequestId,
|
||||
Keyword: req.Keyword,
|
||||
Ip: req.Ip,
|
||||
Domain: req.Domain,
|
||||
}
|
||||
if req.ServerId > 0 {
|
||||
f.ServerIds = []int64{req.ServerId}
|
||||
@@ -216,7 +219,10 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
|
||||
}
|
||||
}
|
||||
|
||||
hasMore := nextCursor != ""
|
||||
hasMore := false
|
||||
if !req.Reverse {
|
||||
hasMore = nextCursor != ""
|
||||
}
|
||||
return &pb.ListHTTPAccessLogsResponse{
|
||||
HttpAccessLogs: result,
|
||||
AccessLogs: result,
|
||||
@@ -233,6 +239,28 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 优先从 ClickHouse 查询
|
||||
store := clickhouse.NewLogsIngestStore()
|
||||
if store.Client().IsConfigured() {
|
||||
row, err := store.FindByTraceId(ctx, req.RequestId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if row != nil {
|
||||
// 检查权限
|
||||
if userId > 0 {
|
||||
var tx = this.NullTx()
|
||||
err = models.SharedServerDAO.CheckUserServer(tx, userId, int64(row.ServerId))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
a := clickhouse.RowToPB(row)
|
||||
return &pb.FindHTTPAccessLogResponse{HttpAccessLog: a}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 如果 ClickHouse 未配置或未找到,则回退到 MySQL
|
||||
var tx = this.NullTx()
|
||||
|
||||
accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
|
||||
return accesslogs.SharedStorageManager.WriteMySQL()
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
|
||||
|
||||
12
EdgeAdmin/.gitignore
vendored
12
EdgeAdmin/.gitignore
vendored
@@ -0,0 +1,12 @@
|
||||
部署启动手册.md
|
||||
|
||||
# Runtime Data
|
||||
data/
|
||||
logs/
|
||||
|
||||
# Local Configs
|
||||
configs/api_admin.yaml
|
||||
configs/server.yaml
|
||||
|
||||
# Build Artifacts
|
||||
bin/
|
||||
@@ -7,6 +7,47 @@ Tea.context(function () {
|
||||
this.accessLogs = []
|
||||
this.isLoaded = false
|
||||
|
||||
this.mergeAccessLogs = function (newAccessLogs, regions, wafInfos) {
|
||||
let mergedLogs = newAccessLogs.concat(this.accessLogs)
|
||||
let result = []
|
||||
let seenRequestIds = {}
|
||||
|
||||
let that = this
|
||||
mergedLogs.forEach(function (accessLog) {
|
||||
if (accessLog == null) {
|
||||
return
|
||||
}
|
||||
|
||||
let requestId = accessLog.requestId || ""
|
||||
if (requestId.length > 0) {
|
||||
if (seenRequestIds[requestId] === true) {
|
||||
return
|
||||
}
|
||||
seenRequestIds[requestId] = true
|
||||
}
|
||||
|
||||
that.formatTime(accessLog)
|
||||
|
||||
let region = regions[accessLog.remoteAddr]
|
||||
if (typeof (region) == "string") {
|
||||
accessLog.region = region
|
||||
} else if (typeof accessLog.region != "string") {
|
||||
accessLog.region = ""
|
||||
}
|
||||
|
||||
let wafInfo = wafInfos[accessLog.firewallRuleSetId]
|
||||
if (accessLog.firewallRuleSetId > 0 && typeof (wafInfo) == "object") {
|
||||
accessLog.wafInfo = wafInfo
|
||||
} else if (typeof accessLog.wafInfo == "undefined") {
|
||||
accessLog.wafInfo = null
|
||||
}
|
||||
|
||||
result.push(accessLog)
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
this.load = function () {
|
||||
// 如果有弹窗时,暂时不更新
|
||||
if (teaweb.hasPopup()) {
|
||||
@@ -27,30 +68,17 @@ Tea.context(function () {
|
||||
nodeId: this.nodeId
|
||||
})
|
||||
.success(function (resp) {
|
||||
this.accessLogs = resp.data.accessLogs.concat(this.accessLogs)
|
||||
|
||||
// 添加区域信息
|
||||
let that = this
|
||||
this.accessLogs.forEach(function (accessLog) {
|
||||
that.formatTime(accessLog)
|
||||
if (typeof (resp.data.regions[accessLog.remoteAddr]) == "string") {
|
||||
accessLog.region = resp.data.regions[accessLog.remoteAddr]
|
||||
} else {
|
||||
accessLog.region = ""
|
||||
}
|
||||
if (accessLog.firewallRuleSetId > 0 && typeof (resp.data.wafInfos[accessLog.firewallRuleSetId]) == "object") {
|
||||
accessLog.wafInfo = resp.data.wafInfos[accessLog.firewallRuleSetId]
|
||||
} else {
|
||||
accessLog.wafInfo = null
|
||||
}
|
||||
})
|
||||
let newAccessLogs = resp.data.accessLogs || []
|
||||
this.accessLogs = this.mergeAccessLogs(newAccessLogs, resp.data.regions || {}, resp.data.wafInfos || {})
|
||||
|
||||
let max = 100
|
||||
if (this.accessLogs.length > max) {
|
||||
this.accessLogs = this.accessLogs.slice(0, max)
|
||||
}
|
||||
this.hasMore = resp.data.hasMore
|
||||
this.requestId = resp.data.requestId
|
||||
if (typeof resp.data.requestId == "string" && resp.data.requestId.length > 0) {
|
||||
this.requestId = resp.data.requestId
|
||||
}
|
||||
})
|
||||
.done(function () {
|
||||
if (!this.isLoaded) {
|
||||
@@ -79,4 +107,4 @@ Tea.context(function () {
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
|
||||
require (
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 // indirect
|
||||
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
|
||||
@@ -17,6 +17,8 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 h1:ouxvoYN6WL482nK1zS1IplyniAut3G4D+0N5JS0YTuw=
|
||||
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879/go.mod h1:+mNMTBuDMdEGhWzoQgc6kBdqeaQpWh5ba8zqmp2MxCU=
|
||||
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
|
||||
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
|
||||
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
//go:build plus
|
||||
|
||||
package systemconfigs
|
||||
|
||||
|
||||
16
EdgeNode/.gitignore
vendored
16
EdgeNode/.gitignore
vendored
@@ -0,0 +1,16 @@
|
||||
# Windows local development
|
||||
*_windows.go
|
||||
configs/api_node.yaml
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Build binaries
|
||||
bin/
|
||||
|
||||
# Runtime Data
|
||||
data/
|
||||
configs/node.json
|
||||
logs/
|
||||
opt/
|
||||
|
||||
@@ -102,7 +102,7 @@ func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog,
|
||||
Host: l.GetHost(),
|
||||
IP: l.GetRawRemoteAddr(),
|
||||
Method: l.GetRequestMethod(),
|
||||
Path: l.GetRequestPath(),
|
||||
Path: l.GetRequestURI(), // 使用 RequestURI 以包含查询参数
|
||||
Status: l.GetStatus(),
|
||||
BytesIn: l.GetRequestLength(),
|
||||
BytesOut: l.GetBytesSent(),
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -197,10 +196,7 @@ func (this *AppCmd) runStart() {
|
||||
_ = os.Setenv("EdgeBackground", "on")
|
||||
|
||||
var cmd = exec.Command(this.exe())
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Foreground: false,
|
||||
Setsid: true,
|
||||
}
|
||||
configureSysProcAttr(cmd)
|
||||
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
|
||||
15
EdgeNode/internal/apps/app_utils_unix.go
Normal file
15
EdgeNode/internal/apps/app_utils_unix.go
Normal file
@@ -0,0 +1,15 @@
|
||||
//go:build !windows
|
||||
|
||||
package apps
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func configureSysProcAttr(cmd *exec.Cmd) {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Foreground: false,
|
||||
Setsid: true,
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
@@ -10,7 +9,6 @@ import (
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"golang.org/x/sys/unix"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
@@ -191,44 +189,6 @@ func (this *Manager) StorageMap() map[int64]StorageInterface {
|
||||
return this.storageMap
|
||||
}
|
||||
|
||||
// TotalDiskSize 消耗的磁盘尺寸
|
||||
func (this *Manager) TotalDiskSize() int64 {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
var total = int64(0)
|
||||
var sidMap = map[string]bool{} // partition sid => bool
|
||||
for _, storage := range this.storageMap {
|
||||
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
|
||||
fileStorage, ok := storage.(*FileStorage)
|
||||
if ok {
|
||||
var options = fileStorage.options // copy
|
||||
if options != nil {
|
||||
var dir = options.Dir // copy
|
||||
if len(dir) == 0 {
|
||||
continue
|
||||
}
|
||||
var stat = &unix.Statfs_t{}
|
||||
err := unix.Statfs(dir, stat)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
|
||||
if sidMap[sid] {
|
||||
continue
|
||||
}
|
||||
sidMap[sid] = true
|
||||
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if total < 0 {
|
||||
total = 0
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
// TotalMemorySize 消耗的内存尺寸
|
||||
func (this *Manager) TotalMemorySize() int64 {
|
||||
|
||||
48
EdgeNode/internal/caches/manager_unix.go
Normal file
48
EdgeNode/internal/caches/manager_unix.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !windows
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// TotalDiskSize 消耗的磁盘尺寸
|
||||
func (this *Manager) TotalDiskSize() int64 {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
var total = int64(0)
|
||||
var sidMap = map[string]bool{} // partition sid => bool
|
||||
for _, storage := range this.storageMap {
|
||||
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
|
||||
fileStorage, ok := storage.(*FileStorage)
|
||||
if ok {
|
||||
var options = fileStorage.options // copy
|
||||
if options != nil {
|
||||
var dir = options.Dir // copy
|
||||
if len(dir) == 0 {
|
||||
continue
|
||||
}
|
||||
var stat = &unix.Statfs_t{}
|
||||
err := unix.Statfs(dir, stat)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
|
||||
if sidMap[sid] {
|
||||
continue
|
||||
}
|
||||
sidMap[sid] = true
|
||||
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if total < 0 {
|
||||
total = 0
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
@@ -35,7 +35,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -736,7 +735,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
|
||||
// 尝试锁定,如果锁定失败,则直接返回
|
||||
fsutils.WriterLimiter.Ack()
|
||||
err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
err = tryLockFile(int(writer.Fd()))
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
removeOnFailure = false
|
||||
|
||||
9
EdgeNode/internal/caches/storage_utils_unix.go
Normal file
9
EdgeNode/internal/caches/storage_utils_unix.go
Normal file
@@ -0,0 +1,9 @@
|
||||
//go:build !windows
|
||||
|
||||
package caches
|
||||
|
||||
import "syscall"
|
||||
|
||||
func tryLockFile(fd int) error {
|
||||
return syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
}
|
||||
@@ -19,11 +19,8 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
|
||||
_ "github.com/biessek/golang-ico"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gowebp"
|
||||
_ "golang.org/x/image/bmp"
|
||||
_ "golang.org/x/image/webp"
|
||||
"image"
|
||||
"image/gif"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
@@ -1045,169 +1042,6 @@ func (this *HTTPWriter) calculateStaleLife() int {
|
||||
return staleLife
|
||||
}
|
||||
|
||||
// 结束WebP
|
||||
func (this *HTTPWriter) finishWebP() {
|
||||
// 处理WebP
|
||||
if this.webpIsEncoding {
|
||||
atomic.AddInt32(&webPThreads, 1)
|
||||
defer func() {
|
||||
atomic.AddInt32(&webPThreads, -1)
|
||||
}()
|
||||
|
||||
var webpCacheWriter caches.Writer
|
||||
|
||||
// 准备WebP Cache
|
||||
if this.cacheReader != nil || this.cacheWriter != nil {
|
||||
var cacheKey = ""
|
||||
var expiredAt int64 = 0
|
||||
|
||||
if this.cacheReader != nil {
|
||||
cacheKey = this.req.cacheKey + caches.SuffixWebP
|
||||
expiredAt = this.cacheReader.ExpiresAt()
|
||||
} else if this.cacheWriter != nil {
|
||||
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
|
||||
expiredAt = this.cacheWriter.ExpiredAt()
|
||||
}
|
||||
|
||||
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
|
||||
if webpCacheWriter != nil {
|
||||
// 写入Header
|
||||
for k, v := range this.Header() {
|
||||
if this.shouldIgnoreHeader(k) {
|
||||
continue
|
||||
}
|
||||
|
||||
// 这里是原始的数据,不需要内容编码
|
||||
if k == "Content-Encoding" || k == "Transfer-Encoding" {
|
||||
continue
|
||||
}
|
||||
for _, v1 := range v {
|
||||
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
|
||||
_ = webpCacheWriter.Discard()
|
||||
webpCacheWriter = nil
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if webpCacheWriter != nil {
|
||||
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
|
||||
teeWriter.OnFail(func(err error) {
|
||||
if webpCacheWriter != nil {
|
||||
_ = webpCacheWriter.Discard()
|
||||
}
|
||||
webpCacheWriter = nil
|
||||
})
|
||||
this.writer = teeWriter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var reader = readers.NewBytesCounterReader(this.rawReader)
|
||||
|
||||
var imageData image.Image
|
||||
var gifImage *gif.GIF
|
||||
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
|
||||
var err error
|
||||
if isGif {
|
||||
gifImage, err = gif.DecodeAll(reader)
|
||||
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
} else {
|
||||
imageData, _, err = image.Decode(reader)
|
||||
if imageData != nil {
|
||||
var bound = imageData.Bounds()
|
||||
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// 发生了错误终止处理
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
|
||||
var f = types.Float32(this.webpQuality)
|
||||
if f <= 0 || f > 100 {
|
||||
if this.size > (8<<20) || this.size <= 0 {
|
||||
f = 30
|
||||
} else if this.size > (1 << 20) {
|
||||
f = 50
|
||||
} else if this.size > (128 << 10) {
|
||||
f = 60
|
||||
} else {
|
||||
f = 75
|
||||
}
|
||||
}
|
||||
|
||||
if imageData != nil {
|
||||
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
|
||||
Lossless: false,
|
||||
Quality: f,
|
||||
Exact: true,
|
||||
})
|
||||
} else if gifImage != nil {
|
||||
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
|
||||
anim.WebPAnimEncoderOptions.SetKmin(9)
|
||||
anim.WebPAnimEncoderOptions.SetKmax(17)
|
||||
var webpConfig = gowebp.NewWebpConfig()
|
||||
//webpConfig.SetLossless(1)
|
||||
webpConfig.SetQuality(f)
|
||||
|
||||
var timeline = 0
|
||||
var lastErr error
|
||||
for i, img := range gifImage.Image {
|
||||
err = anim.AddFrame(img, timeline, webpConfig)
|
||||
if err != nil {
|
||||
// 有错误直接跳过
|
||||
lastErr = err
|
||||
err = nil
|
||||
}
|
||||
timeline += gifImage.Delay[i] * 10
|
||||
}
|
||||
if lastErr != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
||||
}
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
|
||||
anim.ReleaseMemory()
|
||||
}
|
||||
|
||||
if err != nil && !this.req.canIgnore(err) {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
|
||||
}
|
||||
|
||||
if err == nil && webpCacheWriter != nil {
|
||||
err = webpCacheWriter.Close()
|
||||
if err != nil {
|
||||
_ = webpCacheWriter.Discard()
|
||||
} else {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: webpCacheWriter.ItemType(),
|
||||
Key: webpCacheWriter.Key(),
|
||||
ExpiresAt: webpCacheWriter.ExpiredAt(),
|
||||
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
|
||||
HeaderSize: webpCacheWriter.HeaderSize(),
|
||||
BodySize: webpCacheWriter.BodySize(),
|
||||
Host: this.req.ReqHost,
|
||||
ServerId: this.req.ReqServer.Id,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 结束缓存相关处理
|
||||
func (this *HTTPWriter) finishCache() {
|
||||
|
||||
181
EdgeNode/internal/nodes/http_writer_ext_unix.go
Normal file
181
EdgeNode/internal/nodes/http_writer_ext_unix.go
Normal file
@@ -0,0 +1,181 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build !windows
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gowebp"
|
||||
"image"
|
||||
"image/gif"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// 结束WebP
|
||||
func (this *HTTPWriter) finishWebP() {
|
||||
// 处理WebP
|
||||
if this.webpIsEncoding {
|
||||
atomic.AddInt32(&webPThreads, 1)
|
||||
defer func() {
|
||||
atomic.AddInt32(&webPThreads, -1)
|
||||
}()
|
||||
|
||||
var webpCacheWriter caches.Writer
|
||||
|
||||
// 准备WebP Cache
|
||||
if this.cacheReader != nil || this.cacheWriter != nil {
|
||||
var cacheKey = ""
|
||||
var expiredAt int64 = 0
|
||||
|
||||
if this.cacheReader != nil {
|
||||
cacheKey = this.req.cacheKey + caches.SuffixWebP
|
||||
expiredAt = this.cacheReader.ExpiresAt()
|
||||
} else if this.cacheWriter != nil {
|
||||
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
|
||||
expiredAt = this.cacheWriter.ExpiredAt()
|
||||
}
|
||||
|
||||
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
|
||||
if webpCacheWriter != nil {
|
||||
// 写入Header
|
||||
for k, v := range this.Header() {
|
||||
if this.shouldIgnoreHeader(k) {
|
||||
continue
|
||||
}
|
||||
|
||||
// 这里是原始的数据,不需要内容编码
|
||||
if k == "Content-Encoding" || k == "Transfer-Encoding" {
|
||||
continue
|
||||
}
|
||||
for _, v1 := range v {
|
||||
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
|
||||
_ = webpCacheWriter.Discard()
|
||||
webpCacheWriter = nil
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if webpCacheWriter != nil {
|
||||
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
|
||||
teeWriter.OnFail(func(err error) {
|
||||
if webpCacheWriter != nil {
|
||||
_ = webpCacheWriter.Discard()
|
||||
}
|
||||
webpCacheWriter = nil
|
||||
})
|
||||
this.writer = teeWriter
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var reader = readers.NewBytesCounterReader(this.rawReader)
|
||||
|
||||
var imageData image.Image
|
||||
var gifImage *gif.GIF
|
||||
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
|
||||
var err error
|
||||
if isGif {
|
||||
gifImage, err = gif.DecodeAll(reader)
|
||||
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
} else {
|
||||
imageData, _, err = image.Decode(reader)
|
||||
if imageData != nil {
|
||||
var bound = imageData.Bounds()
|
||||
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// 发生了错误终止处理
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
|
||||
var f = types.Float32(this.webpQuality)
|
||||
if f <= 0 || f > 100 {
|
||||
if this.size > (8<<20) || this.size <= 0 {
|
||||
f = 30
|
||||
} else if this.size > (1 << 20) {
|
||||
f = 50
|
||||
} else if this.size > (128 << 10) {
|
||||
f = 60
|
||||
} else {
|
||||
f = 75
|
||||
}
|
||||
}
|
||||
|
||||
if imageData != nil {
|
||||
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
|
||||
Lossless: false,
|
||||
Quality: f,
|
||||
Exact: true,
|
||||
})
|
||||
} else if gifImage != nil {
|
||||
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
|
||||
anim.WebPAnimEncoderOptions.SetKmin(9)
|
||||
anim.WebPAnimEncoderOptions.SetKmax(17)
|
||||
var webpConfig = gowebp.NewWebpConfig()
|
||||
//webpConfig.SetLossless(1)
|
||||
webpConfig.SetQuality(f)
|
||||
|
||||
var timeline = 0
|
||||
var lastErr error
|
||||
for i, img := range gifImage.Image {
|
||||
err = anim.AddFrame(img, timeline, webpConfig)
|
||||
if err != nil {
|
||||
// 有错误直接跳过
|
||||
lastErr = err
|
||||
err = nil
|
||||
}
|
||||
timeline += gifImage.Delay[i] * 10
|
||||
}
|
||||
if lastErr != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
||||
}
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
|
||||
anim.ReleaseMemory()
|
||||
}
|
||||
|
||||
if err != nil && !this.req.canIgnore(err) {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
|
||||
}
|
||||
|
||||
if err == nil && webpCacheWriter != nil {
|
||||
err = webpCacheWriter.Close()
|
||||
if err != nil {
|
||||
_ = webpCacheWriter.Discard()
|
||||
} else {
|
||||
this.cacheStorage.AddToList(&caches.Item{
|
||||
Type: webpCacheWriter.ItemType(),
|
||||
Key: webpCacheWriter.Key(),
|
||||
ExpiresAt: webpCacheWriter.ExpiredAt(),
|
||||
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
|
||||
HeaderSize: webpCacheWriter.HeaderSize(),
|
||||
BodySize: webpCacheWriter.BodySize(),
|
||||
Host: this.req.ReqHost,
|
||||
ServerId: this.req.ReqServer.Id,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build !arm64
|
||||
// +build !arm64
|
||||
//go:build !arm64 && !windows
|
||||
|
||||
package nodes
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/shirou/gopsutil/v3/mem"
|
||||
"math"
|
||||
"sync"
|
||||
@@ -21,7 +20,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
|
||||
var windowsLoadLocker = &sync.Mutex{}
|
||||
|
||||
// 更新内存
|
||||
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
||||
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
|
||||
stat, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
status.Error = err.Error()
|
||||
@@ -32,14 +31,16 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
||||
}
|
||||
|
||||
// 更新负载
|
||||
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
||||
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
|
||||
timestamp := time.Now().Unix()
|
||||
|
||||
currentLoad := 0
|
||||
info, err := cpu.ProcInfo()
|
||||
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||
currentLoad = int(info[0].ProcessorQueueLength)
|
||||
}
|
||||
/*
|
||||
info, err := cpu.ProcInfo()
|
||||
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||
currentLoad = int(info[0].ProcessorQueueLength)
|
||||
}
|
||||
*/
|
||||
|
||||
// 删除15分钟之前的数据
|
||||
windowsLoadLocker.Lock()
|
||||
@@ -93,9 +94,11 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
||||
windowsLoadLocker.Unlock()
|
||||
|
||||
// 在老Windows上不显示错误
|
||||
if err == context.DeadlineExceeded {
|
||||
err = nil
|
||||
}
|
||||
/*
|
||||
if err == context.DeadlineExceeded {
|
||||
err = nil
|
||||
}
|
||||
*/
|
||||
status.Load1m = load1
|
||||
status.Load5m = load5
|
||||
status.Load15m = load15
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !windows
|
||||
|
||||
package fsutils
|
||||
|
||||
@@ -4,20 +4,9 @@ package fsutils
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"golang.org/x/sys/unix"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// StatDevice device contains the path
|
||||
func StatDevice(path string) (*StatResult, error) {
|
||||
var stat = &unix.Statfs_t{}
|
||||
err := unix.Statfs(path, stat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewStatResult(stat), nil
|
||||
}
|
||||
|
||||
var locker = &sync.RWMutex{}
|
||||
var cacheMap = map[string]*StatResult{} // path => StatResult
|
||||
|
||||
@@ -44,38 +33,3 @@ func StatDeviceCache(path string) (*StatResult, error) {
|
||||
cacheMap[path] = stat
|
||||
return stat, nil
|
||||
}
|
||||
|
||||
type StatResult struct {
|
||||
rawStat *unix.Statfs_t
|
||||
blockSize uint64
|
||||
|
||||
updatedAt int64
|
||||
}
|
||||
|
||||
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
|
||||
var blockSize = rawStat.Bsize
|
||||
if blockSize < 0 {
|
||||
blockSize = 0
|
||||
}
|
||||
|
||||
return &StatResult{
|
||||
rawStat: rawStat,
|
||||
blockSize: uint64(blockSize),
|
||||
updatedAt: fasttime.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *StatResult) FreeSize() uint64 {
|
||||
return this.rawStat.Bfree * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) TotalSize() uint64 {
|
||||
return this.rawStat.Blocks * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) UsedSize() uint64 {
|
||||
if this.rawStat.Bfree <= this.rawStat.Blocks {
|
||||
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
54
EdgeNode/internal/utils/fs/stat_unix.go
Normal file
54
EdgeNode/internal/utils/fs/stat_unix.go
Normal file
@@ -0,0 +1,54 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !windows
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// StatDevice device contains the path
|
||||
func StatDevice(path string) (*StatResult, error) {
|
||||
var stat = &unix.Statfs_t{}
|
||||
err := unix.Statfs(path, stat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewStatResult(stat), nil
|
||||
}
|
||||
|
||||
type StatResult struct {
|
||||
rawStat *unix.Statfs_t
|
||||
blockSize uint64
|
||||
|
||||
updatedAt int64
|
||||
}
|
||||
|
||||
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
|
||||
var blockSize = rawStat.Bsize
|
||||
if blockSize < 0 {
|
||||
blockSize = 0
|
||||
}
|
||||
|
||||
return &StatResult{
|
||||
rawStat: rawStat,
|
||||
blockSize: uint64(blockSize),
|
||||
updatedAt: fasttime.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *StatResult) FreeSize() uint64 {
|
||||
return this.rawStat.Bfree * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) TotalSize() uint64 {
|
||||
return this.rawStat.Blocks * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) UsedSize() uint64 {
|
||||
if this.rawStat.Bfree <= this.rawStat.Blocks {
|
||||
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !linux
|
||||
//go:build !linux && !windows
|
||||
|
||||
package mmap
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus
|
||||
//go:build plus && !windows
|
||||
|
||||
package mmap
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:build !freebsd
|
||||
// +build !freebsd
|
||||
//go:build !freebsd && !windows
|
||||
// +build !freebsd,!windows
|
||||
|
||||
package utils
|
||||
|
||||
@@ -15,7 +15,7 @@ func ListenReuseAddr(network string, addr string) (net.Listener, error) {
|
||||
config := &net.ListenConfig{
|
||||
Control: func(network, address string, c syscall.RawConn) error {
|
||||
return c.Control(func(fd uintptr) {
|
||||
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1)
|
||||
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)
|
||||
if err != nil {
|
||||
logs.Println("[LISTEN]" + err.Error())
|
||||
}
|
||||
@@ -8,6 +8,8 @@ package injectionutils
|
||||
#include <libinjection.h>
|
||||
#include <stdlib.h>
|
||||
*/
|
||||
//go:build cgo
|
||||
|
||||
import "C"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
|
||||
18
EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go
Normal file
18
EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !cgo
|
||||
|
||||
package injectionutils
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
|
||||
)
|
||||
|
||||
// DetectSQLInjectionCache detect sql injection in string with cache
|
||||
func DetectSQLInjectionCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// DetectSQLInjection detect sql injection in string
|
||||
func DetectSQLInjection(input string, isStrict bool) bool {
|
||||
return false
|
||||
}
|
||||
@@ -8,6 +8,8 @@ package injectionutils
|
||||
#include <libinjection.h>
|
||||
#include <stdlib.h>
|
||||
*/
|
||||
//go:build cgo
|
||||
|
||||
import "C"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
|
||||
17
EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go
Normal file
17
EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go
Normal file
@@ -0,0 +1,17 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build !cgo
|
||||
|
||||
package injectionutils
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
|
||||
)
|
||||
|
||||
func DetectXSSCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// DetectXSS detect XSS in string
|
||||
func DetectXSS(input string, isStrict bool) bool {
|
||||
return false
|
||||
}
|
||||
2
deploy/fluent-bit/.gitignore
vendored
Normal file
2
deploy/fluent-bit/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
fluent-bit-windows.conf
|
||||
clickhouse-upstream-windows.conf
|
||||
BIN
deploy/fluent-bit/logs.db
Normal file
BIN
deploy/fluent-bit/logs.db
Normal file
Binary file not shown.
BIN
deploy/fluent-bit/logs.db-shm
Normal file
BIN
deploy/fluent-bit/logs.db-shm
Normal file
Binary file not shown.
BIN
deploy/fluent-bit/logs.db-wal
Normal file
BIN
deploy/fluent-bit/logs.db-wal
Normal file
Binary file not shown.
Reference in New Issue
Block a user