日常查询由mysql改为clickhouse
This commit is contained in:
@@ -1,5 +0,0 @@
|
|||||||
user: root
|
|
||||||
password: 123456
|
|
||||||
host: 127.0.0.1:3307
|
|
||||||
database: db_edge
|
|
||||||
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]
|
|
||||||
10
EdgeAPI/.gitignore
vendored
10
EdgeAPI/.gitignore
vendored
@@ -28,3 +28,13 @@ logs/
|
|||||||
# 临时文件
|
# 临时文件
|
||||||
*.tmp
|
*.tmp
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
# Runtime Data
|
||||||
|
data/
|
||||||
|
configs/api.yaml
|
||||||
|
configs/db.yaml
|
||||||
|
db.yaml
|
||||||
|
db.yaml.link.bak
|
||||||
|
|
||||||
|
# Build Artifacts
|
||||||
|
bin/
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
user: root
|
user: root
|
||||||
password: 123456
|
password: 123456
|
||||||
host: 127.0.0.1
|
host: 127.0.0.1
|
||||||
pord: 3307
|
pord: 3308
|
||||||
database: db_edge
|
database: db_edge
|
||||||
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]
|
boolFields: [ "uamIsOn", "followPort", "requestHostExcludingPort", "autoRemoteStart", "autoInstallNftables", "enableIPLists", "detectAgents", "checkingPorts", "enableRecordHealthCheck", "offlineIsNotified", "http2Enabled", "http3Enabled", "enableHTTP2", "retry50X", "retry40X", "autoSystemTuning", "disableDefaultDB", "autoTrimDisks", "enableGlobalPages", "ignoreLocal", "ignoreSearchEngine" ]
|
||||||
@@ -56,6 +56,10 @@ type ListFilter struct {
|
|||||||
LastRequestId string
|
LastRequestId string
|
||||||
ServerIds []int64
|
ServerIds []int64
|
||||||
NodeIds []int64
|
NodeIds []int64
|
||||||
|
// 搜索条件
|
||||||
|
Keyword string
|
||||||
|
Ip string
|
||||||
|
Domain string
|
||||||
}
|
}
|
||||||
|
|
||||||
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
|
// LogsIngestStore 封装对 logs_ingest 的只读列表查询
|
||||||
@@ -126,10 +130,35 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
|||||||
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
|
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 搜索条件
|
||||||
|
if f.Keyword != "" {
|
||||||
|
keyword := escapeString(f.Keyword)
|
||||||
|
// 在 host, path, ip, ua 中模糊搜索
|
||||||
|
conditions = append(conditions, fmt.Sprintf("(host LIKE '%%%s%%' OR path LIKE '%%%s%%' OR ip LIKE '%%%s%%' OR ua LIKE '%%%s%%')", keyword, keyword, keyword, keyword))
|
||||||
|
}
|
||||||
|
if f.Ip != "" {
|
||||||
|
conditions = append(conditions, "ip = '"+escapeString(f.Ip)+"'")
|
||||||
|
}
|
||||||
|
if f.Domain != "" {
|
||||||
|
conditions = append(conditions, "host LIKE '%"+escapeString(f.Domain)+"%'")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 游标分页:使用 trace_id 作为游标
|
||||||
|
// Reverse=false:历史向后翻页,查询更早的数据
|
||||||
|
// Reverse=true:实时增量拉新,查询更新的数据
|
||||||
|
if f.LastRequestId != "" {
|
||||||
|
if f.Reverse {
|
||||||
|
conditions = append(conditions, "trace_id > '"+escapeString(f.LastRequestId)+"'")
|
||||||
|
} else {
|
||||||
|
conditions = append(conditions, "trace_id < '"+escapeString(f.LastRequestId)+"'")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
where := strings.Join(conditions, " AND ")
|
where := strings.Join(conditions, " AND ")
|
||||||
orderDir := "ASC"
|
// 默认按时间倒序(最新的在前面),与前端默认行为一致
|
||||||
|
orderDir := "DESC"
|
||||||
if f.Reverse {
|
if f.Reverse {
|
||||||
orderDir = "DESC"
|
orderDir = "ASC"
|
||||||
}
|
}
|
||||||
limit := f.Size
|
limit := f.Size
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
@@ -138,7 +167,7 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
|||||||
if limit > 1000 {
|
if limit > 1000 {
|
||||||
limit = 1000
|
limit = 1000
|
||||||
}
|
}
|
||||||
orderBy := fmt.Sprintf("timestamp %s, node_id %s, server_id %s, trace_id %s", orderDir, orderDir, orderDir, orderDir)
|
orderBy := fmt.Sprintf("timestamp %s, trace_id %s", orderDir, orderDir)
|
||||||
|
|
||||||
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE %s ORDER BY %s LIMIT %d",
|
||||||
table, where, orderBy, limit+1)
|
table, where, orderBy, limit+1)
|
||||||
@@ -155,13 +184,51 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
|
|||||||
rows = append(rows, r)
|
rows = append(rows, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !f.Reverse {
|
||||||
|
if len(rows) > int(limit) {
|
||||||
|
nextCursor = rows[limit].TraceId
|
||||||
|
rows = rows[:limit]
|
||||||
|
}
|
||||||
|
return rows, nextCursor, nil
|
||||||
|
}
|
||||||
|
|
||||||
if len(rows) > int(limit) {
|
if len(rows) > int(limit) {
|
||||||
nextCursor = rows[limit].TraceId
|
|
||||||
rows = rows[:limit]
|
rows = rows[:limit]
|
||||||
}
|
}
|
||||||
|
if len(rows) > 0 {
|
||||||
|
nextCursor = rows[len(rows)-1].TraceId
|
||||||
|
}
|
||||||
|
|
||||||
|
// 实时模式统一返回为“最新在前”,与前端显示和 MySQL 语义一致。
|
||||||
|
for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 {
|
||||||
|
rows[left], rows[right] = rows[right], rows[left]
|
||||||
|
}
|
||||||
return rows, nextCursor, nil
|
return rows, nextCursor, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FindByTraceId 按 trace_id 查询单条日志详情
|
||||||
|
func (s *LogsIngestStore) FindByTraceId(ctx context.Context, traceId string) (*LogsIngestRow, error) {
|
||||||
|
if !s.client.IsConfigured() {
|
||||||
|
return nil, fmt.Errorf("clickhouse: not configured")
|
||||||
|
}
|
||||||
|
if traceId == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
table := quoteIdent("logs_ingest")
|
||||||
|
query := fmt.Sprintf("SELECT timestamp, node_id, cluster_id, server_id, host, ip, method, path, status, bytes_in, bytes_out, cost_ms, ua, referer, log_type, trace_id, firewall_policy_id, firewall_rule_group_id, firewall_rule_set_id, firewall_rule_id, request_headers, request_body, response_headers, response_body FROM %s WHERE trace_id = '%s' LIMIT 1",
|
||||||
|
table, escapeString(traceId))
|
||||||
|
|
||||||
|
var rawRows []map[string]interface{}
|
||||||
|
if err := s.client.Query(ctx, query, &rawRows); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(rawRows) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return mapToLogsIngestRow(rawRows[0]), nil
|
||||||
|
}
|
||||||
|
|
||||||
func quoteIdent(name string) string {
|
func quoteIdent(name string) string {
|
||||||
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||||
}
|
}
|
||||||
@@ -244,7 +311,7 @@ func mapToLogsIngestRow(m map[string]interface{}) *LogsIngestRow {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用)
|
// RowToPB 将 logs_ingest 一行转为 pb.HTTPAccessLog(列表展示用+详情展示)
|
||||||
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -259,6 +326,9 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
|||||||
RemoteAddr: r.IP,
|
RemoteAddr: r.IP,
|
||||||
RequestMethod: r.Method,
|
RequestMethod: r.Method,
|
||||||
RequestPath: r.Path,
|
RequestPath: r.Path,
|
||||||
|
RequestURI: r.Path, // 前端使用 requestURI 显示完整路径
|
||||||
|
Scheme: "http", // 默认 http,日志中未存储实际值
|
||||||
|
Proto: "HTTP/1.1", // 默认值,日志中未存储实际值
|
||||||
Status: int32(r.Status),
|
Status: int32(r.Status),
|
||||||
RequestLength: int64(r.BytesIn),
|
RequestLength: int64(r.BytesIn),
|
||||||
BytesSent: int64(r.BytesOut),
|
BytesSent: int64(r.BytesOut),
|
||||||
@@ -273,6 +343,39 @@ func RowToPB(r *LogsIngestRow) *pb.HTTPAccessLog {
|
|||||||
if r.TimeISO8601() != "" {
|
if r.TimeISO8601() != "" {
|
||||||
a.TimeISO8601 = r.TimeISO8601()
|
a.TimeISO8601 = r.TimeISO8601()
|
||||||
}
|
}
|
||||||
|
// TimeLocal: 用户友好的时间格式 (e.g., "2026-02-07 23:17:12")
|
||||||
|
if !r.Timestamp.IsZero() {
|
||||||
|
a.TimeLocal = r.Timestamp.Format("2006-01-02 15:04:05")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 解析请求头 (JSON -> map[string]*pb.Strings)
|
||||||
|
// ClickHouse 中存储的是 map[string]string 格式
|
||||||
|
if r.RequestHeaders != "" {
|
||||||
|
var headers map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(r.RequestHeaders), &headers); err == nil {
|
||||||
|
a.Header = make(map[string]*pb.Strings)
|
||||||
|
for k, v := range headers {
|
||||||
|
a.Header[k] = &pb.Strings{Values: []string{v}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 解析响应头 (JSON -> map[string]*pb.Strings)
|
||||||
|
if r.ResponseHeaders != "" {
|
||||||
|
var headers map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(r.ResponseHeaders), &headers); err == nil {
|
||||||
|
a.SentHeader = make(map[string]*pb.Strings)
|
||||||
|
for k, v := range headers {
|
||||||
|
a.SentHeader[k] = &pb.Strings{Values: []string{v}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 请求体
|
||||||
|
if r.RequestBody != "" {
|
||||||
|
a.RequestBody = []byte(r.RequestBody)
|
||||||
|
}
|
||||||
|
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -261,3 +261,23 @@ func (this *SysSettingDAO) ReadDatabaseConfig(tx *dbs.Tx) (config *systemconfigs
|
|||||||
}
|
}
|
||||||
return config, nil
|
return config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadClickHouseConfig 读取ClickHouse配置
|
||||||
|
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
||||||
|
valueJSON, err := this.ReadSetting(tx, "clickhouseConfig")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(valueJSON) == 0 {
|
||||||
|
return &systemconfigs.ClickHouseSetting{
|
||||||
|
Port: 8123,
|
||||||
|
Database: "default",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
var config = &systemconfigs.ClickHouseSetting{}
|
||||||
|
err = json.Unmarshal(valueJSON, config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return config, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -66,25 +66,3 @@ func (this *SysSettingDAO) ReadUserSenderConfig(tx *dbs.Tx) (*userconfigs.UserSe
|
|||||||
return config, nil
|
return config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadClickHouseConfig 读取 ClickHouse 连接配置(后台页面配置,用于访问日志 logs_ingest 查询)
|
|
||||||
func (this *SysSettingDAO) ReadClickHouseConfig(tx *dbs.Tx) (*systemconfigs.ClickHouseSetting, error) {
|
|
||||||
valueJSON, err := this.ReadSetting(tx, systemconfigs.SettingCodeClickHouseConfig)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out := &systemconfigs.ClickHouseSetting{Port: 8123, Database: "default"}
|
|
||||||
if len(valueJSON) == 0 {
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
err = json.Unmarshal(valueJSON, out)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if out.Port <= 0 {
|
|
||||||
out.Port = 8123
|
|
||||||
}
|
|
||||||
if out.Database == "" {
|
|
||||||
out.Database = "default"
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||||
"github.com/go-sql-driver/mysql"
|
"github.com/go-sql-driver/mysql"
|
||||||
"github.com/iwind/TeaGo/dbs"
|
"github.com/iwind/TeaGo/dbs"
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -60,14 +59,7 @@ func CheckHasFreeSpace() bool {
|
|||||||
}
|
}
|
||||||
LocalDatabaseDataDir = dir
|
LocalDatabaseDataDir = dir
|
||||||
|
|
||||||
var stat unix.Statfs_t
|
return checkHasFreeSpace(dir)
|
||||||
err = unix.Statfs(dir, &stat)
|
|
||||||
if err != nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
|
|
||||||
return availableSpace > minFreeSpaceGB
|
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
18
EdgeAPI/internal/db/utils/disk_unix.go
Normal file
18
EdgeAPI/internal/db/utils/disk_unix.go
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package dbutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
func checkHasFreeSpace(dir string) bool {
|
||||||
|
var stat unix.Statfs_t
|
||||||
|
err := unix.Statfs(dir, &stat)
|
||||||
|
if err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
var availableSpace = (stat.Bavail * uint64(stat.Bsize)) / (1 << 30) // GB
|
||||||
|
return availableSpace > minFreeSpaceGB
|
||||||
|
}
|
||||||
7
EdgeAPI/internal/db/utils/disk_windows.go
Normal file
7
EdgeAPI/internal/db/utils/disk_windows.go
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package dbutils
|
||||||
|
|
||||||
|
func checkHasFreeSpace(dir string) bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -4,8 +4,9 @@
|
|||||||
package nodes
|
package nodes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
//"context"
|
||||||
"github.com/shirou/gopsutil/v3/cpu"
|
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||||
|
//"github.com/shirou/gopsutil/v3/cpu"
|
||||||
"github.com/shirou/gopsutil/v3/mem"
|
"github.com/shirou/gopsutil/v3/mem"
|
||||||
"math"
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -21,7 +22,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
|
|||||||
var windowsLoadLocker = &sync.Mutex{}
|
var windowsLoadLocker = &sync.Mutex{}
|
||||||
|
|
||||||
// 更新内存
|
// 更新内存
|
||||||
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
|
||||||
stat, err := mem.VirtualMemory()
|
stat, err := mem.VirtualMemory()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
status.Error = err.Error()
|
status.Error = err.Error()
|
||||||
@@ -32,14 +33,14 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 更新负载
|
// 更新负载
|
||||||
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
|
||||||
timestamp := time.Now().Unix()
|
timestamp := time.Now().Unix()
|
||||||
|
|
||||||
currentLoad := 0
|
currentLoad := 0
|
||||||
info, err := cpu.ProcInfo()
|
//info, err := cpu.ProcInfo()
|
||||||
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
//if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||||
currentLoad = int(info[0].ProcessorQueueLength)
|
// currentLoad = int(info[0].ProcessorQueueLength)
|
||||||
}
|
//}
|
||||||
|
|
||||||
// 删除15分钟之前的数据
|
// 删除15分钟之前的数据
|
||||||
windowsLoadLocker.Lock()
|
windowsLoadLocker.Lock()
|
||||||
@@ -93,9 +94,9 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
|||||||
windowsLoadLocker.Unlock()
|
windowsLoadLocker.Unlock()
|
||||||
|
|
||||||
// 在老Windows上不显示错误
|
// 在老Windows上不显示错误
|
||||||
if err == context.DeadlineExceeded {
|
//if err == context.DeadlineExceeded {
|
||||||
err = nil
|
// err = nil
|
||||||
}
|
//}
|
||||||
status.Load1m = load1
|
status.Load1m = load1
|
||||||
status.Load5m = load5
|
status.Load5m = load5
|
||||||
status.Load15m = load15
|
status.Load15m = load15
|
||||||
|
|||||||
@@ -144,6 +144,9 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
|
|||||||
NodeId: req.NodeId,
|
NodeId: req.NodeId,
|
||||||
ClusterId: req.NodeClusterId,
|
ClusterId: req.NodeClusterId,
|
||||||
LastRequestId: req.RequestId,
|
LastRequestId: req.RequestId,
|
||||||
|
Keyword: req.Keyword,
|
||||||
|
Ip: req.Ip,
|
||||||
|
Domain: req.Domain,
|
||||||
}
|
}
|
||||||
if req.ServerId > 0 {
|
if req.ServerId > 0 {
|
||||||
f.ServerIds = []int64{req.ServerId}
|
f.ServerIds = []int64{req.ServerId}
|
||||||
@@ -216,7 +219,10 @@ func (this *HTTPAccessLogService) listHTTPAccessLogsFromClickHouse(ctx context.C
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hasMore := nextCursor != ""
|
hasMore := false
|
||||||
|
if !req.Reverse {
|
||||||
|
hasMore = nextCursor != ""
|
||||||
|
}
|
||||||
return &pb.ListHTTPAccessLogsResponse{
|
return &pb.ListHTTPAccessLogsResponse{
|
||||||
HttpAccessLogs: result,
|
HttpAccessLogs: result,
|
||||||
AccessLogs: result,
|
AccessLogs: result,
|
||||||
@@ -233,6 +239,28 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 优先从 ClickHouse 查询
|
||||||
|
store := clickhouse.NewLogsIngestStore()
|
||||||
|
if store.Client().IsConfigured() {
|
||||||
|
row, err := store.FindByTraceId(ctx, req.RequestId)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if row != nil {
|
||||||
|
// 检查权限
|
||||||
|
if userId > 0 {
|
||||||
|
var tx = this.NullTx()
|
||||||
|
err = models.SharedServerDAO.CheckUserServer(tx, userId, int64(row.ServerId))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a := clickhouse.RowToPB(row)
|
||||||
|
return &pb.FindHTTPAccessLogResponse{HttpAccessLog: a}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果 ClickHouse 未配置或未找到,则回退到 MySQL
|
||||||
var tx = this.NullTx()
|
var tx = this.NullTx()
|
||||||
|
|
||||||
accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
|
accessLog, err := models.SharedHTTPAccessLogDAO.FindAccessLogWithRequestId(tx, req.RequestId)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
|
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
|
||||||
return accesslogs.SharedStorageManager.WriteMySQL()
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
|
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
|
||||||
|
|||||||
12
EdgeAdmin/.gitignore
vendored
12
EdgeAdmin/.gitignore
vendored
@@ -0,0 +1,12 @@
|
|||||||
|
部署启动手册.md
|
||||||
|
|
||||||
|
# Runtime Data
|
||||||
|
data/
|
||||||
|
logs/
|
||||||
|
|
||||||
|
# Local Configs
|
||||||
|
configs/api_admin.yaml
|
||||||
|
configs/server.yaml
|
||||||
|
|
||||||
|
# Build Artifacts
|
||||||
|
bin/
|
||||||
@@ -7,6 +7,47 @@ Tea.context(function () {
|
|||||||
this.accessLogs = []
|
this.accessLogs = []
|
||||||
this.isLoaded = false
|
this.isLoaded = false
|
||||||
|
|
||||||
|
this.mergeAccessLogs = function (newAccessLogs, regions, wafInfos) {
|
||||||
|
let mergedLogs = newAccessLogs.concat(this.accessLogs)
|
||||||
|
let result = []
|
||||||
|
let seenRequestIds = {}
|
||||||
|
|
||||||
|
let that = this
|
||||||
|
mergedLogs.forEach(function (accessLog) {
|
||||||
|
if (accessLog == null) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let requestId = accessLog.requestId || ""
|
||||||
|
if (requestId.length > 0) {
|
||||||
|
if (seenRequestIds[requestId] === true) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seenRequestIds[requestId] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
that.formatTime(accessLog)
|
||||||
|
|
||||||
|
let region = regions[accessLog.remoteAddr]
|
||||||
|
if (typeof (region) == "string") {
|
||||||
|
accessLog.region = region
|
||||||
|
} else if (typeof accessLog.region != "string") {
|
||||||
|
accessLog.region = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
let wafInfo = wafInfos[accessLog.firewallRuleSetId]
|
||||||
|
if (accessLog.firewallRuleSetId > 0 && typeof (wafInfo) == "object") {
|
||||||
|
accessLog.wafInfo = wafInfo
|
||||||
|
} else if (typeof accessLog.wafInfo == "undefined") {
|
||||||
|
accessLog.wafInfo = null
|
||||||
|
}
|
||||||
|
|
||||||
|
result.push(accessLog)
|
||||||
|
})
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
this.load = function () {
|
this.load = function () {
|
||||||
// 如果有弹窗时,暂时不更新
|
// 如果有弹窗时,暂时不更新
|
||||||
if (teaweb.hasPopup()) {
|
if (teaweb.hasPopup()) {
|
||||||
@@ -27,30 +68,17 @@ Tea.context(function () {
|
|||||||
nodeId: this.nodeId
|
nodeId: this.nodeId
|
||||||
})
|
})
|
||||||
.success(function (resp) {
|
.success(function (resp) {
|
||||||
this.accessLogs = resp.data.accessLogs.concat(this.accessLogs)
|
let newAccessLogs = resp.data.accessLogs || []
|
||||||
|
this.accessLogs = this.mergeAccessLogs(newAccessLogs, resp.data.regions || {}, resp.data.wafInfos || {})
|
||||||
// 添加区域信息
|
|
||||||
let that = this
|
|
||||||
this.accessLogs.forEach(function (accessLog) {
|
|
||||||
that.formatTime(accessLog)
|
|
||||||
if (typeof (resp.data.regions[accessLog.remoteAddr]) == "string") {
|
|
||||||
accessLog.region = resp.data.regions[accessLog.remoteAddr]
|
|
||||||
} else {
|
|
||||||
accessLog.region = ""
|
|
||||||
}
|
|
||||||
if (accessLog.firewallRuleSetId > 0 && typeof (resp.data.wafInfos[accessLog.firewallRuleSetId]) == "object") {
|
|
||||||
accessLog.wafInfo = resp.data.wafInfos[accessLog.firewallRuleSetId]
|
|
||||||
} else {
|
|
||||||
accessLog.wafInfo = null
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
let max = 100
|
let max = 100
|
||||||
if (this.accessLogs.length > max) {
|
if (this.accessLogs.length > max) {
|
||||||
this.accessLogs = this.accessLogs.slice(0, max)
|
this.accessLogs = this.accessLogs.slice(0, max)
|
||||||
}
|
}
|
||||||
this.hasMore = resp.data.hasMore
|
this.hasMore = resp.data.hasMore
|
||||||
this.requestId = resp.data.requestId
|
if (typeof resp.data.requestId == "string" && resp.data.requestId.length > 0) {
|
||||||
|
this.requestId = resp.data.requestId
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.done(function () {
|
.done(function () {
|
||||||
if (!this.isLoaded) {
|
if (!this.isLoaded) {
|
||||||
@@ -79,4 +107,4 @@ Tea.context(function () {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ require (
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/kr/text v0.2.0 // indirect
|
github.com/kr/text v0.2.0 // indirect
|
||||||
|
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 // indirect
|
||||||
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
|
github.com/oschwald/maxminddb-golang v1.13.0 // indirect
|
||||||
golang.org/x/sys v0.38.0 // indirect
|
golang.org/x/sys v0.38.0 // indirect
|
||||||
golang.org/x/text v0.31.0 // indirect
|
golang.org/x/text v0.31.0 // indirect
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
|||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879 h1:ouxvoYN6WL482nK1zS1IplyniAut3G4D+0N5JS0YTuw=
|
||||||
|
github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20260202110255-d427b4dcd879/go.mod h1:+mNMTBuDMdEGhWzoQgc6kBdqeaQpWh5ba8zqmp2MxCU=
|
||||||
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
|
github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
|
||||||
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
|
github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
|
||||||
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
|
github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
//go:build plus
|
|
||||||
|
|
||||||
package systemconfigs
|
package systemconfigs
|
||||||
|
|
||||||
|
|||||||
16
EdgeNode/.gitignore
vendored
16
EdgeNode/.gitignore
vendored
@@ -0,0 +1,16 @@
|
|||||||
|
# Windows local development
|
||||||
|
*_windows.go
|
||||||
|
configs/api_node.yaml
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# Build binaries
|
||||||
|
bin/
|
||||||
|
|
||||||
|
# Runtime Data
|
||||||
|
data/
|
||||||
|
configs/node.json
|
||||||
|
logs/
|
||||||
|
opt/
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ func FromHTTPAccessLog(l *pb.HTTPAccessLog, clusterId int64) (ingest IngestLog,
|
|||||||
Host: l.GetHost(),
|
Host: l.GetHost(),
|
||||||
IP: l.GetRawRemoteAddr(),
|
IP: l.GetRawRemoteAddr(),
|
||||||
Method: l.GetRequestMethod(),
|
Method: l.GetRequestMethod(),
|
||||||
Path: l.GetRequestPath(),
|
Path: l.GetRequestURI(), // 使用 RequestURI 以包含查询参数
|
||||||
Status: l.GetStatus(),
|
Status: l.GetStatus(),
|
||||||
BytesIn: l.GetRequestLength(),
|
BytesIn: l.GetRequestLength(),
|
||||||
BytesOut: l.GetBytesSent(),
|
BytesOut: l.GetBytesSent(),
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -197,10 +196,7 @@ func (this *AppCmd) runStart() {
|
|||||||
_ = os.Setenv("EdgeBackground", "on")
|
_ = os.Setenv("EdgeBackground", "on")
|
||||||
|
|
||||||
var cmd = exec.Command(this.exe())
|
var cmd = exec.Command(this.exe())
|
||||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
configureSysProcAttr(cmd)
|
||||||
Foreground: false,
|
|
||||||
Setsid: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := cmd.Start()
|
err := cmd.Start()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
15
EdgeNode/internal/apps/app_utils_unix.go
Normal file
15
EdgeNode/internal/apps/app_utils_unix.go
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os/exec"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func configureSysProcAttr(cmd *exec.Cmd) {
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||||
|
Foreground: false,
|
||||||
|
Setsid: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package caches
|
package caches
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||||
@@ -10,7 +9,6 @@ import (
|
|||||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||||
"github.com/iwind/TeaGo/lists"
|
"github.com/iwind/TeaGo/lists"
|
||||||
"github.com/iwind/TeaGo/types"
|
"github.com/iwind/TeaGo/types"
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
@@ -191,44 +189,6 @@ func (this *Manager) StorageMap() map[int64]StorageInterface {
|
|||||||
return this.storageMap
|
return this.storageMap
|
||||||
}
|
}
|
||||||
|
|
||||||
// TotalDiskSize 消耗的磁盘尺寸
|
|
||||||
func (this *Manager) TotalDiskSize() int64 {
|
|
||||||
this.locker.RLock()
|
|
||||||
defer this.locker.RUnlock()
|
|
||||||
|
|
||||||
var total = int64(0)
|
|
||||||
var sidMap = map[string]bool{} // partition sid => bool
|
|
||||||
for _, storage := range this.storageMap {
|
|
||||||
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
|
|
||||||
fileStorage, ok := storage.(*FileStorage)
|
|
||||||
if ok {
|
|
||||||
var options = fileStorage.options // copy
|
|
||||||
if options != nil {
|
|
||||||
var dir = options.Dir // copy
|
|
||||||
if len(dir) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var stat = &unix.Statfs_t{}
|
|
||||||
err := unix.Statfs(dir, stat)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
|
|
||||||
if sidMap[sid] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sidMap[sid] = true
|
|
||||||
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if total < 0 {
|
|
||||||
total = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return total
|
|
||||||
}
|
|
||||||
|
|
||||||
// TotalMemorySize 消耗的内存尺寸
|
// TotalMemorySize 消耗的内存尺寸
|
||||||
func (this *Manager) TotalMemorySize() int64 {
|
func (this *Manager) TotalMemorySize() int64 {
|
||||||
|
|||||||
48
EdgeNode/internal/caches/manager_unix.go
Normal file
48
EdgeNode/internal/caches/manager_unix.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package caches
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TotalDiskSize 消耗的磁盘尺寸
|
||||||
|
func (this *Manager) TotalDiskSize() int64 {
|
||||||
|
this.locker.RLock()
|
||||||
|
defer this.locker.RUnlock()
|
||||||
|
|
||||||
|
var total = int64(0)
|
||||||
|
var sidMap = map[string]bool{} // partition sid => bool
|
||||||
|
for _, storage := range this.storageMap {
|
||||||
|
// 这里不能直接用 storage.TotalDiskSize() 相加,因为多个缓存策略缓存目录可能处在同一个分区目录下
|
||||||
|
fileStorage, ok := storage.(*FileStorage)
|
||||||
|
if ok {
|
||||||
|
var options = fileStorage.options // copy
|
||||||
|
if options != nil {
|
||||||
|
var dir = options.Dir // copy
|
||||||
|
if len(dir) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var stat = &unix.Statfs_t{}
|
||||||
|
err := unix.Statfs(dir, stat)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var sid = fmt.Sprintf("%d_%d", stat.Fsid.Val[0], stat.Fsid.Val[1])
|
||||||
|
if sidMap[sid] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sidMap[sid] = true
|
||||||
|
total += int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize) // we add extra int64() for darwin
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if total < 0 {
|
||||||
|
total = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
|
}
|
||||||
@@ -35,7 +35,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -736,7 +735,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
|||||||
|
|
||||||
// 尝试锁定,如果锁定失败,则直接返回
|
// 尝试锁定,如果锁定失败,则直接返回
|
||||||
fsutils.WriterLimiter.Ack()
|
fsutils.WriterLimiter.Ack()
|
||||||
err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
err = tryLockFile(int(writer.Fd()))
|
||||||
fsutils.WriterLimiter.Release()
|
fsutils.WriterLimiter.Release()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
removeOnFailure = false
|
removeOnFailure = false
|
||||||
|
|||||||
9
EdgeNode/internal/caches/storage_utils_unix.go
Normal file
9
EdgeNode/internal/caches/storage_utils_unix.go
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package caches
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func tryLockFile(fd int) error {
|
||||||
|
return syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB)
|
||||||
|
}
|
||||||
@@ -19,11 +19,8 @@ import (
|
|||||||
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
|
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
|
||||||
_ "github.com/biessek/golang-ico"
|
_ "github.com/biessek/golang-ico"
|
||||||
"github.com/iwind/TeaGo/types"
|
"github.com/iwind/TeaGo/types"
|
||||||
"github.com/iwind/gowebp"
|
|
||||||
_ "golang.org/x/image/bmp"
|
_ "golang.org/x/image/bmp"
|
||||||
_ "golang.org/x/image/webp"
|
_ "golang.org/x/image/webp"
|
||||||
"image"
|
|
||||||
"image/gif"
|
|
||||||
_ "image/jpeg"
|
_ "image/jpeg"
|
||||||
_ "image/png"
|
_ "image/png"
|
||||||
"io"
|
"io"
|
||||||
@@ -1045,169 +1042,6 @@ func (this *HTTPWriter) calculateStaleLife() int {
|
|||||||
return staleLife
|
return staleLife
|
||||||
}
|
}
|
||||||
|
|
||||||
// 结束WebP
|
|
||||||
func (this *HTTPWriter) finishWebP() {
|
|
||||||
// 处理WebP
|
|
||||||
if this.webpIsEncoding {
|
|
||||||
atomic.AddInt32(&webPThreads, 1)
|
|
||||||
defer func() {
|
|
||||||
atomic.AddInt32(&webPThreads, -1)
|
|
||||||
}()
|
|
||||||
|
|
||||||
var webpCacheWriter caches.Writer
|
|
||||||
|
|
||||||
// 准备WebP Cache
|
|
||||||
if this.cacheReader != nil || this.cacheWriter != nil {
|
|
||||||
var cacheKey = ""
|
|
||||||
var expiredAt int64 = 0
|
|
||||||
|
|
||||||
if this.cacheReader != nil {
|
|
||||||
cacheKey = this.req.cacheKey + caches.SuffixWebP
|
|
||||||
expiredAt = this.cacheReader.ExpiresAt()
|
|
||||||
} else if this.cacheWriter != nil {
|
|
||||||
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
|
|
||||||
expiredAt = this.cacheWriter.ExpiredAt()
|
|
||||||
}
|
|
||||||
|
|
||||||
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
|
|
||||||
if webpCacheWriter != nil {
|
|
||||||
// 写入Header
|
|
||||||
for k, v := range this.Header() {
|
|
||||||
if this.shouldIgnoreHeader(k) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// 这里是原始的数据,不需要内容编码
|
|
||||||
if k == "Content-Encoding" || k == "Transfer-Encoding" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, v1 := range v {
|
|
||||||
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
|
|
||||||
if err != nil {
|
|
||||||
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
|
|
||||||
_ = webpCacheWriter.Discard()
|
|
||||||
webpCacheWriter = nil
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if webpCacheWriter != nil {
|
|
||||||
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
|
|
||||||
teeWriter.OnFail(func(err error) {
|
|
||||||
if webpCacheWriter != nil {
|
|
||||||
_ = webpCacheWriter.Discard()
|
|
||||||
}
|
|
||||||
webpCacheWriter = nil
|
|
||||||
})
|
|
||||||
this.writer = teeWriter
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var reader = readers.NewBytesCounterReader(this.rawReader)
|
|
||||||
|
|
||||||
var imageData image.Image
|
|
||||||
var gifImage *gif.GIF
|
|
||||||
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
|
|
||||||
var err error
|
|
||||||
if isGif {
|
|
||||||
gifImage, err = gif.DecodeAll(reader)
|
|
||||||
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
|
|
||||||
webPIgnoreURLSet.Push(this.req.URL())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
imageData, _, err = image.Decode(reader)
|
|
||||||
if imageData != nil {
|
|
||||||
var bound = imageData.Bounds()
|
|
||||||
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
|
|
||||||
webPIgnoreURLSet.Push(this.req.URL())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
// 发生了错误终止处理
|
|
||||||
webPIgnoreURLSet.Push(this.req.URL())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var f = types.Float32(this.webpQuality)
|
|
||||||
if f <= 0 || f > 100 {
|
|
||||||
if this.size > (8<<20) || this.size <= 0 {
|
|
||||||
f = 30
|
|
||||||
} else if this.size > (1 << 20) {
|
|
||||||
f = 50
|
|
||||||
} else if this.size > (128 << 10) {
|
|
||||||
f = 60
|
|
||||||
} else {
|
|
||||||
f = 75
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if imageData != nil {
|
|
||||||
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
|
|
||||||
Lossless: false,
|
|
||||||
Quality: f,
|
|
||||||
Exact: true,
|
|
||||||
})
|
|
||||||
} else if gifImage != nil {
|
|
||||||
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
|
||||||
|
|
||||||
anim.WebPAnimEncoderOptions.SetKmin(9)
|
|
||||||
anim.WebPAnimEncoderOptions.SetKmax(17)
|
|
||||||
var webpConfig = gowebp.NewWebpConfig()
|
|
||||||
//webpConfig.SetLossless(1)
|
|
||||||
webpConfig.SetQuality(f)
|
|
||||||
|
|
||||||
var timeline = 0
|
|
||||||
var lastErr error
|
|
||||||
for i, img := range gifImage.Image {
|
|
||||||
err = anim.AddFrame(img, timeline, webpConfig)
|
|
||||||
if err != nil {
|
|
||||||
// 有错误直接跳过
|
|
||||||
lastErr = err
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
timeline += gifImage.Delay[i] * 10
|
|
||||||
}
|
|
||||||
if lastErr != nil {
|
|
||||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
|
||||||
}
|
|
||||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
err = anim.Encode(this.writer)
|
|
||||||
}
|
|
||||||
|
|
||||||
anim.ReleaseMemory()
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil && !this.req.canIgnore(err) {
|
|
||||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err == nil && webpCacheWriter != nil {
|
|
||||||
err = webpCacheWriter.Close()
|
|
||||||
if err != nil {
|
|
||||||
_ = webpCacheWriter.Discard()
|
|
||||||
} else {
|
|
||||||
this.cacheStorage.AddToList(&caches.Item{
|
|
||||||
Type: webpCacheWriter.ItemType(),
|
|
||||||
Key: webpCacheWriter.Key(),
|
|
||||||
ExpiresAt: webpCacheWriter.ExpiredAt(),
|
|
||||||
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
|
|
||||||
HeaderSize: webpCacheWriter.HeaderSize(),
|
|
||||||
BodySize: webpCacheWriter.BodySize(),
|
|
||||||
Host: this.req.ReqHost,
|
|
||||||
ServerId: this.req.ReqServer.Id,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 结束缓存相关处理
|
// 结束缓存相关处理
|
||||||
func (this *HTTPWriter) finishCache() {
|
func (this *HTTPWriter) finishCache() {
|
||||||
|
|||||||
181
EdgeNode/internal/nodes/http_writer_ext_unix.go
Normal file
181
EdgeNode/internal/nodes/http_writer_ext_unix.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package nodes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
|
||||||
|
"github.com/iwind/TeaGo/types"
|
||||||
|
"github.com/iwind/gowebp"
|
||||||
|
"image"
|
||||||
|
"image/gif"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 结束WebP
|
||||||
|
func (this *HTTPWriter) finishWebP() {
|
||||||
|
// 处理WebP
|
||||||
|
if this.webpIsEncoding {
|
||||||
|
atomic.AddInt32(&webPThreads, 1)
|
||||||
|
defer func() {
|
||||||
|
atomic.AddInt32(&webPThreads, -1)
|
||||||
|
}()
|
||||||
|
|
||||||
|
var webpCacheWriter caches.Writer
|
||||||
|
|
||||||
|
// 准备WebP Cache
|
||||||
|
if this.cacheReader != nil || this.cacheWriter != nil {
|
||||||
|
var cacheKey = ""
|
||||||
|
var expiredAt int64 = 0
|
||||||
|
|
||||||
|
if this.cacheReader != nil {
|
||||||
|
cacheKey = this.req.cacheKey + caches.SuffixWebP
|
||||||
|
expiredAt = this.cacheReader.ExpiresAt()
|
||||||
|
} else if this.cacheWriter != nil {
|
||||||
|
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
|
||||||
|
expiredAt = this.cacheWriter.ExpiredAt()
|
||||||
|
}
|
||||||
|
|
||||||
|
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, -1, false)
|
||||||
|
if webpCacheWriter != nil {
|
||||||
|
// 写入Header
|
||||||
|
for k, v := range this.Header() {
|
||||||
|
if this.shouldIgnoreHeader(k) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// 这里是原始的数据,不需要内容编码
|
||||||
|
if k == "Content-Encoding" || k == "Transfer-Encoding" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, v1 := range v {
|
||||||
|
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
|
||||||
|
if err != nil {
|
||||||
|
remotelogs.Error("HTTP_WRITER", "write webp cache failed: "+err.Error())
|
||||||
|
_ = webpCacheWriter.Discard()
|
||||||
|
webpCacheWriter = nil
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if webpCacheWriter != nil {
|
||||||
|
var teeWriter = writers.NewTeeWriterCloser(this.writer, webpCacheWriter)
|
||||||
|
teeWriter.OnFail(func(err error) {
|
||||||
|
if webpCacheWriter != nil {
|
||||||
|
_ = webpCacheWriter.Discard()
|
||||||
|
}
|
||||||
|
webpCacheWriter = nil
|
||||||
|
})
|
||||||
|
this.writer = teeWriter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader = readers.NewBytesCounterReader(this.rawReader)
|
||||||
|
|
||||||
|
var imageData image.Image
|
||||||
|
var gifImage *gif.GIF
|
||||||
|
var isGif = strings.Contains(this.webpOriginContentType, "image/gif")
|
||||||
|
var err error
|
||||||
|
if isGif {
|
||||||
|
gifImage, err = gif.DecodeAll(reader)
|
||||||
|
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
|
||||||
|
webPIgnoreURLSet.Push(this.req.URL())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
imageData, _, err = image.Decode(reader)
|
||||||
|
if imageData != nil {
|
||||||
|
var bound = imageData.Bounds()
|
||||||
|
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
|
||||||
|
webPIgnoreURLSet.Push(this.req.URL())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
// 发生了错误终止处理
|
||||||
|
webPIgnoreURLSet.Push(this.req.URL())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var f = types.Float32(this.webpQuality)
|
||||||
|
if f <= 0 || f > 100 {
|
||||||
|
if this.size > (8<<20) || this.size <= 0 {
|
||||||
|
f = 30
|
||||||
|
} else if this.size > (1 << 20) {
|
||||||
|
f = 50
|
||||||
|
} else if this.size > (128 << 10) {
|
||||||
|
f = 60
|
||||||
|
} else {
|
||||||
|
f = 75
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if imageData != nil {
|
||||||
|
err = gowebp.Encode(this.writer, imageData, &gowebp.Options{
|
||||||
|
Lossless: false,
|
||||||
|
Quality: f,
|
||||||
|
Exact: true,
|
||||||
|
})
|
||||||
|
} else if gifImage != nil {
|
||||||
|
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||||
|
|
||||||
|
anim.WebPAnimEncoderOptions.SetKmin(9)
|
||||||
|
anim.WebPAnimEncoderOptions.SetKmax(17)
|
||||||
|
var webpConfig = gowebp.NewWebpConfig()
|
||||||
|
//webpConfig.SetLossless(1)
|
||||||
|
webpConfig.SetQuality(f)
|
||||||
|
|
||||||
|
var timeline = 0
|
||||||
|
var lastErr error
|
||||||
|
for i, img := range gifImage.Image {
|
||||||
|
err = anim.AddFrame(img, timeline, webpConfig)
|
||||||
|
if err != nil {
|
||||||
|
// 有错误直接跳过
|
||||||
|
lastErr = err
|
||||||
|
err = nil
|
||||||
|
}
|
||||||
|
timeline += gifImage.Delay[i] * 10
|
||||||
|
}
|
||||||
|
if lastErr != nil {
|
||||||
|
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
||||||
|
}
|
||||||
|
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
err = anim.Encode(this.writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
anim.ReleaseMemory()
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil && !this.req.canIgnore(err) {
|
||||||
|
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil && webpCacheWriter != nil {
|
||||||
|
err = webpCacheWriter.Close()
|
||||||
|
if err != nil {
|
||||||
|
_ = webpCacheWriter.Discard()
|
||||||
|
} else {
|
||||||
|
this.cacheStorage.AddToList(&caches.Item{
|
||||||
|
Type: webpCacheWriter.ItemType(),
|
||||||
|
Key: webpCacheWriter.Key(),
|
||||||
|
ExpiresAt: webpCacheWriter.ExpiredAt(),
|
||||||
|
StaleAt: webpCacheWriter.ExpiredAt() + int64(this.calculateStaleLife()),
|
||||||
|
HeaderSize: webpCacheWriter.HeaderSize(),
|
||||||
|
BodySize: webpCacheWriter.BodySize(),
|
||||||
|
Host: this.req.ReqHost,
|
||||||
|
ServerId: this.req.ReqServer.Id,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||||
//go:build !arm64
|
//go:build !arm64 && !windows
|
||||||
// +build !arm64
|
|
||||||
|
|
||||||
package nodes
|
package nodes
|
||||||
|
|
||||||
@@ -4,8 +4,7 @@
|
|||||||
package nodes
|
package nodes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||||
"github.com/shirou/gopsutil/v3/cpu"
|
|
||||||
"github.com/shirou/gopsutil/v3/mem"
|
"github.com/shirou/gopsutil/v3/mem"
|
||||||
"math"
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -21,7 +20,7 @@ var windowsLoadValues = []*WindowsLoadValue{}
|
|||||||
var windowsLoadLocker = &sync.Mutex{}
|
var windowsLoadLocker = &sync.Mutex{}
|
||||||
|
|
||||||
// 更新内存
|
// 更新内存
|
||||||
func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
func (this *NodeStatusExecutor) updateMem(status *nodeconfigs.NodeStatus) {
|
||||||
stat, err := mem.VirtualMemory()
|
stat, err := mem.VirtualMemory()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
status.Error = err.Error()
|
status.Error = err.Error()
|
||||||
@@ -32,14 +31,16 @@ func (this *NodeStatusExecutor) updateMem(status *NodeStatus) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 更新负载
|
// 更新负载
|
||||||
func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
func (this *NodeStatusExecutor) updateLoad(status *nodeconfigs.NodeStatus) {
|
||||||
timestamp := time.Now().Unix()
|
timestamp := time.Now().Unix()
|
||||||
|
|
||||||
currentLoad := 0
|
currentLoad := 0
|
||||||
info, err := cpu.ProcInfo()
|
/*
|
||||||
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
info, err := cpu.ProcInfo()
|
||||||
currentLoad = int(info[0].ProcessorQueueLength)
|
if err == nil && len(info) > 0 && info[0].ProcessorQueueLength < 1000 {
|
||||||
}
|
currentLoad = int(info[0].ProcessorQueueLength)
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
// 删除15分钟之前的数据
|
// 删除15分钟之前的数据
|
||||||
windowsLoadLocker.Lock()
|
windowsLoadLocker.Lock()
|
||||||
@@ -93,9 +94,11 @@ func (this *NodeStatusExecutor) updateLoad(status *NodeStatus) {
|
|||||||
windowsLoadLocker.Unlock()
|
windowsLoadLocker.Unlock()
|
||||||
|
|
||||||
// 在老Windows上不显示错误
|
// 在老Windows上不显示错误
|
||||||
if err == context.DeadlineExceeded {
|
/*
|
||||||
err = nil
|
if err == context.DeadlineExceeded {
|
||||||
}
|
err = nil
|
||||||
|
}
|
||||||
|
*/
|
||||||
status.Load1m = load1
|
status.Load1m = load1
|
||||||
status.Load5m = load5
|
status.Load5m = load5
|
||||||
status.Load15m = load15
|
status.Load15m = load15
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
package fsutils
|
package fsutils
|
||||||
|
|
||||||
@@ -4,20 +4,9 @@ package fsutils
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StatDevice device contains the path
|
|
||||||
func StatDevice(path string) (*StatResult, error) {
|
|
||||||
var stat = &unix.Statfs_t{}
|
|
||||||
err := unix.Statfs(path, stat)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return NewStatResult(stat), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var locker = &sync.RWMutex{}
|
var locker = &sync.RWMutex{}
|
||||||
var cacheMap = map[string]*StatResult{} // path => StatResult
|
var cacheMap = map[string]*StatResult{} // path => StatResult
|
||||||
|
|
||||||
@@ -44,38 +33,3 @@ func StatDeviceCache(path string) (*StatResult, error) {
|
|||||||
cacheMap[path] = stat
|
cacheMap[path] = stat
|
||||||
return stat, nil
|
return stat, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type StatResult struct {
|
|
||||||
rawStat *unix.Statfs_t
|
|
||||||
blockSize uint64
|
|
||||||
|
|
||||||
updatedAt int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
|
|
||||||
var blockSize = rawStat.Bsize
|
|
||||||
if blockSize < 0 {
|
|
||||||
blockSize = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return &StatResult{
|
|
||||||
rawStat: rawStat,
|
|
||||||
blockSize: uint64(blockSize),
|
|
||||||
updatedAt: fasttime.Now().Unix(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *StatResult) FreeSize() uint64 {
|
|
||||||
return this.rawStat.Bfree * this.blockSize
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *StatResult) TotalSize() uint64 {
|
|
||||||
return this.rawStat.Blocks * this.blockSize
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *StatResult) UsedSize() uint64 {
|
|
||||||
if this.rawStat.Bfree <= this.rawStat.Blocks {
|
|
||||||
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|||||||
54
EdgeNode/internal/utils/fs/stat_unix.go
Normal file
54
EdgeNode/internal/utils/fs/stat_unix.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package fsutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StatDevice device contains the path
|
||||||
|
func StatDevice(path string) (*StatResult, error) {
|
||||||
|
var stat = &unix.Statfs_t{}
|
||||||
|
err := unix.Statfs(path, stat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return NewStatResult(stat), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type StatResult struct {
|
||||||
|
rawStat *unix.Statfs_t
|
||||||
|
blockSize uint64
|
||||||
|
|
||||||
|
updatedAt int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
|
||||||
|
var blockSize = rawStat.Bsize
|
||||||
|
if blockSize < 0 {
|
||||||
|
blockSize = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StatResult{
|
||||||
|
rawStat: rawStat,
|
||||||
|
blockSize: uint64(blockSize),
|
||||||
|
updatedAt: fasttime.Now().Unix(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *StatResult) FreeSize() uint64 {
|
||||||
|
return this.rawStat.Bfree * this.blockSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *StatResult) TotalSize() uint64 {
|
||||||
|
return this.rawStat.Blocks * this.blockSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *StatResult) UsedSize() uint64 {
|
||||||
|
if this.rawStat.Bfree <= this.rawStat.Blocks {
|
||||||
|
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
//go:build !linux
|
//go:build !linux && !windows
|
||||||
|
|
||||||
package mmap
|
package mmap
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
//go:build plus
|
//go:build plus && !windows
|
||||||
|
|
||||||
package mmap
|
package mmap
|
||||||
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
//go:build !freebsd
|
//go:build !freebsd && !windows
|
||||||
// +build !freebsd
|
// +build !freebsd,!windows
|
||||||
|
|
||||||
package utils
|
package utils
|
||||||
|
|
||||||
@@ -15,7 +15,7 @@ func ListenReuseAddr(network string, addr string) (net.Listener, error) {
|
|||||||
config := &net.ListenConfig{
|
config := &net.ListenConfig{
|
||||||
Control: func(network, address string, c syscall.RawConn) error {
|
Control: func(network, address string, c syscall.RawConn) error {
|
||||||
return c.Control(func(fd uintptr) {
|
return c.Control(func(fd uintptr) {
|
||||||
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, SO_REUSEPORT, 1)
|
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logs.Println("[LISTEN]" + err.Error())
|
logs.Println("[LISTEN]" + err.Error())
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,8 @@ package injectionutils
|
|||||||
#include <libinjection.h>
|
#include <libinjection.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
*/
|
*/
|
||||||
|
//go:build cgo
|
||||||
|
|
||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||||
|
|||||||
18
EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go
Normal file
18
EdgeNode/internal/waf/injectionutils/utils_sqli_nocgo.go
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
|
//go:build !cgo
|
||||||
|
|
||||||
|
package injectionutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DetectSQLInjectionCache detect sql injection in string with cache
|
||||||
|
func DetectSQLInjectionCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectSQLInjection detect sql injection in string
|
||||||
|
func DetectSQLInjection(input string, isStrict bool) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -8,6 +8,8 @@ package injectionutils
|
|||||||
#include <libinjection.h>
|
#include <libinjection.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
*/
|
*/
|
||||||
|
//go:build cgo
|
||||||
|
|
||||||
import "C"
|
import "C"
|
||||||
import (
|
import (
|
||||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||||
|
|||||||
17
EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go
Normal file
17
EdgeNode/internal/waf/injectionutils/utils_xss_nocgo.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||||
|
//go:build !cgo
|
||||||
|
|
||||||
|
package injectionutils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
func DetectXSSCache(input string, isStrict bool, cacheLife utils.CacheLife) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectXSS detect XSS in string
|
||||||
|
func DetectXSS(input string, isStrict bool) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
2
deploy/fluent-bit/.gitignore
vendored
Normal file
2
deploy/fluent-bit/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
fluent-bit-windows.conf
|
||||||
|
clickhouse-upstream-windows.conf
|
||||||
BIN
deploy/fluent-bit/logs.db
Normal file
BIN
deploy/fluent-bit/logs.db
Normal file
Binary file not shown.
BIN
deploy/fluent-bit/logs.db-shm
Normal file
BIN
deploy/fluent-bit/logs.db-shm
Normal file
Binary file not shown.
BIN
deploy/fluent-bit/logs.db-wal
Normal file
BIN
deploy/fluent-bit/logs.db-wal
Normal file
Binary file not shown.
Reference in New Issue
Block a user