robin/waf-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

主分支代码

Browse Source
This commit is contained in:
robin
2026-02-07 20:30:31 +08:00
parent 3b042d1dad
commit bc223fd1aa
65 changed files with 1969 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
EdgeCommon/build/.DS_Store vendored Normal file
View File

Binary file not shown.

1
EdgeCommon/pkg/langs/codes/codes.go
View File

@@ -168,6 +168,7 @@ const (
DBNode_LogTruncateTable langs.MessageCode = "db_node@log_truncate_table" // 清空数据库节点 %d 数据表 %s 数据
DBNode_LogUpdateDBNode langs.MessageCode = "db_node@log_update_db_node" // 修改数据库节点 %d
DBNode_TabNodes langs.MessageCode = "db_node@tab_nodes" // 数据库节点
DBNode_TabClickHouse langs.MessageCode = "db_node@tab_clickhouse" // ClickHouse 配置
DDoSProtection_LogUpdateClusterDDoSProtection langs.MessageCode = "ddos_protection@log_update_cluster_ddos_protection" // 修改集群 %d 的DDOS防护设置
DDoSProtection_LogUpdateNodeDDoSProtection langs.MessageCode = "ddos_protection@log_update_node_ddos_protection" // 修改节点 %d 的DDOS防护设置
DNS_LogCreateDomain langs.MessageCode = "dns@log_create_domain" // 添加管理域名到DNS服务商 %d

1
EdgeCommon/pkg/langs/messages/messages_en_us.go
View File

@@ -169,6 +169,7 @@ func init() {
"db_node@log_truncate_table": "",
"db_node@log_update_db_node": "",
"db_node@tab_nodes": "",
"db_node@tab_clickhouse": "ClickHouse",
"ddos_protection@log_update_cluster_ddos_protection": "",
"ddos_protection@log_update_node_ddos_protection": "",
"dns@log_create_domain": "",

1
EdgeCommon/pkg/langs/messages/messages_zh_cn.go
View File

@@ -169,6 +169,7 @@ func init() {
"db_node@log_truncate_table": "清空数据库节点 %d 数据表 %s 数据",
"db_node@log_update_db_node": "修改数据库节点 %d",
"db_node@tab_nodes": "数据库节点",
"db_node@tab_clickhouse": "ClickHouse 配置",
"ddos_protection@log_update_cluster_ddos_protection": "修改集群 %d 的DDOS防护设置",
"ddos_protection@log_update_node_ddos_protection": "修改节点 %d 的DDOS防护设置",
"dns@log_create_domain": "添加管理域名到DNS服务商 %d",

1
EdgeCommon/pkg/langs/protos/zh-cn/db_node.json
View File

@@ -1,5 +1,6 @@
{
"tab_nodes": "数据库节点",
"tab_clickhouse": "ClickHouse 配置",
"log_create_db_node": "创建数据库节点 %d",
"log_delete_db_node": "删除数据库节点 %d",

8
EdgeCommon/pkg/rpc/pb/model_http_access_log_policy.pb.go
View File

@@ -31,6 +31,7 @@ type HTTPAccessLogPolicy struct {
IsPublic bool `protobuf:"varint,7,opt,name=isPublic,proto3" json:"isPublic,omitempty"` // 是否公用
FirewallOnly bool `protobuf:"varint,8,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"` // 是否只记录WAF相关访问日志
DisableDefaultDB bool `protobuf:"varint,9,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"` // 停用默认数据库存储
WriteTargetsJSON []byte `protobuf:"bytes,10,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"` // 写入目标 JSON
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -128,6 +129,13 @@ func (x *HTTPAccessLogPolicy) GetDisableDefaultDB() bool {
return false
}
func (x *HTTPAccessLogPolicy) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
var File_models_model_http_access_log_policy_proto protoreflect.FileDescriptor
var file_models_model_http_access_log_policy_proto_rawDesc = []byte{

24
EdgeCommon/pkg/rpc/pb/service_http_access_log_policy.pb.go
View File

@@ -164,6 +164,7 @@ type CreateHTTPAccessLogPolicyRequest struct {
IsPublic bool `protobuf:"varint,5,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
FirewallOnly bool `protobuf:"varint,6,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
DisableDefaultDB bool `protobuf:"varint,7,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
WriteTargetsJSON []byte `protobuf:"bytes,8,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -247,6 +248,13 @@ func (x *CreateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
return false
}
func (x *CreateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
type CreateHTTPAccessLogPolicyResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
@@ -296,12 +304,14 @@ type UpdateHTTPAccessLogPolicyRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
HttpAccessLogPolicyId int64 `protobuf:"varint,1,opt,name=httpAccessLogPolicyId,proto3" json:"httpAccessLogPolicyId,omitempty"`
Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
Type string `protobuf:"bytes,10,opt,name=type,proto3" json:"type,omitempty"`
IsOn bool `protobuf:"varint,3,opt,name=isOn,proto3" json:"isOn,omitempty"`
OptionsJSON []byte `protobuf:"bytes,4,opt,name=optionsJSON,proto3" json:"optionsJSON,omitempty"`
CondsJSON []byte `protobuf:"bytes,5,opt,name=condsJSON,proto3" json:"condsJSON,omitempty"`
IsPublic bool `protobuf:"varint,6,opt,name=isPublic,proto3" json:"isPublic,omitempty"`
FirewallOnly bool `protobuf:"varint,7,opt,name=firewallOnly,proto3" json:"firewallOnly,omitempty"`
DisableDefaultDB bool `protobuf:"varint,8,opt,name=disableDefaultDB,proto3" json:"disableDefaultDB,omitempty"`
WriteTargetsJSON []byte `protobuf:"bytes,9,opt,name=writeTargetsJSON,proto3" json:"writeTargetsJSON,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -350,6 +360,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetName() string {
return ""
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetType() string {
if x != nil {
return x.Type
}
return ""
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetIsOn() bool {
if x != nil {
return x.IsOn
@@ -392,6 +409,13 @@ func (x *UpdateHTTPAccessLogPolicyRequest) GetDisableDefaultDB() bool {
return false
}
func (x *UpdateHTTPAccessLogPolicyRequest) GetWriteTargetsJSON() []byte {
if x != nil {
return x.WriteTargetsJSON
}
return nil
}
// 查找单个访问日志策略
type FindHTTPAccessLogPolicyRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`

1
EdgeCommon/pkg/rpc/protos/models/model_http_access_log_policy.proto
View File

@@ -13,4 +13,5 @@ message HTTPAccessLogPolicy {
bool isPublic = 7; // 是否公用
bool firewallOnly = 8; // 是否只记录WAF相关访问日志
bool disableDefaultDB = 9; // 停用默认数据库存储
bytes writeTargetsJSON = 10; // 写入目标 JSON: {"file":true,"mysql":true,"clickhouse":false}
}

3
EdgeCommon/pkg/rpc/protos/service_http_access_log_policy.proto
View File

@@ -55,6 +55,7 @@ message CreateHTTPAccessLogPolicyRequest {
bool isPublic = 5;
bool firewallOnly = 6;
bool disableDefaultDB = 7;
bytes writeTargetsJSON = 8;
}
message CreateHTTPAccessLogPolicyResponse {
@@ -65,12 +66,14 @@ message CreateHTTPAccessLogPolicyResponse {
message UpdateHTTPAccessLogPolicyRequest {
int64 httpAccessLogPolicyId = 1;
string name = 2;
string type = 10; // 存储类型：file / es / tcp / syslog / command
bool isOn = 3;
bytes optionsJSON = 4;
bytes condsJSON = 5;
bool isPublic = 6;
bool firewallOnly = 7;
bool disableDefaultDB = 8;
bytes writeTargetsJSON = 9;
}
// 查找单个访问日志策略

112
EdgeCommon/pkg/serverconfigs/access_log_storages.go
View File

@@ -4,45 +4,31 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
)
// AccessLogStorageType 访问日志存储类型
// AccessLogStorageType 访问日志存储类型（含「存储+写入目标」组合）
type AccessLogStorageType = string
const (
AccessLogStorageTypeFile AccessLogStorageType = "file"
AccessLogStorageTypeES AccessLogStorageType = "es"
AccessLogStorageTypeTCP AccessLogStorageType = "tcp"
AccessLogStorageTypeSyslog AccessLogStorageType = "syslog"
AccessLogStorageTypeCommand AccessLogStorageType = "command"
AccessLogStorageTypeFile AccessLogStorageType = "file"
AccessLogStorageTypeFileMySQL AccessLogStorageType = "file_mysql"
AccessLogStorageTypeFileClickhouse AccessLogStorageType = "file_clickhouse"
AccessLogStorageTypeFileMySQLClickhouse AccessLogStorageType = "file_mysql_clickhouse"
AccessLogStorageTypeES AccessLogStorageType = "es"
AccessLogStorageTypeTCP AccessLogStorageType = "tcp"
AccessLogStorageTypeSyslog AccessLogStorageType = "syslog"
AccessLogStorageTypeCommand AccessLogStorageType = "command"
)
// FindAllAccessLogStorageTypes 所有存储引擎列表
// FindAllAccessLogStorageTypes 所有存储引擎列表（含写入目标组合：文件、文件+MySQL、文件+ClickHouse 等）
func FindAllAccessLogStorageTypes() []*shared.Definition {
return []*shared.Definition{
{
Name: "文件",
Code: AccessLogStorageTypeFile,
Description: "将日志存储在磁盘文件中",
},
{
Name: "ElasticSearch",
Code: AccessLogStorageTypeES,
Description: "将日志存储在ElasticSearch中",
},
{
Name: "TCP Socket",
Code: AccessLogStorageTypeTCP,
Description: "将日志通过TCP套接字输出",
},
{
Name: "Syslog",
Code: AccessLogStorageTypeSyslog,
Description: "将日志通过syslog输出，仅支持Linux",
},
{
Name: "命令行输入流",
Code: AccessLogStorageTypeCommand,
Description: "启动一个命令通过读取stdin接收日志信息",
},
{Name: "文件", Code: AccessLogStorageTypeFile, Description: "节点写本地 JSON 文件"},
{Name: "文件+MySQL", Code: AccessLogStorageTypeFileMySQL, Description: "节点写文件 + API 写 MySQL"},
{Name: "文件+ClickHouse", Code: AccessLogStorageTypeFileClickhouse, Description: "节点写文件 + 落 ClickHouse（Fluent Bit 或 API 直写）"},
{Name: "文件+MySQL+ClickHouse", Code: AccessLogStorageTypeFileMySQLClickhouse, Description: "节点写文件 + API 写 MySQL + ClickHouse"},
{Name: "ElasticSearch", Code: AccessLogStorageTypeES, Description: "将日志存储在ElasticSearch中"},
{Name: "TCP Socket", Code: AccessLogStorageTypeTCP, Description: "将日志通过TCP套接字输出"},
{Name: "Syslog", Code: AccessLogStorageTypeSyslog, Description: "将日志通过syslog输出，仅支持Linux"},
{Name: "命令行输入流", Code: AccessLogStorageTypeCommand, Description: "启动一个命令通过读取stdin接收日志信息"},
}
}
@@ -55,3 +41,65 @@ func FindAccessLogStorageTypeName(storageType string) string {
}
return ""
}
// IsFileBasedStorageType 是否为基于文件的存储（需要显示文件路径等配置）
func IsFileBasedStorageType(code string) bool {
switch code {
case AccessLogStorageTypeFile, AccessLogStorageTypeFileMySQL, AccessLogStorageTypeFileClickhouse, AccessLogStorageTypeFileMySQLClickhouse:
return true
default:
return false
}
}
// ParseStorageTypeAndWriteTargets 从下拉框选中的类型解析出「实际存储类型」与「写入目标」
// 用于创建/更新策略：options 按 baseType 填（如 file），writeTargets 按组合填。
func ParseStorageTypeAndWriteTargets(selectedType string) (baseType string, writeTargets *AccessLogWriteTargets) {
writeTargets = &AccessLogWriteTargets{}
switch selectedType {
case AccessLogStorageTypeFile:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
case AccessLogStorageTypeFileMySQL:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.MySQL = true
case AccessLogStorageTypeFileClickhouse:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.ClickHouse = true
case AccessLogStorageTypeFileMySQLClickhouse:
baseType = AccessLogStorageTypeFile
writeTargets.File = true
writeTargets.MySQL = true
writeTargets.ClickHouse = true
case AccessLogStorageTypeES, AccessLogStorageTypeTCP, AccessLogStorageTypeSyslog, AccessLogStorageTypeCommand:
baseType = selectedType
writeTargets.MySQL = true
default:
baseType = selectedType
writeTargets.File = true
writeTargets.MySQL = true
}
return baseType, writeTargets
}
// ComposeStorageTypeDisplay 根据策略的 Type + WriteTargets 得到下拉框显示用的类型 code（用于编辑页回显）
func ComposeStorageTypeDisplay(policyType string, writeTargets *AccessLogWriteTargets) string {
if policyType != AccessLogStorageTypeFile {
return policyType
}
if writeTargets == nil {
return AccessLogStorageTypeFile
}
if writeTargets.File && writeTargets.MySQL && writeTargets.ClickHouse {
return AccessLogStorageTypeFileMySQLClickhouse
}
if writeTargets.File && writeTargets.MySQL {
return AccessLogStorageTypeFileMySQL
}
if writeTargets.File && writeTargets.ClickHouse {
return AccessLogStorageTypeFileClickhouse
}
return AccessLogStorageTypeFile
}

49
EdgeCommon/pkg/serverconfigs/access_log_write_targets.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright 2025. All rights reserved.
package serverconfigs
import "encoding/json"
// AccessLogWriteTargets 访问日志写入目标（双写/单写：文件、MySQL、ClickHouse）
type AccessLogWriteTargets struct {
File bool `yaml:"file" json:"file"` // 写本地 JSON 文件（供 Fluent Bit → ClickHouse 或自用）
MySQL bool `yaml:"mysql" json:"mysql"` // 写 MySQL 默认库按日分表
ClickHouse bool `yaml:"clickhouse" json:"clickhouse"` // 需要落 ClickHouse（文件+Fluent Bit 或 API 直写）
}
// NeedReportToAPI 是否需要上报到 API（写 MySQL 或 API 直写 ClickHouse 时需要）
func (t *AccessLogWriteTargets) NeedReportToAPI() bool {
if t == nil {
return true // 兼容：未配置时保持原行为，上报
}
return t.MySQL || t.ClickHouse
}
// NeedWriteFile 节点是否需要写本地文件
func (t *AccessLogWriteTargets) NeedWriteFile() bool {
if t == nil {
return true // 兼容：未配置时保持原行为，写文件
}
return t.File
}
// ParseWriteTargetsFromPolicy 从策略的 writeTargets JSON 与旧字段解析；无 writeTargets 时按 type + disableDefaultDB 推断
func ParseWriteTargetsFromPolicy(writeTargetsJSON []byte, policyType string, disableDefaultDB bool) *AccessLogWriteTargets {
if len(writeTargetsJSON) > 0 {
var t AccessLogWriteTargets
if err := json.Unmarshal(writeTargetsJSON, &t); err == nil {
return &t
}
}
// 兼容旧策略：type=file 视为写文件，!disableDefaultDB 视为写 MySQL
t := &AccessLogWriteTargets{
File: policyType == AccessLogStorageTypeFile,
MySQL: !disableDefaultDB,
ClickHouse: false,
}
if !t.File && !t.MySQL && !t.ClickHouse {
t.File = true
t.MySQL = true
}
return t
}

85
EdgeCommon/pkg/serverconfigs/firewallconfigs/http_firewall_checkpoints.go
View File

@@ -1,10 +1,5 @@
package firewallconfigs
import (
"github.com/iwind/TeaGo/maps"
"regexp"
)
// AllCheckpoints all check points list
var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
{
@@ -307,86 +302,6 @@ var AllCheckpoints = []*HTTPFirewallCheckpointDefinition{
IsComposed: true,
Priority: 20,
},
{
Name: "CC统计（旧）",
Prefix: "cc",
Description: "统计某段时间段内的请求信息（不推荐再使用，请使用新的CC2统计代替）。",
HasParams: true,
Params: []*KeyValue{
NewKeyValue("请求数", "requests"),
},
Options: []OptionInterface{
&FieldOption{
Type: "field",
Name: "统计周期",
Code: "period",
Value: "60",
IsRequired: false,
Size: 8,
Comment: "",
Placeholder: "",
RightLabel: "秒",
MaxLength: 8,
Validate: func(value string) (ok bool, message string) {
if regexp.MustCompile(`^\d+$`).MatchString(value) {
ok = true
return
}
message = "周期需要是一个整数数字"
return
},
},
&OptionsOption{
Type: "options",
Name: "用户识别读取来源",
Code: "userType",
Value: "",
IsRequired: false,
Size: 10,
Comment: "",
RightLabel: "",
Validate: nil,
Options: []maps.Map{
{
"name": "IP",
"value": "ip",
},
{
"name": "Cookie",
"value": "cookie",
},
{
"name": "URL参数",
"value": "get",
},
{
"name": "POST参数",
"value": "post",
},
{
"name": "HTTP Header",
"value": "header",
},
},
},
&FieldOption{
Type: "field",
Name: "用户识别字段",
Code: "userField",
Comment: "识别用户的唯一性字段，在用户读取来源不是IP时使用",
},
&FieldOption{
Type: "field",
Name: "字段读取位置",
Code: "userIndex",
Size: 5,
MaxLength: 5,
Comment: "读取用户识别字段的位置，从0开始，比如user12345的数字ID 12345的位置就是5，在用户读取来源不是IP时使用",
},
},
IsRequest: true,
Priority: 10,
},
{
Name: "响应状态码",
Prefix: "status",

13
EdgeCommon/pkg/serverconfigs/global_server_config.go
View File

@@ -71,12 +71,13 @@ type GlobalServerConfig struct {
} `yaml:"tcpAll" json:"tcpAll"`
HTTPAccessLog struct {
IsOn bool `yaml:"isOn" json:"isOn"` // 是否启用此功能
EnableRequestHeaders bool `yaml:"enableRequestHeaders" json:"enableRequestHeaders"` // 记录请求Header
CommonRequestHeadersOnly bool `yaml:"commonRequestHeadersOnly" json:"commonRequestHeadersOnly"` // 只保留通用Header
EnableResponseHeaders bool `yaml:"enableResponseHeaders" json:"enableResponseHeaders"` // 记录响应Header
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
IsOn bool `yaml:"isOn" json:"isOn"` // 是否启用此功能
EnableRequestHeaders bool `yaml:"enableRequestHeaders" json:"enableRequestHeaders"` // 记录请求Header
CommonRequestHeadersOnly bool `yaml:"commonRequestHeadersOnly" json:"commonRequestHeadersOnly"` // 只保留通用Header
EnableResponseHeaders bool `yaml:"enableResponseHeaders" json:"enableResponseHeaders"` // 记录响应Header
EnableCookies bool `yaml:"enableCookies" json:"enableCookies"` // 记录Cookie
EnableServerNotFound bool `yaml:"enableServerNotFound" json:"enableServerNotFound"` // 记录服务找不到的日志
WriteTargets *AccessLogWriteTargets `yaml:"writeTargets" json:"writeTargets"` // 写入目标：文件/MySQL/ClickHouse（双写/单写）
} `yaml:"httpAccessLog" json:"httpAccessLog"` // 访问日志配置
Stat struct {

12
EdgeCommon/pkg/systemconfigs/clickhouse_setting.go Normal file
View File

@@ -0,0 +1,12 @@
//go:build plus
package systemconfigs
// ClickHouseSetting 后台页面配置的 ClickHouse 连接（访问日志 logs_ingest 查询）
type ClickHouseSetting struct {
Host string `json:"host" yaml:"host"`
Port int `json:"port" yaml:"port"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
Database string `json:"database" yaml:"database"`
}

2
EdgeCommon/pkg/systemconfigs/settings_plus.go
View File

@@ -15,4 +15,6 @@ const (
SettingCodeBillDay SettingCode = "billDay" // 账单日 YYYYMMDD
SettingCodeBillMonth SettingCode = "billMonth" // 账单月 YYYYMM
SettingCodeClickHouseConfig SettingCode = "clickhouseConfig" // ClickHouse 连接配置（访问日志 logs_ingest 查询）
)