错误日志查询问题修复

This commit is contained in:
robin
2026-02-08 22:44:12 +08:00
parent b7388d83b0
commit 4812ad5aaf
14 changed files with 57299 additions and 57088 deletions

View File

@@ -126,6 +126,9 @@ func (s *LogsIngestStore) List(ctx context.Context, f ListFilter) (rows []*LogsI
if f.HasFirewallPolicy {
conditions = append(conditions, "firewall_policy_id > 0")
}
if f.HasError {
conditions = append(conditions, "status >= 400")
}
if f.FirewallPolicyId > 0 {
conditions = append(conditions, "firewall_policy_id = "+strconv.FormatInt(f.FirewallPolicyId, 10))
}

View File

@@ -67,16 +67,35 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() && req.Day != "" {
canReadFromClickHouse := this.shouldReadAccessLogsFromClickHouse() && store.Client().IsConfigured() && req.Day != ""
canReadFromMySQL := this.shouldReadAccessLogsFromMySQL()
if canReadFromClickHouse {
resp, listErr := this.listHTTPAccessLogsFromClickHouse(ctx, tx, store, req, userId)
if listErr == nil && resp != nil {
return resp, nil
}
if !canReadFromMySQL {
if listErr != nil {
return nil, listErr
}
if resp != nil {
return resp, nil
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: []*pb.HTTPAccessLog{},
AccessLogs: []*pb.HTTPAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
}
if !canReadFromMySQL {
return &pb.ListHTTPAccessLogsResponse{
HttpAccessLogs: []*pb.HTTPAccessLog{},
AccessLogs: []*pb.HTTPAccessLog{},
HasMore: false,
RequestId: "",
}, nil
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.Partition, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil {
return nil, err
@@ -241,12 +260,15 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
// 优先从 ClickHouse 查询
store := clickhouse.NewLogsIngestStore()
if store.Client().IsConfigured() {
canReadFromClickHouse := this.shouldReadAccessLogsFromClickHouse() && store.Client().IsConfigured()
canReadFromMySQL := this.shouldReadAccessLogsFromMySQL()
if canReadFromClickHouse {
row, err := store.FindByTraceId(ctx, req.RequestId)
if err != nil {
if !canReadFromMySQL {
return nil, err
}
if row != nil {
} else if row != nil {
// 检查权限
if userId > 0 {
var tx = this.NullTx()
@@ -260,6 +282,10 @@ func (this *HTTPAccessLogService) FindHTTPAccessLog(ctx context.Context, req *pb
}
}
if !canReadFromMySQL {
return &pb.FindHTTPAccessLogResponse{HttpAccessLog: nil}, nil
}
// 如果 ClickHouse 未配置或未找到,则回退到 MySQL
var tx = this.NullTx()

View File

@@ -9,6 +9,14 @@ func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return true
}
func (this *HTTPAccessLogService) shouldReadAccessLogsFromClickHouse() bool {
return true
}
func (this *HTTPAccessLogService) shouldReadAccessLogsFromMySQL() bool {
return true
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {
return nil
}

View File

@@ -11,7 +11,15 @@ import (
)
func (this *HTTPAccessLogService) canWriteAccessLogsToDB() bool {
return false
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *HTTPAccessLogService) shouldReadAccessLogsFromClickHouse() bool {
return accesslogs.SharedStorageManager.WriteClickHouse()
}
func (this *HTTPAccessLogService) shouldReadAccessLogsFromMySQL() bool {
return accesslogs.SharedStorageManager.WriteMySQL()
}
func (this *HTTPAccessLogService) writeAccessLogsToPolicy(pbAccessLogs []*pb.HTTPAccessLog) error {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

BIN
EdgeNode/go.tar.gz Normal file

Binary file not shown.

View File

@@ -15,7 +15,7 @@ func ListenReuseAddr(network string, addr string) (net.Listener, error) {
config := &net.ListenConfig{
Control: func(network, address string, c syscall.RawConn) error {
return c.Control(func(fd uintptr) {
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)
err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, 15, 1) // 15 = SO_REUSEPORT on Linux
if err != nil {
logs.Println("[LISTEN]" + err.Error())
}

View File

@@ -1,3 +1,6 @@
//go:build cgo
// +build cgo
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
package injectionutils
@@ -8,8 +11,6 @@ package injectionutils
#include <libinjection.h>
#include <stdlib.h>
*/
//go:build cgo
import "C"
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"

View File

@@ -1,3 +1,6 @@
//go:build cgo
// +build cgo
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
package injectionutils
@@ -8,8 +11,6 @@ package injectionutils
#include <libinjection.h>
#include <stdlib.h>
*/
//go:build cgo
import "C"
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,26 +1,27 @@
<!DOCTYPE html>
<html lang="zh">
<head>
<title>{$ htmlEncode .teaTitle}</title>
<meta charset="UTF-8"/>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=0">
{$if eq .teaFaviconFileId 0}
<link rel="shortcut icon" href="/images/favicon.png"/>
<link rel="shortcut icon" href="/images/favicon.png" />
{$else}
<link rel="shortcut icon" href="/ui/image/{$.teaFaviconFileId}"/>
<link rel="shortcut icon" href="/ui/image/{$.teaFaviconFileId}" />
{$end}
<link rel="stylesheet" type="text/css" href="/_/@default/@layout.css" media="all"/>
<link rel="stylesheet" type="text/css" href="/_/@default/@layout.css" media="all" />
{$TEA.SEMANTIC}
<link rel="stylesheet" type="text/css" href="/_/@default/@layout_override.css" media="all"/>
<link rel="stylesheet" type="text/css" href="/_/@default/@layout_override.css" media="all" />
{$TEA.VUE}
{$echo "header"}
<!-- 品牌配置 -->
<script type="text/javascript">
window.BRAND_OFFICIAL_SITE = {$ jsonEncode .brandConfig.officialSite};
window.BRAND_DOCS_SITE = {$ jsonEncode .brandConfig.docsSite};
window.BRAND_DOCS_PREFIX = {$ jsonEncode .brandConfig.docsPathPrefix};
window.BRAND_PRODUCT_NAME = {$ jsonEncode .brandConfig.productName};
window.BRAND_OFFICIAL_SITE = { $ jsonEncode .brandConfig.officialSite };
window.BRAND_DOCS_SITE = { $ jsonEncode .brandConfig.docsSite };
window.BRAND_DOCS_PREFIX = { $ jsonEncode .brandConfig.docsPathPrefix };
window.BRAND_PRODUCT_NAME = { $ jsonEncode .brandConfig.productName };
</script>
<script type="text/javascript" src="/js/config/brand.js"></script>
<script type="text/javascript" src="/_/@default/@layout.js"></script>
@@ -31,14 +32,16 @@
{$if not (eq .teaThemeBackgroundColor "")}
<style>
.top-nav, .main-menu, .main-menu .menu {
.top-nav,
.main-menu,
.main-menu .menu {
background: #{$.teaThemeBackgroundColor} !important;
}
</style>
{$end}
<!-- 应用保存的主题颜色 -->
<script type="text/javascript">
(function() {
(function () {
try {
var savedColor = localStorage.getItem("themeColor");
if (savedColor) {
@@ -51,26 +54,31 @@
}
styleEl.textContent = ".top-nav, .main-menu, .main-menu .menu { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu.vertical.blue.inverted.tiny.borderless { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu.vertical.blue.inverted.tiny.borderless .item { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu.vertical.blue.inverted.tiny.borderless .sub-items { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu.vertical.blue.inverted.tiny.borderless .sub-items .item { background: " + savedColor + " !important; } .main-menu .ui.menu .sub-items { background: " + savedColor + " !important; } .main-menu .ui.menu .sub-items .item { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu .sub-items { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu .sub-items .item { background: " + savedColor + " !important; } .main-menu .sub-items { background: " + savedColor + " !important; } .main-menu .sub-items .item { background: " + savedColor + " !important; } .ui.menu.blue.inverted { background: " + savedColor + " !important; } .ui.menu.blue.inverted .item { background: " + savedColor + " !important; } .ui.menu.vertical.blue.inverted { background: " + savedColor + " !important; } .ui.menu.vertical.blue.inverted .item { background: " + savedColor + " !important; } .ui.labeled.menu.vertical.blue.inverted { background: " + savedColor + " !important; } .ui.labeled.menu.vertical.blue.inverted .item { background: " + savedColor + " !important; } .main-menu .ui.labeled.menu.vertical.blue.inverted .item.active { background: rgba(230, 230, 230, 0.45) !important; } .main-menu .ui.menu .sub-items .item.active { background: rgba(230, 230, 230, 0.55) !important; }";
}
} catch(e) {
} catch (e) {
console.log("Failed to apply saved theme color:", e);
}
})();
</script>
</head>
<body>
<div>
<div>
<!-- 顶部导航 -->
<div class="ui menu top-nav blue inverted small borderless" v-cloak="">
<a href="/dashboard" class="item">
<i class="ui icon leaf" v-if="teaLogoFileId == 0"></i><img v-if="teaLogoFileId > 0" :src="'/ui/image/' + teaLogoFileId" style="width: auto;height: 1.6em"/> &nbsp; {{teaTitle}}&nbsp;<sup v-if="teaShowVersion">v{{teaVersion}}</sup> &nbsp;
<i class="ui icon leaf" v-if="teaLogoFileId == 0"></i><img v-if="teaLogoFileId > 0"
:src="'/ui/image/' + teaLogoFileId" style="width: auto;height: 1.6em" /> &nbsp;
{{teaTitle}}&nbsp;<sup v-if="teaShowVersion">v{{teaVersion}}</sup> &nbsp;
</a>
<div class="right menu">
<a href="/messages" class="item" :class="{active:teaMenu == 'message'}"><span :class="{'blink':globalMessageBadge > 0}"><i class="icon bell"></i>消息({{globalMessageBadge}}) </span></a>
<a href="/messages" class="item" :class="{active:teaMenu == 'message'}"><span
:class="{'blink':globalMessageBadge > 0}"><i class="icon bell"></i>消息({{globalMessageBadge}})
</span></a>
<a href="/settings/profile" class="item">
<i class="icon user" v-if="teaUserAvatar.length == 0"></i>
<img class="avatar" alt="" :src="teaUserAvatar" v-if="teaUserAvatar.length > 0"/>
<img class="avatar" alt="" :src="teaUserAvatar" v-if="teaUserAvatar.length > 0" />
{{teaUsername}}
</a>
<a href="/docs" class="item" :class="{active: teaMenu == 'docs'}"><i class="icon file"></i>文档</a>
@@ -94,7 +102,9 @@
<!-- 模块 -->
<div v-for="module in teaModules">
<a class="item" :href="Tea.url(module.code)" :class="{active:teaMenu == module.code && teaSubMenu.length == 0, separator:module.code.length == 0}" :style="(teaMenu == module.code && teaSubMenu.length == 0) ? 'background: rgba(230, 230, 230, 0.45) !important;' : ''">
<a class="item" :href="Tea.url(module.code)"
:class="{active:teaMenu == module.code && teaSubMenu.length == 0, separator:module.code.length == 0}"
:style="(teaMenu == module.code && teaSubMenu.length == 0) ? 'background: rgba(230, 230, 230, 0.45) !important;' : ''">
<span v-if="module.code.length > 0">
<i class="window restore outline icon" v-if="module.icon == null"></i>
<i class="ui icon" v-if="module.icon != null" :class="module.icon"></i>
@@ -102,18 +112,28 @@
</span>
</a>
<div v-if="teaMenu == module.code" class="sub-items">
<a class="item" v-for="subItem in module.subItems" :href="subItem.url" :class="{active:subItem.code == teaSubMenu}" :style="(subItem.code == teaSubMenu) ? 'background: rgba(230, 230, 230, 0.55) !important;' : ''" v-if="subItem.isOn == null || subItem.isOn === true">{{subItem.name}}<i class="icon angle right" v-if="subItem.name != '-' && subItem.code == teaSubMenu"></i></a>
<a class="item" v-for="subItem in module.subItems" :href="subItem.url"
:class="{active:subItem.code == teaSubMenu}"
:style="(subItem.code == teaSubMenu) ? 'background: rgba(230, 230, 230, 0.55) !important;' : ''"
v-if="subItem.isOn == null || subItem.isOn === true">{{subItem.name}}<i
class="icon angle right"
v-if="subItem.name != '-' && subItem.code == teaSubMenu"></i></a>
</div>
</div>
</div>
</div>
<!-- 右侧主操作栏 -->
<div class="main" :class="{'without-menu':teaSubMenus.menus == null || teaSubMenus.menus.length == 0 || (teaSubMenus.menus.length == 1 && teaSubMenus.menus[0].alwaysActive), 'without-secondary-menu':teaSubMenus.alwaysMenu == null || teaSubMenus.alwaysMenu.items.length <= 1, 'without-footer':!teaShowPageFooter}" v-cloak="">
<div class="main"
:class="{'without-menu':teaSubMenus.menus == null || teaSubMenus.menus.length == 0 || (teaSubMenus.menus.length == 1 && teaSubMenus.menus[0].alwaysActive), 'without-secondary-menu':teaSubMenus.alwaysMenu == null || teaSubMenus.alwaysMenu.items.length <= 1, 'without-footer':!teaShowPageFooter}"
v-cloak="">
<!-- 操作菜单 -->
<div class="ui top menu tabular tab-menu small" v-if="teaTabbar.length > 0">
<a class="item" v-for="item in teaTabbar" :class="{'active':item.active,right:item.right}" :href="item.url">
<var>{{item.name}}<span v-if="item.subName.length > 0">({{item.subName}})</span><i class="icon small" :class="item.icon" v-if="item.icon != null && item.icon.length > 0"></i> </var>
<a class="item" v-for="item in teaTabbar" :class="{'active':item.active,right:item.right}"
:href="item.url">
<var>{{item.name}}<span v-if="item.subName.length > 0">({{item.subName}})</span><i
class="icon small" :class="item.icon" v-if="item.icon != null && item.icon.length > 0"></i>
</var>
</a>
</div>
@@ -122,13 +142,16 @@
</div>
<!-- 底部 -->
<div id="footer" class="ui menu inverted light-blue borderless small" v-if="teaShowPageFooter && teaPageFooterHTML.length == 0">
<div id="footer" class="ui menu inverted light-blue borderless small"
v-if="teaShowPageFooter && teaPageFooterHTML.length == 0" v-cloak>
<a class="item" title="点击进入检查版本更新页面">{{teaName}} v{{teaVersion}}</a>
</div>
<div id="footer" class="ui menu inverted light-blue borderless small" v-if="teaShowPageFooter && teaPageFooterHTML.length > 0" v-html="teaPageFooterHTML"> </div>
</div>
<div id="footer" class="ui menu inverted light-blue borderless small"
v-if="teaShowPageFooter && teaPageFooterHTML.length > 0" v-html="teaPageFooterHTML" v-cloak> </div>
</div>
{$echo "footer"}
{$echo "footer"}
</body>
</html>

View File

@@ -0,0 +1,141 @@
# 访问日志策略配置手册(默认安装 / 仅MySQL / 仅ClickHouse / 双写)
## 1. 适用范围
- 代码基线:`e:\AI_PRODUCT\waf-platform`
- 页面入口:`系统设置 -> 访问日志 -> 日志策略`
- 查询入口:`网站 -> 站点 -> 日志``/servers/server/log`
---
## 2. 默认安装后的行为(什么都不配)
```mermaid
flowchart TD
A[EdgeNode 产生日志] --> B[写本地文件 /var/log/edge/edge-node/*.log]
A --> C[上报 EdgeAPI]
C --> D[写 MySQL 访问日志表]
E[日志查询页] --> D
```
- 默认即可写日志,不会因为没配 ClickHouse 就停写。
- 查询默认走 MySQL。
- 是否有“独立日志数据库节点”会影响写到哪个 MySQL
- 有日志库节点:优先写日志库节点池。
- 没有日志库节点:回退写默认数据库。
---
## 3. 必须设置项(上线最小集)
### 3.1 基础必需(任何模式都建议)
1. `EdgeAPI` 数据库连接可用(`db.yaml` / `.db.yaml`)。
2. `EdgeNode``EdgeAPI` 通信正常(节点在线,可上报日志)。
3. 建议创建并启用一个**公用**访问日志策略(避免多环境行为不一致)。
### 3.2 仅 ClickHouse / MySQL+ClickHouse 额外必需
1. `EdgeAPI` 配置 ClickHouse 读取:
- `EdgeAPI/configs/api.yaml`
```yaml
clickhouse:
host: 127.0.0.1
port: 8123
user: default
password: "xxxxxx"
database: default
```
2. Fluent Bit 已部署并运行,采集:
- `/var/log/edge/edge-node/*.log`
3. ClickHouse 已建表:`logs_ingest`(见 `deploy/fluent-bit/README.md`)。
---
## 4. 三种目标模式怎么配
## 4.1 只写入 MySQL
在“日志策略”中:
1. 新建或修改策略,`存储类型` 选 **文件+MySQL**。
2. 设为 **公用**,并确保 **启用**。
3. `日志文件路径` 填一个 API 可写路径(必填校验项):
- 示例:`/var/log/edge/edge-api/http-access-${date}.log`
结果:
- 写入MySQL主路径+ Node 本地日志文件
- 查询MySQL
- 不依赖 ClickHouse
---
## 4.2 只写入 ClickHouse
在“日志策略”中:
1. `存储类型` 选 **文件+ClickHouse**。
2. 设为 **公用**,并确保 **启用**。
3. `日志文件路径` 仍需填写(策略校验要求):
- 示例:`/var/log/edge/edge-api/http-access-${date}.log`
4. 确保 Fluent Bit 正在采集 Node 目录并写入 ClickHouse。
5. 确保 `EdgeAPI` 的 ClickHouse 连接已配置。
结果:
- 写入Node 本地文件 -> Fluent Bit -> ClickHouse
- API 不写 MySQL
- 查询优先 ClickHouse无 CH 时可能查不到数据)
---
## 4.3 同时写入 MySQL + ClickHouse
在“日志策略”中:
1. `存储类型` 选 **文件+MySQL+ClickHouse**。
2. 设为 **公用**,并确保 **启用**。
3. `日志文件路径` 填写有效路径(同上)。
4. ClickHouse + Fluent Bit 同 4.2 要求。
结果:
- 写入MySQL + ClickHouse并行
- 查询:优先 ClickHouse失败可回退 MySQL
---
## 5. 配置生效链路图
```mermaid
flowchart LR
P[公用日志策略 type/writeTargets] --> C[EdgeAPI 解析 writeTargets]
C --> N[下发到 EdgeNode GlobalServerConfig.HTTPAccessLog.WriteTargets]
N --> W1[NeedWriteFile]
N --> W2[NeedReportToAPI]
W1 --> F[Node本地日志文件]
F --> FB[Fluent Bit]
FB --> CH[(ClickHouse.logs_ingest)]
W2 --> API[CreateHTTPAccessLogs]
API --> MYSQL[(MySQL访问日志表)]
```
---
## 6. 验证清单(建议上线前逐项过)
1. 打开 `/servers/server/log`,持续压测 1~2 分钟。
2. 检查最新日志是否持续上顶(不是停在旧时间段)。
3. 错误日志筛选是否只显示 `status>=400`。
4. 仅 CH 模式下,停掉 Fluent Bit 后确认告警和查询表现符合预期。
5. MySQL+CH 模式下,临时断 CH确认页面可回退 MySQL。
---
## 7. 常见问题
### Q1策略里的“日志文件路径”是干嘛的
- 这是策略 `file` 配置的必填项API 侧校验)。
- 即使你用 ClickHouse 链路,当前实现仍要求该字段有值。
- 真正给 Fluent Bit 采集的是 **Node 目录**`/var/log/edge/edge-node/*.log`。
### Q2不勾“停用默认数据库存储”会不会同时写默认库和独立日志库
- 正常不会双写同一条。
- 有独立日志库节点时优先写节点池;节点池不可用时才回退默认库。
### Q3修改策略后要不要重启
- 通常 1 分钟内自动刷新生效。
- 若要立即生效:重启 `edge-api`,并在需要时重启 `edge-node`、`fluent-bit`。