Initial commit (code only without large binaries)
This commit is contained in:
robin
11
deploy/fluent-bit/clickhouse-upstream.conf
Normal file
11
deploy/fluent-bit/clickhouse-upstream.conf
Normal file
@@ -0,0 +1,11 @@
|
||||
# ClickHouse 上游配置（单机或集群只改此文件）
|
||||
# 单机：保留一个 NODE；集群：按需增加 NODE
|
||||
# 有密码时：不在此文件配置，通过环境变量 CH_USER、CH_PASSWORD 在 fluent-bit.conf 的 OUTPUT 中生效。
|
||||
|
||||
[UPSTREAM]
|
||||
Name ch_backends
|
||||
|
||||
[NODE]
|
||||
Name node-01
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
39
deploy/fluent-bit/fluent-bit-dns-https.conf
Normal file
39
deploy/fluent-bit/fluent-bit-dns-https.conf
Normal file
@@ -0,0 +1,39 @@
|
||||
# DNS 节点专用 HTTPS：使用 HTTP 输出写入 ClickHouse（无需 out_clickhouse 插件）
|
||||
# 启动前设置：CH_USER、CH_PASSWORD；按需修改 Host、Port（默认 127.0.0.1:8443）
|
||||
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 128MB
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Parser json
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
tls On
|
||||
tls.verify On
|
||||
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||
# tls.vhost clickhouse.example.com
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
40
deploy/fluent-bit/fluent-bit-dns.conf
Normal file
40
deploy/fluent-bit/fluent-bit-dns.conf
Normal file
@@ -0,0 +1,40 @@
|
||||
# DNS 节点专用：使用 HTTPS 输出写入 ClickHouse（无需 out_clickhouse 插件）
|
||||
# 启动前设置：CH_USER、CH_PASSWORD；若 ClickHouse 不在本机，请修改 Host、Port
|
||||
# Read_from_Head=true：首次启动会发送已有日志；若只采新日志建议改为 false
|
||||
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 128MB
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Parser json
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
tls On
|
||||
tls.verify On
|
||||
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||
# tls.vhost clickhouse.example.com
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
74
deploy/fluent-bit/fluent-bit-https.conf
Normal file
74
deploy/fluent-bit/fluent-bit-https.conf
Normal file
@@ -0,0 +1,74 @@
|
||||
# Fluent Bit HTTPS 配置（边缘节点日志采集 -> ClickHouse HTTPS）
|
||||
# HTTP: /var/log/edge/edge-node/*.log
|
||||
# DNS: /var/log/edge/edge-dns/*.log
|
||||
#
|
||||
# 启动前请设置环境变量：
|
||||
# CH_USER=default
|
||||
# CH_PASSWORD=your_password
|
||||
# 如需改地址/端口，请修改 OUTPUT 中 Host/Port（默认 127.0.0.1:8443）
|
||||
# 如证书为公网CA可省略 tls.ca_file；自签名证书请配置 tls.ca_file
|
||||
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 128MB
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-node/*.log
|
||||
Tag app.http.logs
|
||||
Parser json
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/http-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Parser json
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.http.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
tls On
|
||||
tls.verify On
|
||||
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||
# tls.vhost clickhouse.example.com
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.dns.logs
|
||||
Host 127.0.0.1
|
||||
Port 8443
|
||||
URI /?query=INSERT%20INTO%20default.dns_logs_ingest%20FORMAT%20JSONEachRow
|
||||
Format json_lines
|
||||
http_user ${CH_USER}
|
||||
http_passwd ${CH_PASSWORD}
|
||||
tls On
|
||||
tls.verify On
|
||||
# tls.ca_file /etc/ssl/certs/ca-certificates.crt
|
||||
# tls.vhost clickhouse.example.com
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
56
deploy/fluent-bit/fluent-bit.conf
Normal file
56
deploy/fluent-bit/fluent-bit.conf
Normal file
@@ -0,0 +1,56 @@
|
||||
# Fluent Bit 主配置（边缘节点日志采集 → ClickHouse）
|
||||
# HTTP: /var/log/edge/edge-node/*.log
|
||||
# DNS: /var/log/edge/edge-dns/*.log
|
||||
|
||||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
storage.path /var/lib/fluent-bit/storage
|
||||
storage.sync normal
|
||||
storage.checksum off
|
||||
storage.backlog.mem_limit 128MB
|
||||
|
||||
@INCLUDE clickhouse-upstream.conf
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-node/*.log
|
||||
Tag app.http.logs
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/http-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/edge/edge-dns/*.log
|
||||
Tag app.dns.logs
|
||||
Refresh_Interval 5
|
||||
Read_from_Head false
|
||||
DB /var/lib/fluent-bit/dns-logs.db
|
||||
Mem_Buf_Limit 128MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[OUTPUT]
|
||||
Name clickhouse
|
||||
Match app.http.logs
|
||||
Upstream ch_backends
|
||||
Table logs_ingest
|
||||
Http_User ${CH_USER}
|
||||
Http_Passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
|
||||
[OUTPUT]
|
||||
Name clickhouse
|
||||
Match app.dns.logs
|
||||
Upstream ch_backends
|
||||
Table dns_logs_ingest
|
||||
Http_User ${CH_USER}
|
||||
Http_Passwd ${CH_PASSWORD}
|
||||
json_date_key timestamp
|
||||
json_date_format epoch
|
||||
Retry_Limit 10
|
||||
9
deploy/fluent-bit/parsers.conf
Normal file
9
deploy/fluent-bit/parsers.conf
Normal file
@@ -0,0 +1,9 @@
|
||||
# Fluent Bit 解析器（与主配置 Parsers_File 对应）
|
||||
# 边缘节点日志为 JSON Lines，使用 json 解析器即可
|
||||
|
||||
[PARSER]
|
||||
Name json
|
||||
Format json
|
||||
Time_Key timestamp
|
||||
Time_Format %s
|
||||
Time_Keep On
|
||||
Reference in New Issue
Block a user