-- =============================================================================
-- ClickHouse logs_ingest 表优化脚本
-- 
-- 说明：
--   - 所有 ALTER 操作均为在线操作，无需停服
--   - 建议按阶段顺序执行，每阶段执行后观察 system.parts 确认生效
--   - 压缩编解码器变更仅影响新写入的 part，存量数据需等 merge 或手动 OPTIMIZE
-- 
-- 执行方式：
--   clickhouse-client --host 127.0.0.1 --port 9000 --user default --password 'xxx' < optimize_schema.sql
-- =============================================================================

-- =============================================
-- 阶段 1：大字段压缩优化（效果最显著）
-- =============================================

-- 大文本字段改用 ZSTD(3)，对 JSON / HTTP 文本压缩率远优于默认 LZ4
-- 预期效果：磁盘占用减少 40%-60%
ALTER TABLE logs_ingest MODIFY COLUMN request_headers String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN request_body String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN response_headers String CODEC(ZSTD(3));
ALTER TABLE logs_ingest MODIFY COLUMN response_body String CODEC(ZSTD(3));

-- 中等长度文本字段用 ZSTD(1)，平衡压缩率与 CPU 开销
ALTER TABLE logs_ingest MODIFY COLUMN ua String CODEC(ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN path String CODEC(ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN referer String CODEC(ZSTD(1));

-- 低基数字段改用 LowCardinality（内存+磁盘双降）
-- method 的基数极低（GET/POST/PUT/DELETE 等），host 基数取决于站点数量
ALTER TABLE logs_ingest MODIFY COLUMN method LowCardinality(String);
ALTER TABLE logs_ingest MODIFY COLUMN log_type LowCardinality(String);
ALTER TABLE logs_ingest MODIFY COLUMN host LowCardinality(String);

-- 数值字段使用 Delta + ZSTD 编码（利用相邻行的时间/大小相关性）
ALTER TABLE logs_ingest MODIFY COLUMN bytes_in UInt64 CODEC(Delta, ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN bytes_out UInt64 CODEC(Delta, ZSTD(1));
ALTER TABLE logs_ingest MODIFY COLUMN cost_ms UInt32 CODEC(Delta, ZSTD(1));

-- =============================================
-- 阶段 2：添加 Skipping Index（加速高频过滤查询）
-- =============================================

-- trace_id 精确查找（查看日志详情 FindByTraceId）
-- bloom_filter(0.01) = 1% 误判率，GRANULARITY 4 = 每 4 个 granule 一个 bloom block
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 4;

-- IP 精确查找
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_ip ip TYPE bloom_filter(0.01) GRANULARITY 4;

-- host 模糊查询支持（tokenbf_v1 对 LIKE '%xxx%' 有效）
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_host host TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;

-- firewall_policy_id 过滤（HasFirewallPolicy: > 0）
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_fw_policy firewall_policy_id TYPE minmax GRANULARITY 4;

-- status 范围过滤（HasError: status >= 400）
ALTER TABLE logs_ingest ADD INDEX IF NOT EXISTS idx_status status TYPE minmax GRANULARITY 4;

-- =============================================
-- 阶段 3：物化索引到现有数据（对存量数据生效）
-- =============================================
-- 注意：MATERIALIZE INDEX 会触发后台 mutation，大表可能需要一定时间
-- 可通过 SELECT * FROM system.mutations WHERE is_done = 0 监控进度

ALTER TABLE logs_ingest MATERIALIZE INDEX idx_trace_id;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_ip;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_host;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_fw_policy;
ALTER TABLE logs_ingest MATERIALIZE INDEX idx_status;


-- =============================================================================
-- dns_logs_ingest 表优化（DNS 日志表）
-- =============================================================================

-- 大文本字段压缩
ALTER TABLE dns_logs_ingest MODIFY COLUMN content_json String CODEC(ZSTD(3));
ALTER TABLE dns_logs_ingest MODIFY COLUMN error String CODEC(ZSTD(1));

-- 低基数字段
ALTER TABLE dns_logs_ingest MODIFY COLUMN question_type LowCardinality(String);
ALTER TABLE dns_logs_ingest MODIFY COLUMN record_type LowCardinality(String);
ALTER TABLE dns_logs_ingest MODIFY COLUMN networking LowCardinality(String);

-- request_id 精确查找
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_request_id request_id TYPE bloom_filter(0.01) GRANULARITY 4;

-- remote_addr 精确查找
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_remote_addr remote_addr TYPE bloom_filter(0.01) GRANULARITY 4;

-- question_name 模糊查询
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_question_name question_name TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;

-- domain_id 过滤
ALTER TABLE dns_logs_ingest ADD INDEX IF NOT EXISTS idx_domain_id domain_id TYPE minmax GRANULARITY 4;

-- 物化索引到现有数据
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_request_id;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_remote_addr;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_question_name;
ALTER TABLE dns_logs_ingest MATERIALIZE INDEX idx_domain_id;


-- =============================================================================
-- 验证命令（执行完上述 ALTER 后运行）
-- =============================================================================

-- 查看列的压缩编解码器
-- SELECT name, type, compression_codec FROM system.columns WHERE table = 'logs_ingest' AND database = currentDatabase();

-- 查看表的压缩率
-- SELECT table, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed, round(sum(data_uncompressed_bytes) / sum(data_compressed_bytes), 2) AS ratio FROM system.columns WHERE table IN ('logs_ingest', 'dns_logs_ingest') GROUP BY table;

-- 查看各列占用的磁盘空间（找出最大的列）
-- SELECT name, formatReadableSize(sum(data_compressed_bytes)) AS compressed, formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed FROM system.columns WHERE table = 'logs_ingest' GROUP BY name ORDER BY sum(data_compressed_bytes) DESC;

-- 查看 mutation 进度
-- SELECT database, table, mutation_id, command, is_done, parts_to_do FROM system.mutations WHERE is_done = 0;

-- 强制触发 merge（可选，让压缩编解码器变更对存量数据生效）
-- OPTIMIZE TABLE logs_ingest FINAL;
-- OPTIMIZE TABLE dns_logs_ingest FINAL;