1.4.5.2

EdgeNode/internal/metrics/manager.go

@@ -0,0 +1,179 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
+	"github.com/TeaOSLab/EdgeNode/internal/events"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"strconv"
+	"sync"
+)
+
+var SharedManager = NewManager()
+
+func init() {
+	if !teaconst.IsMain {
+		return
+	}
+
+	events.OnClose(func() {
+		SharedManager.Quit()
+	})
+}
+
+type Manager struct {
+	isQuiting bool
+
+	taskMap         map[int64]Task    // itemId => Task
+	categoryTaskMap map[string][]Task // category => []Task
+	locker          sync.RWMutex
+
+	hasHTTPMetrics bool
+	hasTCPMetrics  bool
+	hasUDPMetrics  bool
+}
+
+func NewManager() *Manager {
+	return &Manager{
+		taskMap:         map[int64]Task{},
+		categoryTaskMap: map[string][]Task{},
+	}
+}
+
+func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
+	if this.isQuiting {
+		return
+	}
+
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var newMap = map[int64]*serverconfigs.MetricItemConfig{}
+	for _, item := range items {
+		newMap[item.Id] = item
+	}
+
+	// 停用以前的 或 修改现在的
+	for itemId, task := range this.taskMap {
+		newItem, ok := newMap[itemId]
+		if !ok || !newItem.IsOn { // 停用以前的
+			remotelogs.Println("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"'")
+			err := task.Stop()
+			if err != nil {
+				remotelogs.Error("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"' failed: "+err.Error())
+			}
+
+			// deleted
+			if newItem != nil && !newItem.IsOn {
+				deleteErr := task.Delete()
+				if deleteErr != nil {
+					remotelogs.Error("METRIC_MANAGER", "delete task '"+strconv.FormatInt(itemId, 10)+"' failed: "+err.Error())
+				}
+			}
+
+			delete(this.taskMap, itemId)
+		} else { // 更新已存在的
+			if newItem.Version != task.Item().Version {
+				remotelogs.Println("METRIC_MANAGER", "update task '"+strconv.FormatInt(itemId, 10)+"'")
+				task.SetItem(newItem)
+			}
+		}
+	}
+
+	// 启动新的
+	for _, newItem := range items {
+		if !newItem.IsOn {
+			continue
+		}
+		_, ok := this.taskMap[newItem.Id]
+		if !ok {
+			remotelogs.Println("METRIC_MANAGER", "start task '"+strconv.FormatInt(newItem.Id, 10)+"'")
+			var task Task
+
+			if CheckSQLiteDB(newItem.Id) || !teaconst.EnableKVCacheStore {
+				task = NewSQLiteTask(newItem)
+			} else {
+				task = NewKVTask(newItem)
+			}
+
+			err := task.Init()
+			if err != nil {
+				remotelogs.Error("METRIC_MANAGER", "initialized task failed: "+err.Error())
+				continue
+			}
+			err = task.Start()
+			if err != nil {
+				remotelogs.Error("METRIC_MANAGER", "start task failed: "+err.Error())
+				continue
+			}
+			this.taskMap[newItem.Id] = task
+		}
+	}
+
+	// 按分类存放
+	this.hasHTTPMetrics = false
+	this.hasTCPMetrics = false
+	this.hasUDPMetrics = false
+	this.categoryTaskMap = map[string][]Task{}
+	for _, task := range this.taskMap {
+		var tasks = this.categoryTaskMap[task.Item().Category]
+		tasks = append(tasks, task)
+		this.categoryTaskMap[task.Item().Category] = tasks
+
+		switch task.Item().Category {
+		case serverconfigs.MetricItemCategoryHTTP:
+			this.hasHTTPMetrics = true
+		case serverconfigs.MetricItemCategoryTCP:
+			this.hasTCPMetrics = true
+		case serverconfigs.MetricItemCategoryUDP:
+			this.hasUDPMetrics = true
+		}
+	}
+}
+
+// Add 添加数据
+func (this *Manager) Add(obj MetricInterface) {
+	if this.isQuiting {
+		return
+	}
+
+	this.locker.RLock()
+	var tasks = this.categoryTaskMap[obj.MetricCategory()]
+	this.locker.RUnlock()
+
+	for _, task := range tasks {
+		task.Add(obj)
+	}
+}
+
+func (this *Manager) HasHTTPMetrics() bool {
+	return this.hasHTTPMetrics
+}
+
+func (this *Manager) HasTCPMetrics() bool {
+	return this.hasTCPMetrics
+}
+
+func (this *Manager) HasUDPMetrics() bool {
+	return this.hasUDPMetrics
+}
+
+func (this *Manager) TaskMap() map[int64]Task {
+	return this.taskMap
+}
+
+// Quit 退出管理器
+func (this *Manager) Quit() {
+	this.isQuiting = true
+
+	remotelogs.Println("METRIC_MANAGER", "quit")
+
+	this.locker.Lock()
+	for _, task := range this.taskMap {
+		_ = task.Stop()
+	}
+	this.taskMap = map[int64]Task{}
+	this.locker.Unlock()
+}

EdgeNode/internal/metrics/manager_test.go

@@ -0,0 +1,64 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/metrics"
+	"testing"
+)
+
+func TestNewManager(t *testing.T) {
+	var manager = metrics.NewManager()
+	{
+		manager.Update([]*serverconfigs.MetricItemConfig{})
+		for _, task := range manager.TaskMap() {
+			t.Log(task.Item().Id)
+		}
+	}
+	{
+		t.Log("====")
+		manager.Update([]*serverconfigs.MetricItemConfig{
+			{
+				Id: 1,
+			},
+			{
+				Id: 2,
+			},
+			{
+				Id: 3,
+			},
+		})
+		for _, task := range manager.TaskMap() {
+			t.Log("task:", task.Item().Id)
+		}
+	}
+
+	{
+		t.Log("====")
+		manager.Update([]*serverconfigs.MetricItemConfig{
+			{
+				Id: 1,
+			},
+			{
+				Id: 2,
+			},
+		})
+		for _, task := range manager.TaskMap() {
+			t.Log("task:", task.Item().Id)
+		}
+	}
+
+	{
+		t.Log("====")
+		manager.Update([]*serverconfigs.MetricItemConfig{
+			{
+				Id:      1,
+				Version: 1,
+			},
+		})
+		for _, task := range manager.TaskMap() {
+			t.Log("task:", task.Item().Id)
+		}
+	}
+}

EdgeNode/internal/metrics/metric_interface.go

@@ -0,0 +1,17 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics
+
+type MetricInterface interface {
+	// MetricKey 指标对象
+	MetricKey(key string) string
+
+	// MetricValue 指标值
+	MetricValue(value string) (result int64, ok bool)
+
+	// MetricServerId 服务ID
+	MetricServerId() int64
+
+	// MetricCategory 指标分类
+	MetricCategory() string
+}

EdgeNode/internal/metrics/stat.go

@@ -0,0 +1,122 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	byteutils "github.com/TeaOSLab/EdgeNode/internal/utils/byte"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	"strconv"
+	"strings"
+)
+
+type Stat struct {
+	ServerId int64    `json:"serverId"`
+	Keys     []string `json:"keys"`
+	Hash     string   `json:"hash"`
+	Value    int64    `json:"value"`
+	Time     string   `json:"time"`
+}
+
+func UniqueKey(serverId int64, keys []string, time string, version int32, itemId int64) string {
+	var keysData = strings.Join(keys, "$EDGE$")
+	return strconv.FormatUint(fnv.HashString(strconv.FormatInt(serverId, 10)+"@"+keysData+"@"+time+"@"+strconv.Itoa(int(version))+"@"+strconv.FormatInt(itemId, 10)), 10)
+}
+
+func (this *Stat) UniqueKey(version int32, itemId int64) string {
+	return UniqueKey(this.ServerId, this.Keys, this.Time, version, itemId)
+}
+
+func (this *Stat) FullKey(version int32, itemId int64) string {
+	return this.Time + "_" + string(int32ToBigEndian(version)) + this.UniqueKey(version, itemId)
+}
+
+func (this *Stat) EncodeValueKey(version int32) string {
+	if this.Value < 0 {
+		this.Value = 0
+	}
+
+	return string(byteutils.Concat([]byte(this.Time), []byte{'_'}, int32ToBigEndian(version), int64ToBigEndian(this.ServerId), int64ToBigEndian(this.Value), []byte(this.Hash)))
+}
+
+func (this *Stat) EncodeSumKey(version int32) string {
+	return string(byteutils.Concat([]byte(this.Time), []byte{'_'}, int32ToBigEndian(version), int64ToBigEndian(this.ServerId)))
+}
+
+func DecodeValueKey(valueKey string) (serverId int64, timeString string, version int32, value int64, hash string, err error) {
+	var b = []byte(valueKey)
+	var timeIndex = bytes.Index(b, []byte{'_'})
+	if timeIndex < 0 {
+		return
+	}
+
+	timeString = string(b[:timeIndex])
+	b = b[timeIndex+1:]
+
+	if len(b) < 20+1 {
+		err = errors.New("invalid value key")
+		return
+	}
+
+	version = int32(binary.BigEndian.Uint32(b[0:4]))
+	serverId = int64(binary.BigEndian.Uint64(b[4:12]))
+	value = int64(binary.BigEndian.Uint64(b[12:20]))
+	hash = string(b[20:])
+	return
+}
+
+func DecodeSumKey(sumKey string) (serverId int64, timeString string, version int32, err error) {
+	var b = []byte(sumKey)
+	var timeIndex = bytes.Index(b, []byte{'_'})
+	if timeIndex < 0 {
+		return
+	}
+
+	timeString = string(b[:timeIndex])
+	b = b[timeIndex+1:]
+
+	if len(b) < 12 {
+		err = errors.New("invalid sum key")
+		return
+	}
+
+	version = int32(binary.BigEndian.Uint32(b[:4]))
+	serverId = int64(binary.BigEndian.Uint64(b[4:12]))
+	return
+}
+
+func EncodeSumValue(count uint64, total uint64) []byte {
+	var result [16]byte
+	binary.BigEndian.PutUint64(result[:8], count)
+	binary.BigEndian.PutUint64(result[8:], total)
+	return result[:]
+}
+
+func DecodeSumValue(data []byte) (count uint64, total uint64) {
+	if len(data) != 16 {
+		return
+	}
+	count = binary.BigEndian.Uint64(data[:8])
+	total = binary.BigEndian.Uint64(data[8:])
+	return
+}
+
+func int64ToBigEndian(i int64) []byte {
+	if i < 0 {
+		i = 0
+	}
+	var b = make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(i))
+	return b
+}
+
+func int32ToBigEndian(i int32) []byte {
+	if i < 0 {
+		i = 0
+	}
+	var b = make([]byte, 4)
+	binary.BigEndian.PutUint32(b, uint32(i))
+	return b
+}

EdgeNode/internal/metrics/stat_test.go

@@ -0,0 +1,69 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/metrics"
+	"github.com/iwind/TeaGo/assert"
+	"testing"
+)
+
+func TestStat_EncodeValueKey(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var stat = &metrics.Stat{
+		ServerId: 1,
+		Keys:     []string{"${remoteAddr}"},
+		Hash:     "123456",
+		Value:    123,
+		Time:     "20240101",
+	}
+
+	var valueKey = stat.EncodeValueKey(100)
+	t.Log(valueKey)
+
+	serverId, timeString, version, value, hash, err := metrics.DecodeValueKey(valueKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(serverId, timeString, value, version, hash)
+	a.IsTrue(serverId == 1)
+	a.IsTrue(timeString == "20240101")
+	a.IsTrue(value == 123)
+	a.IsTrue(version == 100)
+	a.IsTrue(hash == "123456")
+}
+
+func TestStat_EncodeSumKey(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var stat = &metrics.Stat{
+		ServerId: 1,
+		Keys:     []string{"${remoteAddr}"},
+		Hash:     "123456",
+		Value:    123,
+		Time:     "20240101",
+	}
+	var sumKey = stat.EncodeSumKey(100)
+	t.Log(sumKey)
+
+	serverId, timeString, version, err := metrics.DecodeSumKey(sumKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(serverId, timeString, version)
+	a.IsTrue(serverId == 1)
+	a.IsTrue(timeString == "20240101")
+	a.IsTrue(version == 100)
+}
+
+func TestStat_EncodeSumValue(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var b = metrics.EncodeSumValue(123, 456)
+	t.Log(b)
+
+	count, sum := metrics.DecodeSumValue(b)
+	a.IsTrue(count == 123)
+	a.IsTrue(sum == 456)
+}

EdgeNode/internal/metrics/sum_test.go

@@ -0,0 +1,20 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/metrics"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"runtime"
+	"testing"
+)
+
+func BenchmarkSumStat(b *testing.B) {
+	runtime.GOMAXPROCS(2)
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			metrics.UniqueKey(1, []string{"1.2.3.4"}, timeutil.Format("Ymd"), 1, 1)
+		}
+	})
+}

EdgeNode/internal/metrics/task.go

@@ -0,0 +1,21 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"time"
+)
+
+type Task interface {
+	Init() error
+	Item() *serverconfigs.MetricItemConfig
+	SetItem(item *serverconfigs.MetricItemConfig)
+	Add(obj MetricInterface)
+	InsertStat(stat *Stat) error
+	Upload(pauseDuration time.Duration) error
+	Start() error
+	Stop() error
+	Delete() error
+	CleanExpired() error
+}

EdgeNode/internal/metrics/task_base.go

@@ -0,0 +1,75 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"sync"
+	"sync/atomic"
+)
+
+type BaseTask struct {
+	itemConfig *serverconfigs.MetricItemConfig
+	isLoaded   bool
+	isStopped  bool
+
+	statsMap    map[string]*Stat // 待写入队列，hash => *Stat
+	statsLocker sync.RWMutex
+}
+
+// Add 添加数据
+func (this *BaseTask) Add(obj MetricInterface) {
+	if this.isStopped || !this.isLoaded {
+		return
+	}
+
+	var keys = []string{}
+	for _, key := range this.itemConfig.Keys {
+		var k = obj.MetricKey(key)
+
+		// 忽略499状态
+		if key == "${status}" && k == "499" {
+			return
+		}
+
+		keys = append(keys, k)
+	}
+
+	v, ok := obj.MetricValue(this.itemConfig.Value)
+	if !ok {
+		return
+	}
+
+	var hash = UniqueKey(obj.MetricServerId(), keys, this.itemConfig.CurrentTime(), this.itemConfig.Version, this.itemConfig.Id)
+	var countItems int
+	this.statsLocker.RLock()
+	oldStat, ok := this.statsMap[hash]
+	if !ok {
+		countItems = len(this.statsMap)
+	}
+	this.statsLocker.RUnlock()
+	if ok {
+		atomic.AddInt64(&oldStat.Value, 1)
+	} else {
+		// 防止过载
+		if countItems < MaxQueueSize {
+			this.statsLocker.Lock()
+			this.statsMap[hash] = &Stat{
+				ServerId: obj.MetricServerId(),
+				Keys:     keys,
+				Value:    v,
+				Time:     this.itemConfig.CurrentTime(),
+				Hash:     hash,
+			}
+			this.statsLocker.Unlock()
+		}
+	}
+}
+
+func (this *BaseTask) Item() *serverconfigs.MetricItemConfig {
+	return this.itemConfig
+}
+
+func (this *BaseTask) SetItem(itemConfig *serverconfigs.MetricItemConfig) {
+	this.itemConfig = itemConfig
+}

EdgeNode/internal/metrics/task_kv.go

@@ -0,0 +1,538 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	byteutils "github.com/TeaOSLab/EdgeNode/internal/utils/byte"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/idles"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/trackers"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
+	"github.com/cockroachdb/pebble"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TODO sumValues不用每次insertStat的时候都保存
+
+// KVTask KV存储实现的任务管理
+type KVTask struct {
+	BaseTask
+
+	itemsTable  *kvstore.Table[*Stat]  // hash => *Stat
+	valuesTable *kvstore.Table[[]byte] // time_version_serverId_value_hash => []byte(nil)
+	sumTable    *kvstore.Table[[]byte] // time_version_serverId => [count][total]
+
+	serverTimeMap     map[string]zero.Zero // 有变更的网站 serverId_time => Zero
+	serverIdMapLocker sync.Mutex
+
+	statsTicker  *utils.Ticker
+	cleanTicker  *time.Ticker
+	uploadTicker *utils.Ticker
+
+	valuesCacheMap map[string]int64 // hash => value
+}
+
+func NewKVTask(itemConfig *serverconfigs.MetricItemConfig) *KVTask {
+	return &KVTask{
+		BaseTask: BaseTask{
+			itemConfig: itemConfig,
+			statsMap:   map[string]*Stat{},
+		},
+
+		serverTimeMap:  map[string]zero.Zero{},
+		valuesCacheMap: map[string]int64{},
+	}
+}
+
+func (this *KVTask) Init() error {
+	store, err := kvstore.DefaultStore()
+	if err != nil {
+		return err
+	}
+
+	db, err := store.NewDB("metrics" + types.String(this.itemConfig.Id))
+	if err != nil {
+		return err
+	}
+
+	{
+		table, tableErr := kvstore.NewTable[*Stat]("items", &ItemEncoder[*Stat]{})
+		if tableErr != nil {
+			return tableErr
+		}
+		db.AddTable(table)
+		this.itemsTable = table
+	}
+
+	{
+		table, tableErr := kvstore.NewTable[[]byte]("values", kvstore.NewNilValueEncoder())
+		if tableErr != nil {
+			return tableErr
+		}
+		db.AddTable(table)
+		this.valuesTable = table
+	}
+
+	{
+		table, tableErr := kvstore.NewTable[[]byte]("sum_values", kvstore.NewBytesValueEncoder())
+		if tableErr != nil {
+			return tableErr
+		}
+		db.AddTable(table)
+		this.sumTable = table
+	}
+
+	// 所有的服务IDs
+	err = this.loadServerIdMap()
+	if err != nil {
+		return err
+	}
+
+	this.isLoaded = true
+
+	return nil
+}
+
+func (this *KVTask) InsertStat(stat *Stat) error {
+	if this.isStopped {
+		return nil
+	}
+	if stat == nil {
+		return nil
+	}
+
+	var version = this.itemConfig.Version
+
+	this.serverIdMapLocker.Lock()
+	this.serverTimeMap[types.String(stat.ServerId)+"_"+stat.Time] = zero.New()
+	this.serverIdMapLocker.Unlock()
+
+	if len(stat.Hash) == 0 {
+		stat.Hash = stat.UniqueKey(version, this.itemConfig.Id)
+	}
+
+	var isNew bool
+	var newValue = stat.Value
+
+	// insert or update
+	{
+		var statKey = stat.FullKey(version, this.itemConfig.Id)
+		oldStat, err := this.itemsTable.Get(statKey)
+		var oldValue int64
+		if err != nil {
+			if !kvstore.IsNotFound(err) {
+				return err
+			}
+			isNew = true
+		} else {
+			oldValue = oldStat.Value
+
+			// delete old value from valuesTable
+			err = this.valuesTable.Delete(oldStat.EncodeValueKey(version))
+			if err != nil {
+				return err
+			}
+		}
+
+		oldValue += stat.Value
+		stat.Value = oldValue
+		err = this.itemsTable.Set(statKey, stat)
+		if err != nil {
+			return err
+		}
+
+		// insert value into valuesTable
+		err = this.valuesTable.Insert(stat.EncodeValueKey(version), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	// sum
+	{
+		var sumKey = stat.EncodeSumKey(version)
+		sumResult, err := this.sumTable.Get(sumKey)
+		var count uint64
+		var total uint64
+		if err != nil {
+			if !kvstore.IsNotFound(err) {
+				return err
+			}
+		} else {
+			count, total = DecodeSumValue(sumResult)
+		}
+
+		if isNew {
+			count++
+		}
+		total += uint64(newValue)
+
+		err = this.sumTable.Set(sumKey, EncodeSumValue(count, total))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *KVTask) Upload(pauseDuration time.Duration) error {
+	var uploadTr = trackers.Begin("METRIC:UPLOAD_STATS")
+	defer uploadTr.End()
+
+	if this.isStopped {
+		return nil
+	}
+
+	this.serverIdMapLocker.Lock()
+
+	// 服务IDs
+	var serverTimeMap = this.serverTimeMap
+	this.serverTimeMap = map[string]zero.Zero{} // 清空数据
+
+	this.serverIdMapLocker.Unlock()
+
+	if len(serverTimeMap) == 0 {
+		return nil
+	}
+
+	// 控制缓存map不要太长
+	if len(this.valuesCacheMap) > 4096 {
+		var newMap = map[string]int64{}
+		var countElements int
+		for k, v := range this.valuesCacheMap {
+			newMap[k] = v
+			countElements++
+			if countElements >= 2048 {
+				break
+			}
+		}
+		this.valuesCacheMap = newMap
+	}
+
+	// 开始上传
+	rpcClient, err := rpc.SharedRPC()
+	if err != nil {
+		return err
+	}
+
+	var totalCount int
+
+	for serverTime := range serverTimeMap {
+		count, uploadErr := func(serverTime string) (int, error) {
+			serverIdString, timeString, found := strings.Cut(serverTime, "_")
+			if !found {
+				return 0, nil
+			}
+			var serverId = types.Int64(serverIdString)
+			if serverId <= 0 {
+				return 0, nil
+			}
+
+			return this.uploadServerStats(rpcClient, serverId, timeString)
+		}(serverTime)
+		if uploadErr != nil {
+			return uploadErr
+		}
+
+		totalCount += count
+
+		// 休息一下，防止短时间内上传数据过多
+		if pauseDuration > 0 && totalCount >= 100 {
+			time.Sleep(pauseDuration)
+			uploadTr.Add(-pauseDuration)
+		}
+	}
+
+	return nil
+}
+
+func (this *KVTask) Start() error {
+	// 读取数据
+	this.statsTicker = utils.NewTicker(1 * time.Minute)
+	if Tea.IsTesting() {
+		this.statsTicker = utils.NewTicker(10 * time.Second)
+	}
+	goman.New(func() {
+		for this.statsTicker.Next() {
+			var tr = trackers.Begin("METRIC:DUMP_STATS_TO_LOCAL_DATABASE")
+
+			this.statsLocker.Lock()
+			var statsMap = this.statsMap
+			this.statsMap = map[string]*Stat{}
+			this.statsLocker.Unlock()
+
+			for _, stat := range statsMap {
+				err := this.InsertStat(stat)
+				if err != nil {
+					remotelogs.Error("METRIC", "insert stat failed: "+err.Error())
+				}
+			}
+
+			tr.End()
+		}
+	})
+
+	// 清理
+	this.cleanTicker = time.NewTicker(24 * time.Hour)
+	goman.New(func() {
+		idles.RunTicker(this.cleanTicker, func() {
+			var tr = trackers.Begin("METRIC:CLEAN_EXPIRED")
+			err := this.CleanExpired()
+			tr.End()
+			if err != nil {
+				remotelogs.Error("METRIC", "clean expired stats failed: "+err.Error())
+			}
+		})
+	})
+
+	// 上传
+	this.uploadTicker = utils.NewTicker(this.itemConfig.UploadDuration())
+	goman.New(func() {
+		for this.uploadTicker.Next() {
+			err := this.Upload(1 * time.Second)
+			if err != nil && !rpc.IsConnError(err) {
+				remotelogs.Error("METRIC", "upload stats failed: "+err.Error())
+			}
+		}
+	})
+
+	return nil
+}
+
+func (this *KVTask) Stop() error {
+	this.isStopped = true
+
+	if this.cleanTicker != nil {
+		this.cleanTicker.Stop()
+	}
+	if this.uploadTicker != nil {
+		this.uploadTicker.Stop()
+	}
+	if this.statsTicker != nil {
+		this.statsTicker.Stop()
+	}
+
+	return nil
+}
+
+func (this *KVTask) Delete() error {
+	this.isStopped = true
+
+	return this.itemsTable.DB().Truncate()
+}
+
+func (this *KVTask) CleanExpired() error {
+	if this.isStopped {
+		return nil
+	}
+
+	var versionBytes = int32ToBigEndian(this.itemConfig.Version)
+	var expiresTime = this.itemConfig.LocalExpiresTime()
+
+	var rangeEnd = append([]byte(expiresTime+"_"), versionBytes...)
+	rangeEnd = append(rangeEnd, 0xFF, 0xFF)
+
+	{
+		err := this.itemsTable.DeleteRange("", string(rangeEnd))
+		if err != nil {
+			return err
+		}
+	}
+
+	{
+		err := this.valuesTable.DeleteRange("", string(rangeEnd))
+		if err != nil {
+			return err
+		}
+	}
+
+	{
+		err := this.sumTable.DeleteRange("", string(rangeEnd))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *KVTask) Flush() error {
+	return this.itemsTable.DB().Store().Flush()
+}
+
+func (this *KVTask) TestInspect(t *testing.T) {
+	var db = this.itemsTable.DB()
+	it, err := db.Store().RawDB().NewIter(&pebble.IterOptions{
+		LowerBound: []byte(db.Namespace()),
+		UpperBound: append([]byte(db.Namespace()), 0xFF),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = it.Close()
+	}()
+
+	for it.First(); it.Valid(); it.Next() {
+		valueBytes, valueErr := it.ValueAndErr()
+		if valueErr != nil {
+			t.Fatal(valueErr)
+		}
+		var key = string(it.Key()[len(db.Namespace())-1:])
+		t.Log(key, "=>", string(valueBytes))
+		if strings.HasPrefix(key, "$values$K$") {
+			_, _, _, value, hash, _ := DecodeValueKey(key[len("$values$K$"):])
+			t.Log("    |", hash, "=>", value)
+		} else if strings.HasPrefix(key, "$sumValues$K$") {
+			count, sum := DecodeSumValue(valueBytes)
+			t.Log("    |", count, sum)
+		}
+	}
+}
+
+func (this *KVTask) Truncate() error {
+	var db = this.itemsTable.DB()
+	err := db.Truncate()
+	if err != nil {
+		return err
+	}
+	return db.Store().Flush()
+}
+
+func (this *KVTask) uploadServerStats(rpcClient *rpc.RPCClient, serverId int64, currentTime string) (countValues int, uploadErr error) {
+	var pbStats []*pb.UploadingMetricStat
+	var keepKeys []string
+
+	var prefix = string(byteutils.Concat([]byte(currentTime), []byte{'_'}, int32ToBigEndian(this.itemConfig.Version), int64ToBigEndian(serverId)))
+	var newCachedKeys = map[string]int64{}
+	queryErr := this.valuesTable.
+		Query().
+		Prefix(prefix).
+		Desc().
+		Limit(20).
+		FindAll(func(tx *kvstore.Tx[[]byte], item kvstore.Item[[]byte]) (goNext bool, err error) {
+			_, _, version, value, hash, decodeErr := DecodeValueKey(item.Key)
+			if decodeErr != nil {
+				return false, decodeErr
+			}
+			if value <= 0 {
+				return true, nil
+			}
+
+			// value not changed for the key
+			if this.valuesCacheMap[hash] == value {
+				keepKeys = append(keepKeys, hash)
+				return true, nil
+			}
+
+			newCachedKeys[hash] = value
+
+			stat, valueErr := this.itemsTable.Get(string(byteutils.Concat([]byte(currentTime), []byte{'_'}, int32ToBigEndian(version), []byte(hash))))
+			if valueErr != nil {
+				if kvstore.IsNotFound(valueErr) {
+					return true, nil
+				}
+				return false, valueErr
+			}
+			if stat == nil {
+				return true, nil
+			}
+
+			pbStats = append(pbStats, &pb.UploadingMetricStat{
+				Id:    0, // not used in node
+				Hash:  hash,
+				Keys:  stat.Keys,
+				Value: float32(value),
+			})
+
+			return true, nil
+		})
+	if queryErr != nil {
+		return 0, queryErr
+	}
+
+	// count & total
+	var count, total uint64
+	{
+		sumValue, err := this.sumTable.Get(prefix)
+		if err != nil {
+			if kvstore.IsNotFound(err) {
+				return 0, nil
+			}
+			return 0, err
+		}
+		count, total = DecodeSumValue(sumValue)
+	}
+
+	_, err := rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
+		MetricStats: pbStats,
+		Time:        currentTime,
+		ServerId:    serverId,
+		ItemId:      this.itemConfig.Id,
+		Version:     this.itemConfig.Version,
+		Count:       int64(count),
+		Total:       float32(total),
+		KeepKeys:    keepKeys,
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	// put into cache map MUST be after uploading success
+	for k, v := range newCachedKeys {
+		this.valuesCacheMap[k] = v
+	}
+
+	return len(pbStats), nil
+}
+
+func (this *KVTask) loadServerIdMap() error {
+	var offsetKey string
+	var currentTime = this.itemConfig.CurrentTime()
+	for {
+		var found bool
+		err := this.sumTable.
+			Query().
+			Limit(1000).
+			Offset(offsetKey).
+			FindAll(func(tx *kvstore.Tx[[]byte], item kvstore.Item[[]byte]) (goNext bool, err error) {
+				offsetKey = item.Key
+				found = true
+
+				serverId, timeString, version, decodeErr := DecodeSumKey(item.Key)
+				if decodeErr != nil {
+					return false, decodeErr
+				}
+
+				if version != this.itemConfig.Version || timeString != currentTime {
+					return true, nil
+				}
+
+				this.serverIdMapLocker.Lock()
+				this.serverTimeMap[types.String(serverId)+"_"+timeString] = zero.New()
+				this.serverIdMapLocker.Unlock()
+
+				return true, nil
+			})
+		if err != nil {
+			return err
+		}
+		if !found {
+			break
+		}
+	}
+
+	return nil
+}

EdgeNode/internal/metrics/task_kv_objects.go

@@ -0,0 +1,24 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+type ItemEncoder[T interface{ *Stat }] struct {
+}
+
+func (this *ItemEncoder[T]) Encode(value T) ([]byte, error) {
+	return json.Marshal(value)
+}
+
+func (this *ItemEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error) {
+	return nil, errors.New("invalid field name '" + fieldName + "'")
+}
+
+func (this *ItemEncoder[T]) Decode(valueBytes []byte) (value T, err error) {
+	err = json.Unmarshal(valueBytes, &value)
+	return
+}

EdgeNode/internal/metrics/task_kv_test.go

@@ -0,0 +1,367 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics_test
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/metrics"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/rands"
+	"log"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+)
+
+type testObj struct {
+	ip string
+}
+
+func (this *testObj) MetricKey(key string) string {
+	return this.ip
+}
+
+func (this *testObj) MetricValue(value string) (int64, bool) {
+	return 1, true
+}
+
+func (this *testObj) MetricServerId() int64 {
+	return int64(rands.Int(1, 100))
+}
+
+func (this *testObj) MetricCategory() string {
+	return "http"
+}
+
+func TestKVTask_Init(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     0,
+		PeriodUnit: "",
+		Keys:       nil,
+		Value:      "",
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+	t.Log("ok")
+}
+
+func TestKVTask_Add(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = task.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+
+	task.Add(&testObj{ip: "127.0.0.2"})
+
+	if testutils.IsSingleTesting() {
+		time.Sleep(1 * time.Second) // waiting for inserting
+	}
+}
+
+func TestKVTask_Add_Many(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = task.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+
+	for i := 0; i < 4_000_000; i++ {
+		task.Add(&testObj{
+			ip: fmt.Sprintf("%d.%d.%d.%d", rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255)),
+		})
+		if i%10000 == 0 {
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
+func TestKVTask_InsertStat(t *testing.T) {
+	var item = &serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	}
+	var task = metrics.NewKVTask(item)
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		err = task.Flush()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err = task.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+
+	{
+		err = task.InsertStat(&metrics.Stat{
+			ServerId: 1,
+			Keys:     []string{"127.0.0.1"},
+			Hash:     "",
+			Value:    1,
+			Time:     item.CurrentTime(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	{
+		err = task.InsertStat(&metrics.Stat{
+			ServerId: 2,
+			Keys:     []string{"127.0.0.2"},
+			Hash:     "",
+			Value:    3,
+			Time:     item.CurrentTime(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	{
+		err = task.InsertStat(&metrics.Stat{
+			ServerId: 1,
+			Keys:     []string{"127.0.0.3"},
+			Hash:     "",
+			Value:    2,
+			Time:     item.CurrentTime(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	TestKVTask_TestInspect(t)
+}
+
+func TestKVTask_CleanExpired(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = task.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+
+	err = task.CleanExpired()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		_ = task.Flush()
+	}()
+
+	t.Log("=== inspect ===")
+	task.TestInspect(t)
+}
+
+func TestKVTask_Upload(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         31,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = task.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = task.Stop()
+	}()
+
+	err = task.Upload(0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestKVTask_TestInspect(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	task.TestInspect(t)
+}
+
+func TestKVTask_Truncate(t *testing.T) {
+	var task = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+		Version:    1,
+	})
+	err := task.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if testutils.IsSingleTesting() {
+		_ = task.Truncate()
+	}
+}
+
+var testingTask metrics.Task
+var testingTaskInitOnce = &sync.Once{}
+
+func initTestingTask() {
+	testingTask = metrics.NewKVTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "tcp",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+	})
+
+	err := testingTask.Init()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = testingTask.Start()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func BenchmarkKVTask_Add(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	testingTaskInitOnce.Do(func() {
+		initTestingTask()
+	})
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			testingTask.Add(&taskRequest{})
+		}
+	})
+}
+
+type taskRequest struct {
+}
+
+func (this *taskRequest) MetricKey(key string) string {
+	return configutils.ParseVariables(key, func(varName string) (value string) {
+		return "1.2.3.4"
+	})
+}
+
+func (this *taskRequest) MetricValue(value string) (result int64, ok bool) {
+	return 1, true
+}
+
+func (this *taskRequest) MetricServerId() int64 {
+	return 1
+}
+
+func (this *taskRequest) MetricCategory() string {
+	return "http"
+}

EdgeNode/internal/metrics/task_sqlite.go

@@ -0,0 +1,506 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package metrics
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/idles"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/trackers"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/types"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const MaxQueueSize = 256 // TODO 可以配置，可以在单个任务里配置
+
+// SQLiteTask 单个指标任务
+// 数据库存储：
+//
+//	data/
+//	   metric.$ID.db
+//	      stats
+//	         id, keys, value, time, serverId, hash
+//	原理：
+//	   添加或者有变更时 isUploaded = false
+//	   上传时检查 isUploaded 状态
+//	   只上传每个服务中排序最前面的 N 个数据
+type SQLiteTask struct {
+	BaseTask
+
+	db            *dbs.DB
+	statTableName string
+
+	cleanTicker  *time.Ticker
+	uploadTicker *utils.Ticker
+
+	cleanVersion int32
+
+	insertStatStmt          *dbs.Stmt
+	deleteByVersionStmt     *dbs.Stmt
+	deleteByExpiresTimeStmt *dbs.Stmt
+	selectTopStmt           *dbs.Stmt
+	sumStmt                 *dbs.Stmt
+
+	serverIdMap       map[int64]zero.Zero  // 所有的服务Ids
+	timeMap           map[string]zero.Zero // time => bool
+	serverIdMapLocker sync.Mutex
+
+	statsTicker *utils.Ticker
+}
+
+// NewSQLiteTask 获取新任务
+func NewSQLiteTask(item *serverconfigs.MetricItemConfig) *SQLiteTask {
+	return &SQLiteTask{
+		BaseTask: BaseTask{
+			itemConfig: item,
+			statsMap:   map[string]*Stat{},
+		},
+
+		serverIdMap: map[int64]zero.Zero{},
+		timeMap:     map[string]zero.Zero{},
+	}
+}
+
+func CheckSQLiteDB(itemId int64) bool {
+	var path = Tea.Root + "/data/metric." + types.String(itemId) + ".db"
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// Init 初始化
+func (this *SQLiteTask) Init() error {
+	this.statTableName = "stats"
+
+	// 检查目录是否存在
+	var dir = Tea.Root + "/data"
+	_, err := os.Stat(dir)
+	if err != nil {
+		err = os.MkdirAll(dir, 0777)
+		if err != nil {
+			return err
+		}
+		remotelogs.Println("METRIC", "create data dir '"+dir+"'")
+	}
+
+	var path = dir + "/metric." + types.String(this.itemConfig.Id) + ".db"
+
+	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
+	if err != nil {
+		return err
+	}
+	db.SetMaxOpenConns(1)
+	this.db = db
+
+	// 恢复数据库
+	var recoverEnv, _ = os.LookupEnv("EdgeRecover")
+	if len(recoverEnv) > 0 {
+		for _, indexName := range []string{"serverId", "hash"} {
+			_, _ = db.Exec(`REINDEX "` + indexName + `"`)
+		}
+	}
+
+	if teaconst.EnableDBStat {
+		this.db.EnableStat(true)
+	}
+
+	//创建统计表
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.statTableName + `" (
+  "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
+  "hash" varchar(32),
+  "keys" varchar(1024),
+  "value" real DEFAULT 0,
+  "time" varchar(32),
+  "serverId" integer DEFAULT 0,
+  "version" integer DEFAULT 0,
+  "isUploaded" integer DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS "serverId"
+ON "` + this.statTableName + `" (
+  "serverId" ASC,
+  "version" ASC
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS "hash"
+ON "` + this.statTableName + `" (
+  "hash" ASC
+);`)
+	if err != nil {
+		return err
+	}
+
+	// insert stat stmt
+	this.insertStatStmt, err = db.Prepare(`INSERT INTO "stats" ("serverId", "hash", "keys", "value", "time", "version", "isUploaded") VALUES (?, ?, ?, ?, ?, ?, 0) ON CONFLICT("hash") DO UPDATE SET "value"="value"+?, "isUploaded"=0`)
+	if err != nil {
+		return err
+	}
+
+	// delete by version
+	this.deleteByVersionStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "version"<?`)
+	if err != nil {
+		return err
+	}
+
+	// delete by expires time
+	this.deleteByExpiresTimeStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "time"<?`)
+	if err != nil {
+		return err
+	}
+
+	// select topN stmt
+	this.selectTopStmt, err = db.Prepare(`SELECT "id", "hash", "keys", "value", "isUploaded" FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=? ORDER BY "value" DESC LIMIT 20`)
+	if err != nil {
+		return err
+	}
+
+	// sum stmt
+	this.sumStmt, err = db.Prepare(`SELECT COUNT(*), IFNULL(SUM(value), 0) FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=?`)
+	if err != nil {
+		return err
+	}
+
+	// 所有的服务IDs
+	err = this.loadServerIdMap()
+	if err != nil {
+		return err
+	}
+
+	this.isLoaded = true
+
+	return nil
+}
+
+// Start 启动任务
+func (this *SQLiteTask) Start() error {
+	// 读取数据
+	this.statsTicker = utils.NewTicker(1 * time.Minute)
+	goman.New(func() {
+		for this.statsTicker.Next() {
+			var tr = trackers.Begin("METRIC:DUMP_STATS_TO_LOCAL_DATABASE")
+
+			this.statsLocker.Lock()
+			var statsMap = this.statsMap
+			this.statsMap = map[string]*Stat{}
+			this.statsLocker.Unlock()
+
+			for _, stat := range statsMap {
+				err := this.InsertStat(stat)
+				if err != nil {
+					remotelogs.Error("METRIC", "insert stat failed: "+err.Error())
+				}
+			}
+
+			tr.End()
+		}
+	})
+
+	// 清理
+	this.cleanTicker = time.NewTicker(24 * time.Hour)
+	goman.New(func() {
+		idles.RunTicker(this.cleanTicker, func() {
+			var tr = trackers.Begin("METRIC:CLEAN_EXPIRED")
+			err := this.CleanExpired()
+			tr.End()
+			if err != nil {
+				remotelogs.Error("METRIC", "clean expired stats failed: "+err.Error())
+			}
+		})
+	})
+
+	// 上传
+	this.uploadTicker = utils.NewTicker(this.itemConfig.UploadDuration())
+	goman.New(func() {
+		for this.uploadTicker.Next() {
+			err := this.Upload(1 * time.Second)
+			if err != nil && !rpc.IsConnError(err) {
+				remotelogs.Error("METRIC", "upload stats failed: "+err.Error())
+			}
+		}
+	})
+
+	return nil
+}
+
+// Stop 停止任务
+func (this *SQLiteTask) Stop() error {
+	this.isStopped = true
+
+	if this.cleanTicker != nil {
+		this.cleanTicker.Stop()
+	}
+	if this.uploadTicker != nil {
+		this.uploadTicker.Stop()
+	}
+	if this.statsTicker != nil {
+		this.statsTicker.Stop()
+	}
+
+	_ = this.insertStatStmt.Close()
+	_ = this.deleteByVersionStmt.Close()
+	_ = this.deleteByExpiresTimeStmt.Close()
+	_ = this.selectTopStmt.Close()
+	_ = this.sumStmt.Close()
+
+	if this.db != nil {
+		_ = this.db.Close()
+	}
+
+	return nil
+}
+
+func (this *SQLiteTask) Delete() error {
+	return nil
+}
+
+// InsertStat 写入数据
+func (this *SQLiteTask) InsertStat(stat *Stat) error {
+	if this.isStopped {
+		return nil
+	}
+	if stat == nil {
+		return nil
+	}
+
+	this.serverIdMapLocker.Lock()
+	this.serverIdMap[stat.ServerId] = zero.New()
+	this.timeMap[stat.Time] = zero.New()
+	this.serverIdMapLocker.Unlock()
+
+	keyData, err := json.Marshal(stat.Keys)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.insertStatStmt.Exec(stat.ServerId, stat.Hash, keyData, stat.Value, stat.Time, this.itemConfig.Version, stat.Value)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CleanExpired 清理数据
+func (this *SQLiteTask) CleanExpired() error {
+	if this.isStopped {
+		return nil
+	}
+
+	// 清除低版本数据
+	if this.cleanVersion < this.itemConfig.Version {
+		_, err := this.deleteByVersionStmt.Exec(this.itemConfig.Version)
+		if err != nil {
+			return err
+		}
+		this.cleanVersion = this.itemConfig.Version
+	}
+
+	// 清除过期的数据
+	_, err := this.deleteByExpiresTimeStmt.Exec(this.itemConfig.LocalExpiresTime())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Upload 上传数据
+func (this *SQLiteTask) Upload(pauseDuration time.Duration) error {
+	var uploadTr = trackers.Begin("METRIC:UPLOAD_STATS")
+	defer uploadTr.End()
+
+	if this.isStopped {
+		return nil
+	}
+
+	this.serverIdMapLocker.Lock()
+
+	// 服务IDs
+	var serverIds []int64
+	for serverId := range this.serverIdMap {
+		serverIds = append(serverIds, serverId)
+	}
+	this.serverIdMap = map[int64]zero.Zero{} // 清空数据
+
+	// 时间
+	var times = []string{}
+	for t := range this.timeMap {
+		times = append(times, t)
+	}
+	this.timeMap = map[string]zero.Zero{} // 清空数据
+
+	this.serverIdMapLocker.Unlock()
+
+	rpcClient, err := rpc.SharedRPC()
+	if err != nil {
+		return err
+	}
+
+	for _, serverId := range serverIds {
+		for _, currentTime := range times {
+			idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
+				var t = trackers.Begin("METRIC:SELECT_TOP_STMT")
+				rows, err := this.selectTopStmt.Query(serverId, this.itemConfig.Version, currentTime)
+				t.End()
+				if err != nil {
+					return nil, err
+				}
+				var isClosed bool
+				defer func() {
+					if isClosed {
+						return
+					}
+					_ = rows.Close()
+				}()
+
+				var pbStats []*pb.UploadingMetricStat
+				for rows.Next() {
+					var pbStat = &pb.UploadingMetricStat{}
+					// "id", "hash", "keys", "value", "isUploaded"
+					var isUploaded int
+					var keysData []byte
+					err = rows.Scan(&pbStat.Id, &pbStat.Hash, &keysData, &pbStat.Value, &isUploaded)
+					if err != nil {
+						return nil, err
+					}
+
+					// TODO 先不判断是否已经上传，需要改造API进行配合
+					/**if isUploaded == 1 {
+						continue
+					}**/
+					if len(keysData) > 0 {
+						err = json.Unmarshal(keysData, &pbStat.Keys)
+						if err != nil {
+							return nil, err
+						}
+					}
+					pbStats = append(pbStats, pbStat)
+					ids = append(ids, strconv.FormatInt(pbStat.Id, 10))
+				}
+
+				// 提前关闭
+				_ = rows.Close()
+				isClosed = true
+
+				// 上传
+				if len(pbStats) > 0 {
+					// 计算总和
+					count, total, err := this.sum(serverId, currentTime)
+					if err != nil {
+						return nil, err
+					}
+
+					_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
+						MetricStats: pbStats,
+						Time:        currentTime,
+						ServerId:    serverId,
+						ItemId:      this.itemConfig.Id,
+						Version:     this.itemConfig.Version,
+						Count:       count,
+						Total:       float32(total),
+					})
+					if err != nil {
+						return nil, err
+					}
+				}
+
+				return
+			}(serverId, currentTime)
+			if err != nil {
+				return err
+			}
+
+			if len(idStrings) > 0 {
+				// 设置为已上传
+				// TODO 先不判断是否已经上传，需要改造API进行配合
+				/**_, err = this.db.Exec(`UPDATE "` + this.statTableName + `" SET isUploaded=1 WHERE id IN (` + strings.Join(idStrings, ",") + `)`)
+				if err != nil {
+					return err
+				}**/
+			}
+		}
+
+		// 休息一下，防止短时间内上传数据过多
+		if pauseDuration > 0 {
+			time.Sleep(pauseDuration)
+			uploadTr.Add(-pauseDuration)
+		}
+	}
+
+	return nil
+}
+
+// 加载服务ID
+func (this *SQLiteTask) loadServerIdMap() error {
+	{
+		rows, err := this.db.Query(`SELECT DISTINCT "serverId" FROM `+this.statTableName+" WHERE version=?", this.itemConfig.Version)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = rows.Close()
+		}()
+
+		var serverId int64
+		for rows.Next() {
+			err = rows.Scan(&serverId)
+			if err != nil {
+				return err
+			}
+			this.serverIdMapLocker.Lock()
+			this.serverIdMap[serverId] = zero.New()
+			this.serverIdMapLocker.Unlock()
+		}
+	}
+
+	{
+		rows, err := this.db.Query(`SELECT DISTINCT "time" FROM `+this.statTableName+" WHERE version=?", this.itemConfig.Version)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = rows.Close()
+		}()
+
+		var timeString string
+		for rows.Next() {
+			err = rows.Scan(&timeString)
+			if err != nil {
+				return err
+			}
+			this.serverIdMapLocker.Lock()
+			this.timeMap[timeString] = zero.New()
+			this.serverIdMapLocker.Unlock()
+		}
+	}
+
+	return nil
+}
+
+// 计算数量和综合
+func (this *SQLiteTask) sum(serverId int64, time string) (count int64, total float64, err error) {
+	rows, err := this.sumStmt.Query(serverId, this.itemConfig.Version, time)
+	if err != nil {
+		return 0, 0, err
+	}
+	defer func() {
+		_ = rows.Close()
+	}()
+	if rows.Next() {
+		err = rows.Scan(&count, &total)
+		if err != nil {
+			return 0, 0, err
+		}
+	}
+	return
+}