v1.5.1 增强程序稳定性

EdgeNode/internal/caches/list_file_db_sqlite.go

@@ -3,6 +3,7 @@
 package caches
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -141,7 +142,7 @@ func (this *SQLiteFileListDB) Init() error {
 		return err
 	}
 
-	this.selectByHashStmt, err = this.readDB.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
+	this.selectByHashStmt, err = this.readDB.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
 	if err != nil {
 		return err
 	}
@@ -302,6 +303,28 @@ func (this *SQLiteFileListDB) ListHashes(lastId int64) (hashList []string, maxId
 	return
 }
 
+func (this *SQLiteFileListDB) ReadItem(hash string) (*Item, error) {
+	if len(hash) == 0 {
+		return nil, nil
+	}
+
+	row := this.selectByHashStmt.QueryRow(hash)
+	if row == nil {
+		return nil, nil
+	}
+
+	var item = &Item{Type: ItemTypeFile}
+	err := row.Scan(&item.Key, &item.HeaderSize, &item.BodySize, &item.MetaSize, &item.ExpiresAt, &item.StaleAt, &item.Host, &item.ServerId, &item.CreatedAt)
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return item, nil
+}
+
 func (this *SQLiteFileListDB) IncreaseHitAsync(hash string) error {
 	// do nothing
 	return nil

EdgeNode/internal/caches/list_file_migrate.go

@@ -0,0 +1,96 @@
+package caches
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"os"
+)
+
+func MigrateSQLiteFileListDir(sqliteDir string, kvDir string) error {
+	if len(sqliteDir) == 0 || len(kvDir) == 0 {
+		return nil
+	}
+
+	_, err := os.Stat(sqliteDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	remotelogs.Println("CACHE", "migrating sqlite indexes from '"+sqliteDir+"' to '"+kvDir+"' ...")
+
+	src := NewSQLiteFileList(sqliteDir).(*SQLiteFileList)
+	err = src.Init()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = src.Close()
+	}()
+
+	dst := NewKVFileList(kvDir)
+	err = dst.Init()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = dst.Close()
+	}()
+
+	err = dst.CleanAll()
+	if err != nil {
+		return err
+	}
+
+	for _, db := range src.dbList {
+		if db == nil {
+			continue
+		}
+
+		var lastId int64
+		for {
+			hashes, maxId, listErr := db.ListHashes(lastId)
+			if listErr != nil {
+				return listErr
+			}
+			if len(hashes) == 0 {
+				break
+			}
+
+			for _, hash := range hashes {
+				item, readErr := db.ReadItem(hash)
+				if readErr != nil {
+					return readErr
+				}
+				if item == nil {
+					continue
+				}
+
+				addErr := dst.Add(hash, item)
+				if addErr != nil {
+					return addErr
+				}
+			}
+
+			lastId = maxId
+		}
+	}
+
+	for _, store := range dst.stores {
+		if store != nil && store.rawStore != nil {
+			err = store.rawStore.Flush()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	err = os.RemoveAll(sqliteDir)
+	if err != nil {
+		return err
+	}
+
+	remotelogs.Println("CACHE", "migrated sqlite indexes to pebble")
+	return nil
+}

EdgeNode/internal/caches/storage_file.go

@@ -330,16 +330,25 @@ func (this *FileStorage) Init() error {
 	// read list
 	var list ListInterface
 	var sqliteIndexesDir = dir + "/p" + types.String(this.policy.Id) + "/.indexes"
+	var kvStoresDir = dir + "/p" + types.String(this.policy.Id) + "/.stores"
 	_, sqliteIndexesDirErr := os.Stat(sqliteIndexesDir)
-	if sqliteIndexesDirErr == nil || !teaconst.EnableKVCacheStore {
+	var useSQLite bool
+	if sqliteIndexesDirErr == nil {
+		err = MigrateSQLiteFileListDir(sqliteIndexesDir, kvStoresDir)
+		if err != nil {
+			remotelogs.Error("CACHE", "migrate sqlite indexes failed: "+err.Error())
+			useSQLite = true
+		}
+	}
+
+	if useSQLite {
 		list = NewSQLiteFileList(sqliteIndexesDir)
 		err = list.Init()
 		if err != nil {
 			return err
 		}
-		list.(*SQLiteFileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
 	} else {
-		list = NewKVFileList(dir + "/p" + types.String(this.policy.Id) + "/.stores")
+		list = NewKVFileList(kvStoresDir)
 		err = list.Init()
 		if err != nil {
 			return err