1.4.5.2
This commit is contained in:
126
EdgeNode/internal/utils/fs/disk.go
Normal file
126
EdgeNode/internal/utils/fs/disk.go
Normal file
@@ -0,0 +1,126 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"math"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const diskSpeedDataFile = "disk.speed.json"
|
||||
|
||||
type DiskSpeedCache struct {
|
||||
Speed Speed `json:"speed"`
|
||||
SpeedMB float64 `json:"speedMB"`
|
||||
CountTests int `json:"countTests"` // test times
|
||||
}
|
||||
|
||||
// CheckDiskWritingSpeed test disk writing speed
|
||||
func CheckDiskWritingSpeed() (speedMB float64, err error) {
|
||||
var tempDir = os.TempDir()
|
||||
if len(tempDir) == 0 {
|
||||
tempDir = "/tmp"
|
||||
}
|
||||
|
||||
const filename = "edge-disk-writing-test.data"
|
||||
var path = tempDir + "/" + filename
|
||||
_ = os.Remove(path) // always try to delete the file
|
||||
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var isClosed bool
|
||||
defer func() {
|
||||
if !isClosed {
|
||||
_ = fp.Close()
|
||||
}
|
||||
|
||||
_ = os.Remove(path)
|
||||
}()
|
||||
|
||||
var data = bytes.Repeat([]byte{'A'}, 16<<20)
|
||||
var before = time.Now()
|
||||
_, err = fp.Write(data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = fp.Sync()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = fp.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
speedMB = float64(len(data)) / (1 << 20) / costSeconds
|
||||
speedMB = math.Ceil(speedMB/10) * 10
|
||||
|
||||
isClosed = true
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// CheckDiskIsFast check disk is 'fast' disk to write
|
||||
func CheckDiskIsFast() (speedMB float64, isFast bool, err error) {
|
||||
speedMB, err = CheckDiskWritingSpeed()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// read old cached info
|
||||
var cacheFile = Tea.Root + "/data/" + diskSpeedDataFile
|
||||
var cacheInfo = &DiskSpeedCache{}
|
||||
{
|
||||
cacheData, cacheErr := os.ReadFile(cacheFile)
|
||||
if cacheErr == nil {
|
||||
var oldCacheInfo = &DiskSpeedCache{}
|
||||
cacheErr = json.Unmarshal(cacheData, oldCacheInfo)
|
||||
if cacheErr == nil {
|
||||
cacheInfo = oldCacheInfo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cacheInfo.CountTests++
|
||||
|
||||
defer func() {
|
||||
// write to local file
|
||||
cacheData, jsonErr := json.Marshal(cacheInfo)
|
||||
if jsonErr == nil {
|
||||
_ = os.WriteFile(cacheFile, cacheData, 0666)
|
||||
}
|
||||
}()
|
||||
|
||||
isFast = speedMB > 150
|
||||
|
||||
if speedMB <= DiskSpeedMB {
|
||||
return
|
||||
}
|
||||
|
||||
if speedMB > 1000 {
|
||||
DiskSpeed = SpeedExtremelyFast
|
||||
} else if speedMB > 150 {
|
||||
DiskSpeed = SpeedFast
|
||||
} else if speedMB > 60 {
|
||||
DiskSpeed = SpeedLow
|
||||
} else {
|
||||
DiskSpeed = SpeedExtremelySlow
|
||||
}
|
||||
|
||||
DiskSpeedMB = speedMB
|
||||
|
||||
cacheInfo.Speed = DiskSpeed
|
||||
cacheInfo.SpeedMB = DiskSpeedMB
|
||||
|
||||
return
|
||||
}
|
||||
16
EdgeNode/internal/utils/fs/disk_test_test.go
Normal file
16
EdgeNode/internal/utils/fs/disk_test_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckDiskWritingSpeed(t *testing.T) {
|
||||
t.Log(fsutils.CheckDiskWritingSpeed())
|
||||
}
|
||||
|
||||
func TestCheckDiskIsFast(t *testing.T) {
|
||||
t.Log(fsutils.CheckDiskIsFast())
|
||||
}
|
||||
94
EdgeNode/internal/utils/fs/file.go
Normal file
94
EdgeNode/internal/utils/fs/file.go
Normal file
@@ -0,0 +1,94 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import "os"
|
||||
|
||||
const FlagRead = 0x1
|
||||
const FlagWrite = 0x2
|
||||
|
||||
type File struct {
|
||||
rawFile *os.File
|
||||
readonly bool
|
||||
}
|
||||
|
||||
func NewFile(rawFile *os.File, flags int) *File {
|
||||
return &File{
|
||||
rawFile: rawFile,
|
||||
readonly: flags&FlagRead == FlagRead,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *File) Name() string {
|
||||
return this.rawFile.Name()
|
||||
}
|
||||
|
||||
func (this *File) Fd() uintptr {
|
||||
return this.rawFile.Fd()
|
||||
}
|
||||
|
||||
func (this *File) Raw() *os.File {
|
||||
return this.rawFile
|
||||
}
|
||||
|
||||
func (this *File) Stat() (os.FileInfo, error) {
|
||||
return this.rawFile.Stat()
|
||||
}
|
||||
|
||||
func (this *File) Seek(offset int64, whence int) (ret int64, err error) {
|
||||
ret, err = this.rawFile.Seek(offset, whence)
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) Read(b []byte) (n int, err error) {
|
||||
ReaderLimiter.Ack()
|
||||
n, err = this.rawFile.Read(b)
|
||||
ReaderLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) ReadAt(b []byte, off int64) (n int, err error) {
|
||||
ReaderLimiter.Ack()
|
||||
n, err = this.rawFile.ReadAt(b, off)
|
||||
ReaderLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) Write(b []byte) (n int, err error) {
|
||||
WriterLimiter.Ack()
|
||||
n, err = this.rawFile.Write(b)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) WriteAt(b []byte, off int64) (n int, err error) {
|
||||
WriterLimiter.Ack()
|
||||
n, err = this.rawFile.WriteAt(b, off)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) Sync() (err error) {
|
||||
WriterLimiter.Ack()
|
||||
err = this.rawFile.Sync()
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) Truncate(size int64) (err error) {
|
||||
WriterLimiter.Ack()
|
||||
err = this.rawFile.Truncate(size)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *File) Close() (err error) {
|
||||
if !this.readonly {
|
||||
WriterLimiter.Ack()
|
||||
}
|
||||
err = this.rawFile.Close()
|
||||
if !this.readonly {
|
||||
WriterLimiter.Release()
|
||||
}
|
||||
return
|
||||
}
|
||||
16
EdgeNode/internal/utils/fs/file_test.go
Normal file
16
EdgeNode/internal/utils/fs/file_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileFlags(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(fsutils.FlagRead&fsutils.FlagRead == fsutils.FlagRead)
|
||||
a.IsTrue(fsutils.FlagWrite&fsutils.FlagWrite != fsutils.FlagRead)
|
||||
a.IsTrue((fsutils.FlagWrite|fsutils.FlagRead)&fsutils.FlagRead == fsutils.FlagRead)
|
||||
}
|
||||
100
EdgeNode/internal/utils/fs/limiter.go
Normal file
100
EdgeNode/internal/utils/fs/limiter.go
Normal file
@@ -0,0 +1,100 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
var maxThreads = runtime.NumCPU()
|
||||
var WriterLimiter = NewLimiter(max(maxThreads*16, 32))
|
||||
var ReaderLimiter = NewLimiter(max(maxThreads*16, 32))
|
||||
|
||||
type Limiter struct {
|
||||
threads chan struct{}
|
||||
count int
|
||||
countDefault int
|
||||
timers chan *time.Timer
|
||||
}
|
||||
|
||||
func NewLimiter(threads int) *Limiter {
|
||||
if threads < 32 {
|
||||
threads = 32
|
||||
}
|
||||
if threads > 1024 {
|
||||
threads = 1024
|
||||
}
|
||||
|
||||
var threadsChan = make(chan struct{}, threads)
|
||||
for i := 0; i < threads; i++ {
|
||||
threadsChan <- struct{}{}
|
||||
}
|
||||
|
||||
return &Limiter{
|
||||
countDefault: threads,
|
||||
count: threads,
|
||||
threads: threadsChan,
|
||||
timers: make(chan *time.Timer, 4096),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Limiter) SetThreads(newThreads int) {
|
||||
if newThreads <= 0 {
|
||||
newThreads = this.countDefault
|
||||
}
|
||||
|
||||
if newThreads != this.count {
|
||||
var threadsChan = make(chan struct{}, newThreads)
|
||||
for i := 0; i < newThreads; i++ {
|
||||
threadsChan <- struct{}{}
|
||||
}
|
||||
|
||||
this.threads = threadsChan
|
||||
this.count = newThreads
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Limiter) Ack() {
|
||||
<-this.threads
|
||||
}
|
||||
|
||||
func (this *Limiter) TryAck() bool {
|
||||
const timeoutDuration = 500 * time.Millisecond
|
||||
|
||||
var timeout *time.Timer
|
||||
select {
|
||||
case timeout = <-this.timers:
|
||||
timeout.Reset(timeoutDuration)
|
||||
default:
|
||||
timeout = time.NewTimer(timeoutDuration)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
timeout.Stop()
|
||||
|
||||
select {
|
||||
case this.timers <- timeout:
|
||||
default:
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-this.threads:
|
||||
return true
|
||||
case <-timeout.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Limiter) Release() {
|
||||
select {
|
||||
case this.threads <- struct{}{}:
|
||||
default:
|
||||
// 由于容量可能有变化,这里忽略多余的thread
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Limiter) FreeThreads() int {
|
||||
return len(this.threads)
|
||||
}
|
||||
123
EdgeNode/internal/utils/fs/limiter_test.go
Normal file
123
EdgeNode/internal/utils/fs/limiter_test.go
Normal file
@@ -0,0 +1,123 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLimiter_SetThreads(t *testing.T) {
|
||||
var limiter = fsutils.NewLimiter(4)
|
||||
|
||||
var concurrent = 1024
|
||||
|
||||
var wg = sync.WaitGroup{}
|
||||
wg.Add(concurrent)
|
||||
|
||||
for i := 0; i < concurrent; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
limiter.SetThreads(rand.Int() % 128)
|
||||
limiter.TryAck()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestLimiter_Ack(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var limiter = fsutils.NewLimiter(4)
|
||||
a.IsTrue(limiter.FreeThreads() == 4)
|
||||
limiter.Ack()
|
||||
a.IsTrue(limiter.FreeThreads() == 3)
|
||||
limiter.Ack()
|
||||
a.IsTrue(limiter.FreeThreads() == 2)
|
||||
limiter.Release()
|
||||
a.IsTrue(limiter.FreeThreads() == 3)
|
||||
limiter.Release()
|
||||
a.IsTrue(limiter.FreeThreads() == 4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimiter_TryAck(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var limiter = fsutils.NewLimiter(4)
|
||||
var count = limiter.FreeThreads()
|
||||
a.IsTrue(count == 4)
|
||||
for i := 0; i < count; i++ {
|
||||
limiter.Ack()
|
||||
}
|
||||
a.IsTrue(limiter.FreeThreads() == 0)
|
||||
a.IsFalse(limiter.TryAck())
|
||||
a.IsTrue(limiter.FreeThreads() == 0)
|
||||
}
|
||||
|
||||
{
|
||||
var limiter = fsutils.NewLimiter(4)
|
||||
var count = limiter.FreeThreads()
|
||||
a.IsTrue(count == 4)
|
||||
for i := 0; i < count-1; i++ {
|
||||
limiter.Ack()
|
||||
}
|
||||
a.IsTrue(limiter.FreeThreads() == 1)
|
||||
a.IsTrue(limiter.TryAck())
|
||||
a.IsTrue(limiter.FreeThreads() == 0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimiter_TryAck2(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var limiter = fsutils.NewLimiter(4)
|
||||
var count = limiter.FreeThreads()
|
||||
a.IsTrue(count == 4)
|
||||
for i := 0; i < count-1; i++ {
|
||||
limiter.Ack()
|
||||
}
|
||||
a.IsTrue(limiter.FreeThreads() == 1)
|
||||
a.IsTrue(limiter.TryAck())
|
||||
a.IsFalse(limiter.TryAck())
|
||||
a.IsFalse(limiter.TryAck())
|
||||
|
||||
limiter.Release()
|
||||
a.IsTrue(limiter.TryAck())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLimiter_Timout(t *testing.T) {
|
||||
var timeout = time.NewTimer(100 * time.Millisecond)
|
||||
|
||||
var r = make(chan bool, 1)
|
||||
r <- true
|
||||
|
||||
var before = time.Now()
|
||||
select {
|
||||
case <-r:
|
||||
case <-timeout.C:
|
||||
}
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
timeout.Stop()
|
||||
|
||||
before = time.Now()
|
||||
timeout.Reset(100 * time.Millisecond)
|
||||
<-timeout.C
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
82
EdgeNode/internal/utils/fs/locker.go
Normal file
82
EdgeNode/internal/utils/fs/locker.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Locker struct {
|
||||
path string
|
||||
fp *os.File
|
||||
}
|
||||
|
||||
func NewLocker(path string) *Locker {
|
||||
return &Locker{
|
||||
path: path + ".lock",
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Locker) TryLock() (ok bool, err error) {
|
||||
if this.fp == nil {
|
||||
fp, err := os.OpenFile(this.path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
this.fp = fp
|
||||
}
|
||||
return this.tryLock()
|
||||
}
|
||||
|
||||
func (this *Locker) Lock() error {
|
||||
if this.fp == nil {
|
||||
fp, err := os.OpenFile(this.path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.fp = fp
|
||||
}
|
||||
|
||||
for {
|
||||
b, err := this.tryLock()
|
||||
if err != nil {
|
||||
_ = this.fp.Close()
|
||||
return err
|
||||
}
|
||||
if b {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Locker) Release() error {
|
||||
err := this.fp.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.fp = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Locker) tryLock() (ok bool, err error) {
|
||||
err = syscall.Flock(int(this.fp.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
errno, isErrNo := err.(syscall.Errno)
|
||||
if !isErrNo {
|
||||
return
|
||||
}
|
||||
|
||||
if !errno.Temporary() {
|
||||
return
|
||||
}
|
||||
|
||||
err = nil // 不提示错误
|
||||
|
||||
return
|
||||
}
|
||||
24
EdgeNode/internal/utils/fs/locker_test.go
Normal file
24
EdgeNode/internal/utils/fs/locker_test.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocker_Lock(t *testing.T) {
|
||||
var path = "/tmp/file-test"
|
||||
var locker = fsutils.NewLocker(path)
|
||||
err := locker.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = locker.Release()
|
||||
|
||||
var locker2 = fsutils.NewLocker(path)
|
||||
err = locker2.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
68
EdgeNode/internal/utils/fs/os.go
Normal file
68
EdgeNode/internal/utils/fs/os.go
Normal file
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func Remove(filename string) (err error) {
|
||||
WriterLimiter.Ack()
|
||||
err = os.Remove(filename)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func Rename(oldPath string, newPath string) (err error) {
|
||||
WriterLimiter.Ack()
|
||||
err = os.Rename(oldPath, newPath)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func ReadFile(filename string) (data []byte, err error) {
|
||||
ReaderLimiter.Ack()
|
||||
data, err = os.ReadFile(filename)
|
||||
ReaderLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
|
||||
WriterLimiter.Ack()
|
||||
err = os.WriteFile(filename, data, perm)
|
||||
WriterLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
func OpenFile(name string, flag int, perm os.FileMode) (f *os.File, err error) {
|
||||
if flag&os.O_RDONLY == os.O_RDONLY {
|
||||
ReaderLimiter.Ack()
|
||||
}
|
||||
|
||||
f, err = os.OpenFile(name, flag, perm)
|
||||
|
||||
if flag&os.O_RDONLY == os.O_RDONLY {
|
||||
ReaderLimiter.Release()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func Open(name string) (f *os.File, err error) {
|
||||
ReaderLimiter.Ack()
|
||||
f, err = os.Open(name)
|
||||
ReaderLimiter.Release()
|
||||
return
|
||||
}
|
||||
|
||||
// ExistFile 检查文件是否存在
|
||||
func ExistFile(path string) (bool, error) {
|
||||
stat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return !stat.IsDir(), nil
|
||||
}
|
||||
38
EdgeNode/internal/utils/fs/os_test.go
Normal file
38
EdgeNode/internal/utils/fs/os_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenFile(t *testing.T) {
|
||||
f, err := fsutils.OpenFile("./os_test.go", os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = f.Close()
|
||||
}
|
||||
|
||||
func TestExistFile(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
b, err := fsutils.ExistFile("./os_test.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsTrue(b)
|
||||
}
|
||||
|
||||
{
|
||||
b, err := fsutils.ExistFile("./os_test2.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.IsFalse(b)
|
||||
}
|
||||
}
|
||||
184
EdgeNode/internal/utils/fs/ssd.go
Normal file
184
EdgeNode/internal/utils/fs/ssd.go
Normal file
@@ -0,0 +1,184 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// StorageType 存储类型
|
||||
type StorageType int
|
||||
|
||||
const (
|
||||
StorageTypeUnknown StorageType = 0
|
||||
StorageTypeSSD StorageType = 1
|
||||
StorageTypeHDD StorageType = 2
|
||||
)
|
||||
|
||||
var (
|
||||
ssdCache = map[string]StorageType{} // path => StorageType
|
||||
ssdCacheLock sync.RWMutex
|
||||
)
|
||||
|
||||
// IsSSD 检测指定路径是否为 SSD
|
||||
// 方法:
|
||||
// 1. 检查 /sys/block/*/queue/rotational(Linux)
|
||||
// 2. 使用 lsblk 命令(Linux)
|
||||
// 3. 检查设备名称(nvme, ssd 等关键词)
|
||||
func IsSSD(path string) (bool, error) {
|
||||
storageType, err := DetectStorageType(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return storageType == StorageTypeSSD, nil
|
||||
}
|
||||
|
||||
// DetectStorageType 检测存储类型
|
||||
func DetectStorageType(path string) (StorageType, error) {
|
||||
// 检查缓存
|
||||
ssdCacheLock.RLock()
|
||||
if cachedType, ok := ssdCache[path]; ok {
|
||||
ssdCacheLock.RUnlock()
|
||||
return cachedType, nil
|
||||
}
|
||||
ssdCacheLock.RUnlock()
|
||||
|
||||
// 获取绝对路径
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return StorageTypeUnknown, err
|
||||
}
|
||||
|
||||
// 获取设备路径
|
||||
devicePath, err := getDevicePath(absPath)
|
||||
if err != nil {
|
||||
return StorageTypeUnknown, err
|
||||
}
|
||||
|
||||
// 方法 1:检查 /sys/block/*/queue/rotational(Linux)
|
||||
if isSSD, err := checkRotational(devicePath); err == nil {
|
||||
storageType := StorageTypeHDD
|
||||
if isSSD {
|
||||
storageType = StorageTypeSSD
|
||||
}
|
||||
ssdCacheLock.Lock()
|
||||
ssdCache[path] = storageType
|
||||
ssdCacheLock.Unlock()
|
||||
return storageType, nil
|
||||
}
|
||||
|
||||
// 方法 2:使用 lsblk 命令(Linux)
|
||||
if isSSD, err := checkWithLsblk(devicePath); err == nil {
|
||||
storageType := StorageTypeHDD
|
||||
if isSSD {
|
||||
storageType = StorageTypeSSD
|
||||
}
|
||||
ssdCacheLock.Lock()
|
||||
ssdCache[path] = storageType
|
||||
ssdCacheLock.Unlock()
|
||||
return storageType, nil
|
||||
}
|
||||
|
||||
// 方法 3:检查设备名称(nvme, ssd 等关键词)
|
||||
if isSSD := checkDeviceName(devicePath); isSSD {
|
||||
ssdCacheLock.Lock()
|
||||
ssdCache[path] = StorageTypeSSD
|
||||
ssdCacheLock.Unlock()
|
||||
return StorageTypeSSD, nil
|
||||
}
|
||||
|
||||
// 默认返回未知
|
||||
ssdCacheLock.Lock()
|
||||
ssdCache[path] = StorageTypeUnknown
|
||||
ssdCacheLock.Unlock()
|
||||
return StorageTypeUnknown, nil
|
||||
}
|
||||
|
||||
// getDevicePath 获取设备路径
|
||||
func getDevicePath(path string) (string, error) {
|
||||
// 从路径提取设备名(简化实现)
|
||||
// 实际应该通过 stat 获取设备信息
|
||||
// 这里使用路径的第一级目录作为设备标识
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
if len(parts) > 0 {
|
||||
return parts[0], nil
|
||||
}
|
||||
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
|
||||
// checkRotational 检查 /sys/block/*/queue/rotational
|
||||
func checkRotational(devicePath string) (bool, error) {
|
||||
// 提取设备名(如 sda, nvme0n1)
|
||||
deviceName := extractDeviceName(devicePath)
|
||||
if len(deviceName) == 0 {
|
||||
return false, os.ErrNotExist
|
||||
}
|
||||
|
||||
// 检查 /sys/block/{device}/queue/rotational
|
||||
rotationalPath := "/sys/block/" + deviceName + "/queue/rotational"
|
||||
data, err := os.ReadFile(rotationalPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// 0 表示 SSD,1 表示 HDD
|
||||
value := strings.TrimSpace(string(data))
|
||||
return value == "0", nil
|
||||
}
|
||||
|
||||
// checkWithLsblk 使用 lsblk 命令检查
|
||||
func checkWithLsblk(devicePath string) (bool, error) {
|
||||
cmd := exec.Command("lsblk", "-d", "-o", "name,rota", "-n")
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
lines := strings.Split(string(output), "\n")
|
||||
deviceName := extractDeviceName(devicePath)
|
||||
for _, line := range lines {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) >= 2 && fields[0] == deviceName {
|
||||
// rota=0 表示 SSD,rota=1 表示 HDD
|
||||
return fields[1] == "0", nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, os.ErrNotExist
|
||||
}
|
||||
|
||||
// checkDeviceName 检查设备名称
|
||||
func checkDeviceName(devicePath string) bool {
|
||||
deviceName := strings.ToLower(extractDeviceName(devicePath))
|
||||
|
||||
// 检查常见 SSD 关键词
|
||||
ssdKeywords := []string{"nvme", "ssd", "flash"}
|
||||
for _, keyword := range ssdKeywords {
|
||||
if strings.Contains(deviceName, keyword) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// extractDeviceName 从路径提取设备名
|
||||
func extractDeviceName(path string) string {
|
||||
// 简化实现:从路径中提取可能的设备名
|
||||
// 实际应该通过更可靠的方法获取
|
||||
parts := strings.Split(path, "/")
|
||||
for _, part := range parts {
|
||||
if len(part) > 0 && (strings.HasPrefix(part, "sd") ||
|
||||
strings.HasPrefix(part, "nvme") ||
|
||||
strings.HasPrefix(part, "hd") ||
|
||||
strings.HasPrefix(part, "vd")) {
|
||||
return part
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
81
EdgeNode/internal/utils/fs/stat.go
Normal file
81
EdgeNode/internal/utils/fs/stat.go
Normal file
@@ -0,0 +1,81 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"golang.org/x/sys/unix"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// StatDevice device contains the path
|
||||
func StatDevice(path string) (*StatResult, error) {
|
||||
var stat = &unix.Statfs_t{}
|
||||
err := unix.Statfs(path, stat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewStatResult(stat), nil
|
||||
}
|
||||
|
||||
var locker = &sync.RWMutex{}
|
||||
var cacheMap = map[string]*StatResult{} // path => StatResult
|
||||
|
||||
const cacheLife = 3 // seconds
|
||||
|
||||
// StatDeviceCache stat device with cache
|
||||
func StatDeviceCache(path string) (*StatResult, error) {
|
||||
locker.RLock()
|
||||
stat, ok := cacheMap[path]
|
||||
if ok && stat.updatedAt >= fasttime.Now().Unix()-cacheLife {
|
||||
locker.RUnlock()
|
||||
return stat, nil
|
||||
}
|
||||
locker.RUnlock()
|
||||
|
||||
locker.Lock()
|
||||
defer locker.Unlock()
|
||||
|
||||
stat, err := StatDevice(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cacheMap[path] = stat
|
||||
return stat, nil
|
||||
}
|
||||
|
||||
type StatResult struct {
|
||||
rawStat *unix.Statfs_t
|
||||
blockSize uint64
|
||||
|
||||
updatedAt int64
|
||||
}
|
||||
|
||||
func NewStatResult(rawStat *unix.Statfs_t) *StatResult {
|
||||
var blockSize = rawStat.Bsize
|
||||
if blockSize < 0 {
|
||||
blockSize = 0
|
||||
}
|
||||
|
||||
return &StatResult{
|
||||
rawStat: rawStat,
|
||||
blockSize: uint64(blockSize),
|
||||
updatedAt: fasttime.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *StatResult) FreeSize() uint64 {
|
||||
return this.rawStat.Bfree * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) TotalSize() uint64 {
|
||||
return this.rawStat.Blocks * this.blockSize
|
||||
}
|
||||
|
||||
func (this *StatResult) UsedSize() uint64 {
|
||||
if this.rawStat.Bfree <= this.rawStat.Blocks {
|
||||
return (this.rawStat.Blocks - this.rawStat.Bfree) * this.blockSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
69
EdgeNode/internal/utils/fs/stat_test.go
Normal file
69
EdgeNode/internal/utils/fs/stat_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStat(t *testing.T) {
|
||||
stat, err := fsutils.StatDevice("/usr/local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("free:", stat.FreeSize()/(1<<30), "total:", stat.TotalSize()/(1<<30), "used:", stat.UsedSize()/(1<<30))
|
||||
}
|
||||
|
||||
func TestStatCache(t *testing.T) {
|
||||
for i := 0; i < 10; i++ {
|
||||
stat, err := fsutils.StatDeviceCache("/usr/local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("free:", stat.FreeSize()/(1<<30), "total:", stat.TotalSize()/(1<<30), "used:", stat.UsedSize()/(1<<30))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrent(t *testing.T) {
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
var count = 10000
|
||||
var wg = sync.WaitGroup{}
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
_, _ = fsutils.StatDevice("/usr/local")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func BenchmarkStatDevice(b *testing.B) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, err := fsutils.StatDevice("/usr/local")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkStatCacheDevice(b *testing.B) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, err := fsutils.StatDeviceCache("/usr/local")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
125
EdgeNode/internal/utils/fs/status.go
Normal file
125
EdgeNode/internal/utils/fs/status.go
Normal file
@@ -0,0 +1,125 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Speed int
|
||||
|
||||
func (this Speed) String() string {
|
||||
switch this {
|
||||
case SpeedExtremelyFast:
|
||||
return "extremely fast"
|
||||
case SpeedFast:
|
||||
return "fast"
|
||||
case SpeedLow:
|
||||
return "low"
|
||||
case SpeedExtremelySlow:
|
||||
return "extremely slow"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
const (
|
||||
SpeedExtremelyFast Speed = 1
|
||||
SpeedFast Speed = 2
|
||||
SpeedLow Speed = 3
|
||||
SpeedExtremelySlow Speed = 4
|
||||
)
|
||||
|
||||
var (
|
||||
DiskSpeed = SpeedLow
|
||||
DiskSpeedMB float64
|
||||
)
|
||||
|
||||
var IsInHighLoad = false
|
||||
var IsInExtremelyHighLoad = false
|
||||
|
||||
const (
|
||||
highLoad1Threshold = 20
|
||||
extremelyHighLoad1Threshold = 40
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
return
|
||||
}
|
||||
|
||||
// test disk
|
||||
goman.New(func() {
|
||||
// load last result from local disk
|
||||
var countTests int
|
||||
cacheData, cacheErr := os.ReadFile(Tea.Root + "/data/" + diskSpeedDataFile)
|
||||
if cacheErr == nil {
|
||||
var cache = &DiskSpeedCache{}
|
||||
err := json.Unmarshal(cacheData, cache)
|
||||
if err == nil && cache.SpeedMB > 0 {
|
||||
DiskSpeedMB = cache.SpeedMB
|
||||
DiskSpeed = cache.Speed
|
||||
countTests = cache.CountTests
|
||||
}
|
||||
}
|
||||
|
||||
if countTests < 12 {
|
||||
// initial check
|
||||
_, _, _ = CheckDiskIsFast()
|
||||
|
||||
// check every one hour
|
||||
var ticker = time.NewTicker(1 * time.Hour)
|
||||
var count = 0
|
||||
for range ticker.C {
|
||||
_, _, err := CheckDiskIsFast()
|
||||
if err == nil {
|
||||
count++
|
||||
if count > 24 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// check high load
|
||||
goman.New(func() {
|
||||
var ticker = time.NewTicker(5 * time.Second)
|
||||
for range ticker.C {
|
||||
stat, _ := load.Avg()
|
||||
IsInExtremelyHighLoad = stat != nil && stat.Load1 > extremelyHighLoad1Threshold
|
||||
IsInHighLoad = stat != nil && stat.Load1 > highLoad1Threshold && !DiskIsFast()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func DiskIsFast() bool {
|
||||
return DiskSpeed == SpeedExtremelyFast || DiskSpeed == SpeedFast
|
||||
}
|
||||
|
||||
func DiskIsExtremelyFast() bool {
|
||||
// 在开发环境下返回false,以便于测试
|
||||
if Tea.IsTesting() {
|
||||
return false
|
||||
}
|
||||
return DiskSpeed == SpeedExtremelyFast
|
||||
}
|
||||
|
||||
// WaitLoad wait system load to downgrade
|
||||
func WaitLoad(maxLoad float64, maxLoops int, delay time.Duration) {
|
||||
for i := 0; i < maxLoops; i++ {
|
||||
stat, err := load.Avg()
|
||||
if err == nil {
|
||||
if stat.Load1 > maxLoad {
|
||||
time.Sleep(delay)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
13
EdgeNode/internal/utils/fs/status_test.go
Normal file
13
EdgeNode/internal/utils/fs/status_test.go
Normal file
@@ -0,0 +1,13 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package fsutils_test
|
||||
|
||||
import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWaitLoad(t *testing.T) {
|
||||
fsutils.WaitLoad(100, 5, 1*time.Minute)
|
||||
}
|
||||
Reference in New Issue
Block a user