Initial commit (code only without large binaries)

EdgeNode/internal/utils/mmap/flags_linux.go

@@ -0,0 +1,10 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build linux
+
+package mmap
+
+import "syscall"
+
+// 不要使用 MAP_POPULATE，会导致map成本大幅上升
+
+const MMAPReadFlags = syscall.MAP_PRIVATE

EdgeNode/internal/utils/mmap/flags_others.go

@@ -0,0 +1,8 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build !linux && !windows
+
+package mmap
+
+import "syscall"
+
+const MMAPReadFlags = syscall.MAP_PRIVATE

EdgeNode/internal/utils/mmap/mmap_ext_unix.go

@@ -0,0 +1,95 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus && !windows
+
+package mmap
+
+import (
+	"fmt"
+	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
+	"io"
+	"os"
+	"runtime"
+	"sync/atomic"
+	"syscall"
+)
+
+var totalMMAPFileSize int64
+
+func TotalMMAPFileSize() int64 {
+	return atomic.LoadInt64(&totalMMAPFileSize)
+}
+
+// OpenFile memory-maps the file pointer for reading.
+func OpenFile(f *os.File, size int64) (r *ReaderAt, err error) {
+	// update stat
+	atomic.AddInt64(&totalMMAPFileSize, size)
+
+	defer func() {
+		if r == nil {
+			atomic.AddInt64(&totalMMAPFileSize, -size)
+		}
+	}()
+
+	defer func() {
+		_ = f.Close()
+	}()
+
+	if size == 0 {
+		return &ReaderAt{
+			data: make([]byte, 0),
+		}, nil
+	}
+	if size < 0 {
+		return nil, fmt.Errorf("mmap: file %q has negative size", f.Name())
+	}
+	if size != int64(int(size)) {
+		return nil, fmt.Errorf("mmap: file %q is too large", f.Name())
+	}
+
+	fsutils.ReaderLimiter.Ack()
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, MMAPReadFlags)
+	fsutils.ReaderLimiter.Release()
+	if err != nil {
+		return nil, err
+	}
+	r = &ReaderAt{
+		data: data,
+		size: size,
+	}
+	runtime.SetFinalizer(r, (*ReaderAt).Close)
+	return r, nil
+}
+
+func (r *ReaderAt) Write(writer io.Writer, offset int) (int, error) {
+	var l = len(r.data)
+	const segmentSize = 16 << 10
+
+	if l-offset <= segmentSize {
+		return writer.Write(r.data[offset:])
+	}
+
+	var n int
+
+	for {
+		var n1 int
+		var err error
+		fsutils.ReaderLimiter.Ack()
+		if offset+segmentSize <= l {
+			n1, err = writer.Write(r.data[offset : offset+segmentSize])
+		} else {
+			n1, err = writer.Write(r.data[offset:])
+		}
+		fsutils.ReaderLimiter.Release()
+		n += n1
+		offset += n1
+		if err != nil {
+			return n, err
+		}
+
+		if offset >= l {
+			break
+		}
+	}
+
+	return n, nil
+}

EdgeNode/internal/utils/mmap/mmap_unix.go

@@ -0,0 +1,146 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// modified by goedge cdn
+
+//go:build plus && (linux || darwin)
+
+// Package mmap provides a way to memory-map a file.
+package mmap
+
+import (
+	"errors"
+	"fmt"
+	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
+	"io"
+	"runtime"
+	"sync/atomic"
+	"syscall"
+)
+
+// debug is whether to print debugging messages for manual testing.
+//
+// The runtime.SetFinalizer documentation says that, "The finalizer for x is
+// scheduled to run at some arbitrary time after x becomes unreachable. There
+// is no guarantee that finalizers will run before a program exits", so we
+// cannot automatically test that the finalizer runs. Instead, set this to true
+// when running the manual test.
+const debug = false
+
+// ReaderAt reads a memory-mapped file.
+//
+// Like any io.ReaderAt, clients can execute parallel ReadAt calls, but it is
+// not safe to call Close and reading methods concurrently.
+type ReaderAt struct {
+	data []byte
+	size int64
+}
+
+// Close closes the reader.
+func (r *ReaderAt) Close() error {
+	if r.data == nil {
+		return nil
+	} else if len(r.data) == 0 {
+		r.data = nil
+		return nil
+	}
+	data := r.data
+	r.data = nil
+	if debug {
+		var p *byte
+		if len(data) != 0 {
+			p = &data[0]
+		}
+		println("munmap", r, p)
+	}
+	runtime.SetFinalizer(r, nil)
+
+	// update stat
+	atomic.AddInt64(&totalMMAPFileSize, -r.size)
+
+	fsutils.ReaderLimiter.Ack()
+	err := syscall.Munmap(data)
+	fsutils.ReaderLimiter.Release()
+	return err
+}
+
+// Len returns the length of the underlying memory-mapped file.
+func (r *ReaderAt) Len() int {
+	return len(r.data)
+}
+
+// At returns the byte at index i.
+func (r *ReaderAt) At(i int) byte {
+	return r.data[i]
+}
+
+// ReadAt implements the io.ReaderAt interface.
+func (r *ReaderAt) ReadAt(p []byte, off int64) (int, error) {
+	if r.data == nil {
+		return 0, errors.New("mmap: closed")
+	}
+	if off < 0 || int64(len(r.data)) < off {
+		return 0, fmt.Errorf("mmap: invalid ReadAt offset %d", off)
+	}
+	fsutils.ReaderLimiter.Ack()
+	n := copy(p, r.data[off:])
+	fsutils.ReaderLimiter.Release()
+	if n < len(p) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+// Open memory-maps the named file for reading.
+func Open(filename string) (*ReaderAt, error) {
+	f, err := fsutils.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	size := fi.Size()
+	if size == 0 {
+		// Treat (size == 0) as a special case, avoiding the syscall, since
+		// "man 2 mmap" says "the length... must be greater than 0".
+		//
+		// As we do not call syscall.Mmap, there is no need to call
+		// runtime.SetFinalizer to enforce a balancing syscall.Munmap.
+		return &ReaderAt{
+			data: make([]byte, 0),
+		}, nil
+	}
+	if size < 0 {
+		return nil, fmt.Errorf("mmap: file %q has negative size", filename)
+	}
+	if size != int64(int(size)) {
+		return nil, fmt.Errorf("mmap: file %q is too large", filename)
+	}
+
+	fsutils.ReaderLimiter.Ack()
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, MMAPReadFlags)
+	fsutils.ReaderLimiter.Release()
+	if err != nil {
+		return nil, err
+	}
+	var r = &ReaderAt{
+		data: data,
+		size: size,
+	}
+	if debug {
+		var p *byte
+		if len(data) != 0 {
+			p = &data[0]
+		}
+		println("mmap", r, p)
+	}
+	runtime.SetFinalizer(r, (*ReaderAt).Close)
+	return r, nil
+}

EdgeNode/internal/utils/mmap/shared_mmap.go

@@ -0,0 +1,89 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package mmap
+
+import (
+	"io"
+	"os"
+)
+
+type SharedMMAP struct {
+	filename  string
+	stat      os.FileInfo
+	rawReader *ReaderAt
+	countRefs int32
+	mod       uint64
+}
+
+func NewSharedMMAP(filename string, stat os.FileInfo, reader *ReaderAt, mod uint64) *SharedMMAP {
+	return &SharedMMAP{
+		filename:  filename,
+		stat:      stat,
+		rawReader: reader,
+		countRefs: 1,
+		mod:       mod,
+	}
+}
+
+func (this *SharedMMAP) ReadAt(p []byte, off int64) (n int, err error) {
+	n, err = this.rawReader.ReadAt(p, off)
+	return
+}
+
+func (this *SharedMMAP) Size() int64 {
+	return this.stat.Size()
+}
+
+func (this *SharedMMAP) Stat() os.FileInfo {
+	return this.stat
+}
+
+func (this *SharedMMAP) Name() string {
+	return this.filename
+}
+
+// AddRef increase one reference
+func (this *SharedMMAP) AddRef() (ok bool) {
+	mmapMapMu[this.mod].Lock()
+	if this.countRefs > 0 { // not closing
+		this.countRefs++
+		ok = true
+	}
+	mmapMapMu[this.mod].Unlock()
+
+	return
+}
+
+func (this *SharedMMAP) Write(writer io.Writer, offset int) (int, error) {
+	return this.rawReader.Write(writer, offset)
+}
+
+// Close release one reference
+// 单个引用 必须 只能close一次，防止提前close
+func (this *SharedMMAP) Close() error {
+	if !enableShareMode {
+		return this.rawReader.Close()
+	}
+
+	mmapMapMu[this.mod].Lock()
+
+	this.countRefs--
+	var shouldClose = this.countRefs == 0
+
+	if shouldClose {
+		var mmapMap = mmapMaps[this.mod]
+		delete(mmapMap[this.filename], this)
+		if len(mmapMap[this.filename]) == 0 {
+			delete(mmapMap, this.filename)
+		}
+	}
+
+	mmapMapMu[this.mod].Unlock()
+
+	if shouldClose {
+		return this.rawReader.Close()
+	}
+
+	return nil
+}

EdgeNode/internal/utils/mmap/shared_mmap_test.go

@@ -0,0 +1,252 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package mmap_test
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/bytepool"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/mmap"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"io"
+	"os"
+	"sync"
+	"testing"
+)
+
+func TestOpenSharedMMAP(t *testing.T) {
+	for i := 0; i < 3; i++ {
+		func() {
+			fp, err := mmap.OpenSharedMMAP("./shared_mmap.go")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			defer func() {
+				_ = fp.Close()
+			}()
+
+			var buf = make([]byte, fp.Size())
+			n, err := fp.ReadAt(buf, 0)
+			if err != nil && err != io.EOF {
+				t.Fatal(err)
+			}
+			t.Log(len(buf[:n]), fp.Stat())
+		}()
+	}
+}
+
+func TestOpenSharedMMAP_Memory(t *testing.T) {
+	var isSingleTesting = testutils.IsSingleTesting()
+
+	var count = 10
+	if isSingleTesting {
+		count = 10_000
+	}
+
+	var stat1 = testutils.ReadMemoryStat()
+
+	var fps []*mmap.SharedMMAP
+
+	var buf = make([]byte, 1024)
+	for i := 0; i < count; i++ {
+		//if i > 0 && i%100 == 0 {
+		//	t.Log(i)
+		//}
+		fp, openErr := mmap.OpenSharedMMAP("./shared_mmap.go")
+		if openErr != nil {
+			t.Fatal(openErr)
+		}
+		fps = append(fps, fp)
+
+		var off int64
+		for {
+			n, err := fp.ReadAt(buf, off)
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Fatal(err)
+			}
+			off += int64(n)
+		}
+	}
+
+	var stat2 = testutils.ReadMemoryStat()
+	t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
+
+	for _, fp := range fps {
+		_ = fp.Close()
+	}
+}
+
+func TestOpenSharedMMAP_Concurrent(t *testing.T) {
+	const filename = "./shared_mmap.go"
+
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var rawMdString string
+	{
+		var md = md5.New()
+		md.Write(data)
+		rawMdString = fmt.Sprintf("%x", md.Sum(nil))
+	}
+	t.Log(rawMdString)
+
+	fp, err := mmap.OpenSharedMMAP(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const concurrent = 2048
+	var wg = &sync.WaitGroup{}
+	wg.Add(concurrent)
+
+	for i := 0; i < concurrent; i++ {
+		go func() {
+			defer wg.Done()
+
+			var buf = bytes.NewBuffer(nil)
+			n, readErr := fp.Write(buf, 0)
+			if readErr != nil && readErr != io.EOF {
+				t.Log(readErr)
+				return
+			}
+			var md = md5.New()
+			md.Write(buf.Bytes()[:n])
+			var mdString = fmt.Sprintf("%x", md.Sum(nil))
+			if mdString != rawMdString {
+				t.Log("md not equal")
+				return
+			}
+		}()
+	}
+
+	wg.Wait()
+}
+
+func TestOpenSharedMMAP_Close(t *testing.T) {
+	fp, err := mmap.OpenSharedMMAP("./shared_mmap.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 10; i++ {
+		_ = fp.Close()
+	}
+}
+
+func TestCountSharedMMAP(t *testing.T) {
+	t.Log(mmap.CountSharedMMAP())
+}
+
+func BenchmarkMMAP_Shared(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			func() {
+				reader, err := mmap.OpenSharedMMAP("../kvstore/query.go")
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer func() {
+					_ = reader.Close()
+				}()
+
+				var data = make([]byte, reader.Size())
+				n, readErr := reader.ReadAt(data, 0)
+				if readErr != nil {
+					b.Fatal(readErr)
+				}
+				if int64(n) != reader.Size() {
+					b.Fatal("size not equal")
+				}
+			}()
+		}
+	})
+}
+
+func BenchmarkMMAP_Shared2(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			func() {
+				reader, err := mmap.OpenSharedMMAP("../kvstore/query.go")
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer func() {
+					_ = reader.Close()
+				}()
+
+				var writer = &bytes.Buffer{}
+				n, readErr := reader.Write(writer, 0)
+				if readErr != nil {
+					b.Fatal(readErr)
+				}
+				if int64(n) != reader.Size() {
+					b.Fatal("size not equal")
+				}
+			}()
+		}
+	})
+}
+
+func BenchmarkMMAP_MMAP(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			func() {
+				fp, err := os.Open("../kvstore/query.go")
+				if err != nil {
+					b.Fatal(err)
+				}
+
+				stat, err := fp.Stat()
+				if err != nil {
+					b.Fatal(err)
+				}
+
+				reader, err := mmap.OpenFile(fp, stat.Size())
+				if err != nil {
+					b.Fatal(err)
+				}
+				defer func() {
+					_ = reader.Close()
+				}()
+
+				_, _ = reader.Write(io.Discard, 0)
+			}()
+		}
+	})
+}
+
+func BenchmarkMMAP_FP_Read(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			func() {
+				fp, err := os.Open("../kvstore/query.go")
+				if err != nil {
+					b.Fatal(err)
+				}
+
+				defer func() {
+					_ = fp.Close()
+				}()
+
+				var buf = bytepool.Pool1k.Get()
+				defer bytepool.Pool1k.Put(buf)
+
+				for {
+					_, readErr := fp.Read(buf.Bytes)
+					if readErr != nil {
+						return
+					}
+				}
+			}()
+		}
+	})
+}

EdgeNode/internal/utils/mmap/shared_utils.go

@@ -0,0 +1,93 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package mmap
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
+	"sync"
+)
+
+// TODO 当缓存文件更新时在mmap map中删除对应mmap，以便于及时更新？修改inotify相关代码？
+
+const mmapMuShards = 32
+const enableShareMode = false // 暂时不启用，需要更多时间的测试
+
+var mmapMaps = [mmapMuShards]map[string]map[*SharedMMAP]zero.Zero{} //  [{ filename => { mmap => Zero } }, ...]
+var mmapMapMu = [mmapMuShards]*sync.RWMutex{}
+
+func init() {
+	for i := 0; i < mmapMuShards; i++ {
+		mmapMaps[i] = map[string]map[*SharedMMAP]zero.Zero{}
+		mmapMapMu[i] = &sync.RWMutex{}
+	}
+}
+
+func OpenSharedMMAP(filename string) (*SharedMMAP, error) {
+	var mod = fnv.HashString(filename) % mmapMuShards
+	var mu = mmapMapMu[mod]
+	var m *SharedMMAP
+	var mmapMap map[string]map[*SharedMMAP]zero.Zero
+
+	if enableShareMode {
+		mu.RLock()
+		mmapMap = mmapMaps[mod]
+		bucket, ok := mmapMap[filename]
+		if ok && len(bucket) > 0 {
+			for firstM := range bucket {
+				m = firstM
+				break
+			}
+		}
+		mu.RUnlock()
+		if m != nil && m.AddRef() {
+			return m, nil
+		}
+	}
+
+	fp, err := fsutils.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	stat, err := fp.Stat()
+	if err != nil {
+		_ = fp.Close()
+		return nil, err
+	}
+
+	reader, err := OpenFile(fp, stat.Size())
+	if err != nil {
+		return nil, err
+	}
+
+	m = NewSharedMMAP(filename, stat, reader, mod)
+
+	if enableShareMode {
+		mu.Lock()
+		if mmapMap[filename] == nil {
+			mmapMap[filename] = map[*SharedMMAP]zero.Zero{
+				m: {},
+			}
+		} else {
+			mmapMap[filename][m] = zero.Zero{}
+		}
+		mu.Unlock()
+	}
+
+	return m, nil
+}
+
+func CountSharedMMAP() int {
+	var count int
+	for i := 0; i < mmapMuShards; i++ {
+		mmapMapMu[i].RLock()
+		for _, bucket := range mmapMaps[i] {
+			count += len(bucket)
+		}
+		mmapMapMu[i].RUnlock()
+	}
+	return count
+}