1.4.5.2

EdgeNode/internal/utils/readers/handlers.go

@@ -0,0 +1,3 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers

EdgeNode/internal/utils/readers/reader_base.go

@@ -0,0 +1,6 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+type BaseReader struct {
+}

EdgeNode/internal/utils/readers/reader_bytes_counter.go

@@ -0,0 +1,26 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+import "io"
+
+type BytesCounterReader struct {
+	rawReader io.Reader
+	count     int64
+}
+
+func NewBytesCounterReader(rawReader io.Reader) *BytesCounterReader {
+	return &BytesCounterReader{
+		rawReader: rawReader,
+	}
+}
+
+func (this *BytesCounterReader) Read(p []byte) (n int, err error) {
+	n, err = this.rawReader.Read(p)
+	this.count += int64(n)
+	return
+}
+
+func (this *BytesCounterReader) TotalBytes() int64 {
+	return this.count
+}

EdgeNode/internal/utils/readers/reader_closer_byte_ranges.go

@@ -0,0 +1,157 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+import (
+	"bytes"
+	"github.com/iwind/TeaGo/types"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+	"regexp"
+	"strings"
+)
+
+type OnPartReadHandler func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader)
+
+var contentRangeRegexp = regexp.MustCompile(`^(\d+)-(\d+)/(\d+|\*)`)
+
+type ByteRangesReaderCloser struct {
+	BaseReader
+
+	rawReader io.ReadCloser
+	boundary  string
+
+	mReader *multipart.Reader
+	part    *multipart.Part
+
+	buf   *bytes.Buffer
+	isEOF bool
+
+	onPartReadHandler OnPartReadHandler
+	rangeStart        int64
+	rangeEnd          int64
+	total             int64
+
+	isStarted bool
+	nl        string
+}
+
+func NewByteRangesReaderCloser(reader io.ReadCloser, boundary string) *ByteRangesReaderCloser {
+	return &ByteRangesReaderCloser{
+		rawReader: reader,
+		mReader:   multipart.NewReader(reader, boundary),
+		boundary:  boundary,
+		buf:       &bytes.Buffer{},
+		nl:        "\r\n",
+	}
+}
+
+func (this *ByteRangesReaderCloser) Read(p []byte) (n int, err error) {
+	n, err = this.read(p)
+	return
+}
+
+func (this *ByteRangesReaderCloser) Close() error {
+	return this.rawReader.Close()
+}
+
+func (this *ByteRangesReaderCloser) OnPartRead(handler OnPartReadHandler) {
+	this.onPartReadHandler = handler
+}
+
+func (this *ByteRangesReaderCloser) read(p []byte) (n int, err error) {
+	// read from buffer
+	n, err = this.buf.Read(p)
+	if !this.isEOF {
+		err = nil
+	}
+	if n > 0 {
+		return
+	}
+	if this.isEOF {
+		return
+	}
+
+	if this.part == nil {
+		part, partErr := this.mReader.NextPart()
+		if partErr != nil {
+			if partErr == io.EOF {
+				this.buf.WriteString(this.nl + "--" + this.boundary + "--" + this.nl)
+				this.isEOF = true
+				n, _ = this.buf.Read(p)
+				return
+			}
+
+			return 0, partErr
+		}
+
+		if !this.isStarted {
+			this.isStarted = true
+			this.buf.WriteString("--" + this.boundary + this.nl)
+		} else {
+			this.buf.WriteString(this.nl + "--" + this.boundary + this.nl)
+		}
+
+		// Headers
+		var hasRange = false
+		for k, v := range part.Header {
+			for _, v1 := range v {
+				this.buf.WriteString(k + ": " + v1 + this.nl)
+
+				// parse range
+				if k == "Content-Range" {
+					var bytesPrefix = "bytes "
+					if strings.HasPrefix(v1, bytesPrefix) {
+						var r = v1[len(bytesPrefix):]
+						var matches = contentRangeRegexp.FindStringSubmatch(r)
+						if len(matches) > 2 {
+							var start = types.Int64(matches[1])
+							var end = types.Int64(matches[2])
+							var total int64 = 0
+							if matches[3] != "*" {
+								total = types.Int64(matches[3])
+							}
+							if start <= end {
+								hasRange = true
+								this.rangeStart = start
+								this.rangeEnd = end
+								this.total = total
+							}
+						}
+					}
+				}
+			}
+		}
+
+		if !hasRange {
+			this.rangeStart = -1
+			this.rangeEnd = -1
+		}
+
+		this.buf.WriteString(this.nl)
+		this.part = part
+
+		n, _ = this.buf.Read(p)
+		return
+	}
+
+	n, err = this.part.Read(p)
+
+	if this.onPartReadHandler != nil && n > 0 && this.rangeStart >= 0 && this.rangeEnd >= 0 {
+		this.onPartReadHandler(this.rangeStart, this.rangeEnd, this.total, p[:n], this.part.Header)
+		this.rangeStart += int64(n)
+	}
+
+	if err == io.EOF {
+		this.part = nil
+		err = nil
+
+		// 如果没有读取到内容，则直接跳到下一个Part
+		if n == 0 {
+			return this.read(p)
+		}
+	}
+
+	return
+}

EdgeNode/internal/utils/readers/reader_closer_byte_ranges_test.go

@@ -0,0 +1,51 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers_test
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
+	"io"
+	"net/textproto"
+	"testing"
+)
+
+func TestNewByteRangesReader(t *testing.T) {
+	var boundary = "7143cd51d2ee12a1"
+	var dashBoundary = "--" + boundary
+	var b = bytes.NewReader([]byte(dashBoundary + "\r\nContent-Range: bytes 0-4/36\r\nContent-Type: text/plain\r\n\r\n01234\r\n" + dashBoundary + "\r\nContent-Range: bytes 5-9/36\r\nContent-Type: text/plain\r\n\r\n56789\r\n--" + boundary + "\r\nContent-Range: bytes 10-12/36\r\nContent-Type: text/plain\r\n\r\nabc\r\n" + dashBoundary + "--\r\n"))
+
+	var reader = readers.NewByteRangesReaderCloser(io.NopCloser(b), boundary)
+	var p = make([]byte, 16)
+	for {
+		n, err := reader.Read(p)
+		if n > 0 {
+			fmt.Print(string(p[:n]))
+		}
+		if err != nil {
+			if err != io.EOF {
+				t.Fatal(err)
+			}
+			break
+		}
+	}
+}
+
+func TestByteRangesReader_OnPartRead(t *testing.T) {
+	var boundary = "7143cd51d2ee12a1"
+	var dashBoundary = "--" + boundary
+	var b = bytes.NewReader([]byte(dashBoundary + "\r\nContent-Range: bytes 0-4/36\r\nContent-Type: text/plain\r\n\r\n01234\r\n" + dashBoundary + "\r\nContent-Range: bytes 5-9/36\r\nContent-Type: text/plain\r\n\r\n56789\r\n--" + boundary + "\r\nContent-Range: bytes 10-12/36\r\nContent-Type: text/plain\r\n\r\nabc\r\n" + dashBoundary + "--\r\n"))
+
+	var reader = readers.NewByteRangesReaderCloser(io.NopCloser(b), boundary)
+	reader.OnPartRead(func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader) {
+		t.Log(start, "-", end, "/", total, string(data))
+	})
+	var p = make([]byte, 3)
+	for {
+		_, err := reader.Read(p)
+		if err != nil {
+			break
+		}
+	}
+}

EdgeNode/internal/utils/readers/reader_closer_filter.go

@@ -0,0 +1,42 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+import "io"
+
+type FilterFunc = func(p []byte, readErr error) error
+
+type FilterReaderCloser struct {
+	rawReader io.Reader
+	filters   []FilterFunc
+}
+
+func NewFilterReaderCloser(rawReader io.Reader) *FilterReaderCloser {
+	return &FilterReaderCloser{
+		rawReader: rawReader,
+	}
+}
+
+func (this *FilterReaderCloser) Add(filter FilterFunc) {
+	this.filters = append(this.filters, filter)
+}
+
+func (this *FilterReaderCloser) Read(p []byte) (n int, err error) {
+	n, err = this.rawReader.Read(p)
+	for _, filter := range this.filters {
+		filterErr := filter(p[:n], err)
+		if (err == nil || err != io.EOF) && filterErr != nil {
+			err = filterErr
+			return
+		}
+	}
+	return
+}
+
+func (this *FilterReaderCloser) Close() error {
+	closer, ok := this.rawReader.(io.Closer)
+	if ok {
+		return closer.Close()
+	}
+	return nil
+}

EdgeNode/internal/utils/readers/reader_closer_filter_test.go

@@ -0,0 +1,41 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers_test
+
+import (
+	"bytes"
+	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
+	"testing"
+)
+
+func TestNewFilterReader(t *testing.T) {
+	var reader = readers.NewFilterReaderCloser(bytes.NewBufferString("0123456789"))
+	reader.Add(func(p []byte, err error) error {
+		t.Log("filter1:", string(p), err)
+		return nil
+	})
+	reader.Add(func(p []byte, err error) error {
+		t.Log("filter2:", string(p), err)
+		if string(p) == "345" {
+			return errors.New("end")
+		}
+		return nil
+	})
+	reader.Add(func(p []byte, err error) error {
+		t.Log("filter3:", string(p), err)
+		return nil
+	})
+
+	var buf = make([]byte, 3)
+	for {
+		n, err := reader.Read(buf)
+		if n > 0 {
+			t.Log(string(buf[:n]))
+		}
+		if err != nil {
+			t.Log(err)
+			break
+		}
+	}
+}

EdgeNode/internal/utils/readers/reader_closer_tee.go

@@ -0,0 +1,71 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+import (
+	"io"
+)
+
+type TeeReaderCloser struct {
+	r io.Reader
+	w io.Writer
+
+	onFail func(err error)
+	onEOF  func()
+
+	mustWrite bool
+}
+
+// NewTeeReaderCloser
+// mustWrite - ensure writing MUST be successfully
+func NewTeeReaderCloser(reader io.Reader, writer io.Writer, mustWrite bool) *TeeReaderCloser {
+	return &TeeReaderCloser{
+		r:         reader,
+		w:         writer,
+		mustWrite: mustWrite,
+	}
+}
+
+func (this *TeeReaderCloser) Read(p []byte) (n int, err error) {
+	n, err = this.r.Read(p)
+	if n > 0 {
+		_, wErr := this.w.Write(p[:n])
+		if (err == nil || err == io.EOF) && wErr != nil {
+			if this.mustWrite {
+				err = wErr
+			} else {
+				if this.onFail != nil {
+					this.onFail(wErr)
+				}
+			}
+		}
+	}
+	if err != nil {
+		if err == io.EOF {
+			if this.onEOF != nil {
+				this.onEOF()
+			}
+		} else {
+			if this.onFail != nil {
+				this.onFail(err)
+			}
+		}
+	}
+	return
+}
+
+func (this *TeeReaderCloser) Close() error {
+	r, ok := this.r.(io.Closer)
+	if ok {
+		return r.Close()
+	}
+	return nil
+}
+
+func (this *TeeReaderCloser) OnFail(onFail func(err error)) {
+	this.onFail = onFail
+}
+
+func (this *TeeReaderCloser) OnEOF(onEOF func()) {
+	this.onEOF = onEOF
+}

EdgeNode/internal/utils/readers/reader_print.go

@@ -0,0 +1,28 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package readers
+
+import (
+	"io"
+	"log"
+)
+
+type PrintReader struct {
+	rawReader io.Reader
+	tag       string
+}
+
+func NewPrintReader(rawReader io.Reader, tag string) io.Reader {
+	return &PrintReader{
+		rawReader: rawReader,
+		tag:       tag,
+	}
+}
+
+func (this *PrintReader) Read(p []byte) (n int, err error) {
+	n, err = this.rawReader.Read(p)
+	if n > 0 {
+		log.Println("[" + this.tag + "]" + string(p[:n]))
+	}
+	return
+}

EdgeNode/internal/utils/readers/reader_tee.go

@@ -0,0 +1,52 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package readers
+
+import (
+	"io"
+)
+
+type TeeReader struct {
+	r io.Reader
+	w io.Writer
+
+	onFail func(err error)
+	onEOF  func()
+}
+
+func NewTeeReader(reader io.Reader, writer io.Writer) *TeeReader {
+	return &TeeReader{
+		r: reader,
+		w: writer,
+	}
+}
+
+func (this *TeeReader) Read(p []byte) (n int, err error) {
+	n, err = this.r.Read(p)
+	if n > 0 {
+		_, wErr := this.w.Write(p[:n])
+		if err == nil && wErr != nil {
+			err = wErr
+		}
+	}
+	if err != nil {
+		if err == io.EOF {
+			if this.onEOF != nil {
+				this.onEOF()
+			}
+		} else {
+			if this.onFail != nil {
+				this.onFail(err)
+			}
+		}
+	}
+	return
+}
+
+func (this *TeeReader) OnFail(onFail func(err error)) {
+	this.onFail = onFail
+}
+
+func (this *TeeReader) OnEOF(onEOF func()) {
+	this.onEOF = onEOF
+}

EdgeNode/internal/utils/readers/readers_concurrent_file.go

@@ -0,0 +1,117 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package readers
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
+	"io"
+	"os"
+	"sync"
+)
+
+type subFileReader struct {
+	fp             *os.File
+	notify         chan zero.Zero
+	parentIsClosed bool
+}
+
+func newSubFileReader(fp *os.File) *subFileReader {
+	return &subFileReader{
+		fp:     fp,
+		notify: make(chan zero.Zero, 2),
+	}
+}
+
+func (this *subFileReader) Read(p []byte) (n int, err error) {
+	n, err = this.fp.Read(p)
+	if n > 0 {
+		return n, nil
+	}
+	if err != nil {
+		if this.parentIsClosed {
+			return
+		}
+
+		if err != io.EOF {
+			return
+		}
+
+		// try next read
+		<-this.notify
+		return this.Read(p)
+	}
+	return
+}
+
+func (this *subFileReader) Close() error {
+	close(this.notify)
+	return this.fp.Close()
+}
+
+func (this *subFileReader) NotifyRead() {
+	select {
+	case this.notify <- zero.Zero{}:
+	default:
+	}
+}
+
+func (this *subFileReader) NotifyClose() {
+	select {
+	case this.notify <- zero.Zero{}:
+	default:
+	}
+	this.parentIsClosed = true
+}
+
+type ConcurrentFileReaders struct {
+	filePath string
+	isClosed bool
+
+	subReaders []*subFileReader
+	locker     sync.RWMutex
+}
+
+func NewConcurrentFileReaders(filePath string) *ConcurrentFileReaders {
+	return &ConcurrentFileReaders{
+		filePath: filePath,
+	}
+}
+
+func (this *ConcurrentFileReaders) Get() (io.ReadCloser, error) {
+	this.locker.Lock()
+	if this.isClosed {
+		this.locker.Unlock()
+		return nil, os.ErrNotExist
+	}
+	this.locker.Unlock()
+
+	fp, err := os.OpenFile(this.filePath, os.O_RDONLY, 0444)
+	if err != nil {
+		return nil, os.ErrNotExist
+	}
+	var subReader = newSubFileReader(fp)
+	this.locker.Lock()
+	this.subReaders = append(this.subReaders, subReader)
+	this.locker.Unlock()
+	return subReader, nil
+}
+
+func (this *ConcurrentFileReaders) NotifyRead() {
+	this.locker.RLock()
+	for _, subReader := range this.subReaders {
+		subReader.NotifyRead()
+	}
+	this.locker.RUnlock()
+}
+
+func (this *ConcurrentFileReaders) NotifyClose() {
+	this.locker.Lock()
+	this.isClosed = true
+	this.locker.Unlock()
+
+	this.locker.RLock()
+	for _, subReader := range this.subReaders {
+		subReader.NotifyClose()
+	}
+	this.locker.RUnlock()
+}

EdgeNode/internal/utils/readers/readers_concurrent_file_test.go

@@ -0,0 +1,67 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package readers_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/types"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestConcurrentFileReaders_Get(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var path = "readers_concurrent_file.txt"
+	writeFp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var concurrentReaders = readers.NewConcurrentFileReaders(path)
+
+	defer func() {
+		_ = writeFp.Close()
+		concurrentReaders.NotifyClose()
+		_ = os.Remove(path)
+		time.Sleep(2 * time.Second)
+	}()
+
+	// pre-write
+	_, _ = writeFp.WriteString("0")
+
+	for i := 0; i < 5; i++ {
+		go func(i int) {
+			reader, readerErr := concurrentReaders.Get()
+			if readerErr != nil {
+				t.Error(i, "Get():", readerErr)
+			} else {
+				defer func() {
+					_ = reader.Close()
+				}()
+
+				var buf = make([]byte, 1024)
+				for {
+					n, readErr := reader.Read(buf)
+					if n > 0 {
+						t.Log(i, "Read:", string(buf[:n]))
+					}
+					if readErr != nil {
+						t.Log(i, "readErr:", readErr)
+						break
+					}
+				}
+			}
+		}(i)
+	}
+
+	for i := 1; i <= 10; i++ {
+		_, _ = writeFp.WriteString(types.String(i))
+		concurrentReaders.NotifyRead()
+		time.Sleep(2 * time.Second)
+	}
+}