Initial commit (code only without large binaries)

EdgeNode/internal/waf/utils/utils.go

@@ -0,0 +1,131 @@
+package utils
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/cachehits"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/re"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache"
+	"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
+	"github.com/cespare/xxhash/v2"
+	"github.com/iwind/TeaGo/types"
+	stringutil "github.com/iwind/TeaGo/utils/string"
+	"strconv"
+)
+
+var SharedCache = ttlcache.NewCache[int8]()
+var cacheHits *cachehits.Stat
+
+func init() {
+	if !teaconst.IsMain {
+		return
+	}
+	cacheHits = cachehits.NewStat(5)
+}
+
+const (
+	MaxCacheDataSize = 1024
+)
+
+type CacheLife = int64
+
+const (
+	CacheDisabled   CacheLife = 0
+	CacheShortLife  CacheLife = 600
+	CacheMiddleLife CacheLife = 1800
+	CacheLongLife   CacheLife = 7200
+)
+
+// MatchStringCache 正则表达式匹配字符串，并缓存结果
+func MatchStringCache(regex *re.Regexp, s string, cacheLife CacheLife) bool {
+	if regex == nil {
+		return false
+	}
+
+	var regIdString = regex.IdString()
+
+	// 如果长度超过一定数量，大概率是不能重用的
+	if cacheLife <= 0 || len(s) > MaxCacheDataSize || !cacheHits.IsGood(regIdString) {
+		return regex.MatchString(s)
+	}
+
+	var hash = xxhash.Sum64String(s)
+	var key = regIdString + "@" + strconv.FormatUint(hash, 10)
+	var item = SharedCache.Read(key)
+	if item != nil {
+		cacheHits.IncreaseHit(regIdString)
+		return item.Value == 1
+	}
+	var b = regex.MatchString(s)
+	if b {
+		SharedCache.Write(key, 1, fasttime.Now().Unix()+cacheLife)
+	} else {
+		SharedCache.Write(key, 0, fasttime.Now().Unix()+cacheLife)
+	}
+	cacheHits.IncreaseCached(regIdString)
+	return b
+}
+
+// MatchBytesCache 正则表达式匹配字节slice，并缓存结果
+func MatchBytesCache(regex *re.Regexp, byteSlice []byte, cacheLife CacheLife) bool {
+	if regex == nil {
+		return false
+	}
+
+	var regIdString = regex.IdString()
+
+	// 如果长度超过一定数量，大概率是不能重用的
+	if cacheLife <= 0 || len(byteSlice) > MaxCacheDataSize || !cacheHits.IsGood(regIdString) {
+		return regex.Match(byteSlice)
+	}
+
+	var hash = xxhash.Sum64(byteSlice)
+	var key = regIdString + "@" + strconv.FormatUint(hash, 10)
+	var item = SharedCache.Read(key)
+	if item != nil {
+		cacheHits.IncreaseHit(regIdString)
+		return item.Value == 1
+	}
+	var b = regex.Match(byteSlice)
+	if b {
+		SharedCache.Write(key, 1, fasttime.Now().Unix()+cacheLife)
+	} else {
+		SharedCache.Write(key, 0, fasttime.Now().Unix()+cacheLife)
+	}
+	cacheHits.IncreaseCached(regIdString)
+	return b
+}
+
+// ComposeIPType 组合IP类型
+func ComposeIPType(setId int64, req requests.Request) string {
+	return "set:" + types.String(setId) + "@" + stringutil.Md5(req.WAFRaw().UserAgent())
+}
+
+var searchEngineProviderMap = map[string]bool{
+	"谷歌":       true,
+	"雅虎":       true,
+	"脸书":       true,
+	"百度":       true,
+	"Facebook": true,
+	"Yandex":   true,
+}
+
+// CheckSearchEngine check if ip is from search engines
+func CheckSearchEngine(ip string) bool {
+	if len(ip) == 0 {
+		return false
+	}
+
+	if agents.SharedManager.ContainsIP(ip) {
+		return true
+	}
+
+	var result = iplibrary.LookupIP(ip)
+	if result == nil {
+		return false
+	}
+
+	return searchEngineProviderMap[result.ProviderName()]
+}

EdgeNode/internal/waf/utils/utils_test.go

@@ -0,0 +1,113 @@
+package utils_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/re"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/TeaOSLab/EdgeNode/internal/waf/utils"
+	"github.com/iwind/TeaGo/rands"
+	"net/http"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestMatchStringCache(t *testing.T) {
+	regex := re.MustCompile(`\d+`)
+	t.Log(utils.MatchStringCache(regex, "123", utils.CacheShortLife))
+	t.Log(utils.MatchStringCache(regex, "123", utils.CacheShortLife))
+	t.Log(utils.MatchStringCache(regex, "123", utils.CacheShortLife))
+}
+
+func TestMatchBytesCache(t *testing.T) {
+	regex := re.MustCompile(`\d+`)
+	t.Log(utils.MatchBytesCache(regex, []byte("123"), utils.CacheShortLife))
+	t.Log(utils.MatchBytesCache(regex, []byte("123"), utils.CacheShortLife))
+	t.Log(utils.MatchBytesCache(regex, []byte("123"), utils.CacheShortLife))
+}
+
+func TestMatchRemoteCache(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var client = http.Client{}
+	for i := 0; i < 200_0000; i++ {
+		req, err := http.NewRequest(http.MethodGet, "http://192.168.2.30:8882/?arg="+strconv.Itoa(i), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Set("User-Agent", "GoTest/"+strconv.Itoa(i))
+		resp, err := client.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_ = resp.Body.Close()
+	}
+}
+
+func TestMatchBytesCache_WithoutCache(t *testing.T) {
+	data := []byte(strings.Repeat("HELLO", 512))
+	regex := regexp.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+	before := time.Now()
+	t.Log(regex.Match(data))
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+}
+
+func BenchmarkMatchStringCache(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	var data = strings.Repeat("HELLO", 128)
+	var regex = re.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+	//b.Log(regex.Keywords())
+	_ = utils.MatchStringCache(regex, data, utils.CacheShortLife)
+
+	for i := 0; i < b.N; i++ {
+		_ = utils.MatchStringCache(regex, data, utils.CacheShortLife)
+	}
+}
+
+func BenchmarkMatchStringCache_LowHit(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	var regex = re.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+	//b.Log(regex.Keywords())
+
+	for i := 0; i < b.N; i++ {
+		var data = strings.Repeat("A", rands.Int(0, 100))
+		_ = utils.MatchStringCache(regex, data, utils.CacheShortLife)
+	}
+}
+
+func BenchmarkMatchStringCache_WithoutCache(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	data := strings.Repeat("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", 8)
+	regex := re.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+	for i := 0; i < b.N; i++ {
+		_ = regex.MatchString(data)
+	}
+}
+
+func BenchmarkMatchStringCache_WithoutCache2(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	data := strings.Repeat("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", 8)
+	regex := regexp.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+	for i := 0; i < b.N; i++ {
+		_ = regex.MatchString(data)
+	}
+}
+
+func BenchmarkMatchBytesCache_WithoutCache(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	data := []byte(strings.Repeat("HELLO", 128))
+	regex := re.MustCompile(`(?iU)\b(eval|system|exec|execute|passthru|shell_exec|phpinfo)\b`)
+
+	for i := 0; i < b.N; i++ {
+		_ = regex.Match(data)
+	}
+}