Initial commit (code only without large binaries)

EdgeDNS/internal/agents/agent.go

@@ -0,0 +1,36 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package agents
+
+import (
+	"regexp"
+	"strings"
+)
+
+type Agent struct {
+	Code     string
+	suffixes []string
+	reg      *regexp.Regexp
+}
+
+func NewAgent(code string, suffixes []string, reg *regexp.Regexp) *Agent {
+	return &Agent{
+		Code:     code,
+		suffixes: suffixes,
+		reg:      reg,
+	}
+}
+
+func (this *Agent) Match(ptr string) bool {
+	if len(this.suffixes) > 0 {
+		for _, suffix := range this.suffixes {
+			if strings.HasSuffix(ptr, suffix) {
+				return true
+			}
+		}
+	}
+	if this.reg != nil {
+		return this.reg.MatchString(ptr)
+	}
+	return false
+}

EdgeDNS/internal/agents/agents.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package agents
+
+var AllAgents = []*Agent{
+	NewAgent("baidu", []string{".baidu.com."}, nil),
+	NewAgent("google", []string{".googlebot.com."}, nil),
+	NewAgent("bing", []string{".search.msn.com."}, nil),
+	NewAgent("sogou", []string{".sogou.com."}, nil),
+	NewAgent("youdao", []string{".163.com."}, nil),
+	NewAgent("yahoo", []string{".yahoo.com."}, nil),
+	NewAgent("bytedance", []string{".bytedance.com."}, nil),
+	NewAgent("sm", []string{".sm.cn."}, nil),
+	NewAgent("yandex", []string{".yandex.com.", ".yndx.net."}, nil),
+	NewAgent("semrush", []string{".semrush.com."}, nil),
+	NewAgent("facebook", []string{"facebook-waw.1-ix.net.", "facebook.b-ix.net."}, nil),
+}

EdgeDNS/internal/agents/ip_cache_map.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package agents
+
+import (
+	"github.com/TeaOSLab/EdgeDNS/internal/utils/zero"
+	"sync"
+)
+
+type IPCacheMap struct {
+	m    map[string]zero.Zero
+	list []string
+
+	locker sync.RWMutex
+	maxLen int
+}
+
+func NewIPCacheMap(maxLen int) *IPCacheMap {
+	if maxLen <= 0 {
+		maxLen = 65535
+	}
+	return &IPCacheMap{
+		m:      map[string]zero.Zero{},
+		maxLen: maxLen,
+	}
+}
+
+func (this *IPCacheMap) Add(ip string) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	// 是否已经存在
+	_, ok := this.m[ip]
+	if ok {
+		return
+	}
+
+	// 超出长度删除第一个
+	if len(this.list) >= this.maxLen {
+		delete(this.m, this.list[0])
+		this.list = this.list[1:]
+	}
+
+	// 加入新数据
+	this.m[ip] = zero.Zero{}
+	this.list = append(this.list, ip)
+}
+
+func (this *IPCacheMap) Contains(ip string) bool {
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+	_, ok := this.m[ip]
+	return ok
+}

EdgeDNS/internal/agents/ip_cache_map_test.go

@@ -0,0 +1,34 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package agents
+
+import (
+	"github.com/iwind/TeaGo/logs"
+	"testing"
+)
+
+func TestNewIPCacheMap(t *testing.T) {
+	var cacheMap = NewIPCacheMap(3)
+
+	t.Log("====")
+	cacheMap.Add("1")
+	cacheMap.Add("2")
+	logs.PrintAsJSON(cacheMap.m, t)
+	logs.PrintAsJSON(cacheMap.list, t)
+
+	t.Log("====")
+	cacheMap.Add("3")
+	logs.PrintAsJSON(cacheMap.m, t)
+	logs.PrintAsJSON(cacheMap.list, t)
+
+	t.Log("====")
+	cacheMap.Add("4")
+	logs.PrintAsJSON(cacheMap.m, t)
+	logs.PrintAsJSON(cacheMap.list, t)
+
+	t.Log("====")
+	cacheMap.Add("3")
+	logs.PrintAsJSON(cacheMap.m, t)
+	logs.PrintAsJSON(cacheMap.list, t)
+}

EdgeDNS/internal/agents/manager.go

@@ -0,0 +1,173 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package agents
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeDNS/internal/dbs"
+	"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeDNS/internal/rpc"
+	"github.com/iwind/TeaGo/Tea"
+	"sync"
+	"time"
+)
+
+// SharedManager 此值在外面调用时指定
+var SharedManager *Manager
+
+// Manager Agent管理器
+type Manager struct {
+	ipMap  map[string]string // ip => agentCode
+	locker sync.RWMutex
+
+	db *dbs.DB
+
+	lastId int64
+}
+
+func NewManager(db *dbs.DB) *Manager {
+	return &Manager{
+		ipMap: map[string]string{},
+		db:    db,
+	}
+}
+
+func (this *Manager) Start() {
+	remotelogs.Println("AGENT_MANAGER", "starting ...")
+
+	// 从本地数据库中加载
+	err := this.Load()
+	if err != nil {
+		remotelogs.Error("AGENT_MANAGER", "load failed: "+err.Error())
+	}
+
+	// 先从API获取
+	err = this.LoopAll()
+	if err != nil {
+		if rpc.IsConnError(err) {
+			remotelogs.Debug("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error())
+		} else {
+			remotelogs.Error("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error())
+		}
+	}
+
+	// 定时获取
+	var duration = 30 * time.Minute
+	if Tea.IsTesting() {
+		duration = 30 * time.Second
+	}
+	var ticker = time.NewTicker(duration)
+	for range ticker.C {
+		err = this.LoopAll()
+		if err != nil {
+			remotelogs.Error("AGENT_MANAGER", "retrieve latest agent ip failed: "+err.Error())
+		}
+	}
+}
+
+func (this *Manager) Load() error {
+	var offset int64 = 0
+	var size int64 = 10000
+	for {
+		agentIPs, err := this.db.ListAgentIPs(offset, size)
+		if err != nil {
+			return err
+		}
+		if len(agentIPs) == 0 {
+			break
+		}
+		for _, agentIP := range agentIPs {
+			this.locker.Lock()
+			this.ipMap[agentIP.IP] = agentIP.AgentCode
+			this.locker.Unlock()
+
+			if agentIP.Id > this.lastId {
+				this.lastId = agentIP.Id
+			}
+		}
+		offset += size
+	}
+
+	return nil
+}
+
+func (this *Manager) LoopAll() error {
+	for {
+		hasNext, err := this.Loop()
+		if err != nil {
+			return err
+		}
+		if !hasNext {
+			break
+		}
+	}
+	return nil
+}
+
+// Loop 单次循环获取数据
+func (this *Manager) Loop() (hasNext bool, err error) {
+	rpcClient, err := rpc.SharedRPC()
+	if err != nil {
+		return false, err
+	}
+	ipsResp, err := rpcClient.ClientAgentIPRPC.ListClientAgentIPsAfterId(rpcClient.Context(), &pb.ListClientAgentIPsAfterIdRequest{
+		Id:   this.lastId,
+		Size: 10000,
+	})
+	if err != nil {
+		return false, err
+	}
+	if len(ipsResp.ClientAgentIPs) == 0 {
+		return false, nil
+	}
+	for _, agentIP := range ipsResp.ClientAgentIPs {
+		if agentIP.ClientAgent == nil {
+			// 设置ID
+			if agentIP.Id > this.lastId {
+				this.lastId = agentIP.Id
+			}
+
+			continue
+		}
+
+		// 写入到数据库
+		err = this.db.InsertAgentIP(agentIP.Id, agentIP.Ip, agentIP.ClientAgent.Code)
+		if err != nil {
+			return false, err
+		}
+
+		// 写入Map
+		this.locker.Lock()
+		this.ipMap[agentIP.Ip] = agentIP.ClientAgent.Code
+		this.locker.Unlock()
+
+		// 设置ID
+		if agentIP.Id > this.lastId {
+			this.lastId = agentIP.Id
+		}
+	}
+
+	return true, nil
+}
+
+// AddIP 添加记录
+func (this *Manager) AddIP(ip string, agentCode string) {
+	this.locker.Lock()
+	this.ipMap[ip] = agentCode
+	this.locker.Unlock()
+}
+
+// LookupIP 查询IP所属Agent
+func (this *Manager) LookupIP(ip string) (agentCode string) {
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+	return this.ipMap[ip]
+}
+
+// ContainsIP 检查是否有IP相关数据
+func (this *Manager) ContainsIP(ip string) bool {
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+	_, ok := this.ipMap[ip]
+	return ok
+}

EdgeDNS/internal/agents/manager_test.go

@@ -0,0 +1,33 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package agents_test
+
+import (
+	"github.com/TeaOSLab/EdgeDNS/internal/agents"
+	"github.com/TeaOSLab/EdgeDNS/internal/dbs"
+	"github.com/iwind/TeaGo/Tea"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
+)
+
+func TestNewManager(t *testing.T) {
+	var db = dbs.NewDB(Tea.Root + "/data/data.db")
+	err := db.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var manager = agents.NewManager(db)
+	err = manager.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = manager.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(manager.LookupIP("192.168.3.100"))
+}

EdgeDNS/internal/agents/queue.go

@@ -0,0 +1,138 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package agents
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	teaconst "github.com/TeaOSLab/EdgeDNS/internal/const"
+	"github.com/TeaOSLab/EdgeDNS/internal/events"
+	"github.com/TeaOSLab/EdgeDNS/internal/goman"
+	"github.com/TeaOSLab/EdgeDNS/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeDNS/internal/rpc"
+	"github.com/iwind/TeaGo/Tea"
+	"net"
+)
+
+func init() {
+	if !teaconst.IsMain {
+		return
+	}
+
+	events.On(events.EventLoaded, func() {
+		goman.New(func() {
+			SharedQueue.Start()
+		})
+	})
+}
+
+var SharedQueue = NewQueue()
+
+type Queue struct {
+	c        chan string // chan ip
+	cacheMap *IPCacheMap
+}
+
+func NewQueue() *Queue {
+	return &Queue{
+		c:        make(chan string, 128),
+		cacheMap: NewIPCacheMap(65535),
+	}
+}
+
+func (this *Queue) Start() {
+	for ip := range this.c {
+		err := this.Process(ip)
+		if err != nil {
+			// 不需要上报错误
+			if Tea.IsTesting() {
+				remotelogs.Debug("SharedParseQueue", err.Error())
+			}
+			continue
+		}
+	}
+}
+
+// Push 将IP加入到处理队列
+func (this *Queue) Push(ip string) {
+	// 是否在处理中
+	if this.cacheMap.Contains(ip) {
+		return
+	}
+	this.cacheMap.Add(ip)
+
+	// 加入到队列
+	select {
+	case this.c <- ip:
+	default:
+	}
+}
+
+// Process 处理IP
+func (this *Queue) Process(ip string) error {
+	// 是否已经在库中
+	if SharedManager.ContainsIP(ip) {
+		return nil
+	}
+
+	ptr, err := this.ParseIP(ip)
+	if err != nil {
+		return err
+	}
+	if len(ptr) == 0 || ptr == "." {
+		return nil
+	}
+
+	//remotelogs.Debug("AGENT", ip+" => "+ptr)
+
+	var agentCode = this.ParsePtr(ptr)
+	if len(agentCode) == 0 {
+		return nil
+	}
+
+	// 加入到本地
+	SharedManager.AddIP(ip, agentCode)
+
+	var pbAgentIP = &pb.CreateClientAgentIPsRequest_AgentIPInfo{
+		AgentCode: agentCode,
+		Ip:        ip,
+		Ptr:       ptr,
+	}
+	rpcClient, err := rpc.SharedRPC()
+	if err != nil {
+		return err
+	}
+	_, err = rpcClient.ClientAgentIPRPC.CreateClientAgentIPs(rpcClient.Context(), &pb.CreateClientAgentIPsRequest{AgentIPs: []*pb.CreateClientAgentIPsRequest_AgentIPInfo{pbAgentIP}})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ParseIP 分析IP的PTR值
+func (this *Queue) ParseIP(ip string) (ptr string, err error) {
+	if len(ip) == 0 {
+		return "", nil
+	}
+
+	names, err := net.LookupAddr(ip)
+	if err != nil {
+		return "", err
+	}
+
+	if len(names) == 0 {
+		return "", nil
+	}
+
+	return names[0], nil
+}
+
+// ParsePtr 分析PTR对应的Agent
+func (this *Queue) ParsePtr(ptr string) (agentCode string) {
+	for _, agent := range AllAgents {
+		if agent.Match(ptr) {
+			return agent.Code
+		}
+	}
+	return ""
+}

EdgeDNS/internal/agents/queue_test.go

@@ -0,0 +1,77 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package agents_test
+
+import (
+	"github.com/TeaOSLab/EdgeDNS/internal/agents"
+	"github.com/iwind/TeaGo/assert"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
+	"time"
+)
+
+func TestParseQueue_Process(t *testing.T) {
+	var queue = agents.NewQueue()
+	go queue.Start()
+	time.Sleep(1 * time.Second)
+	queue.Push("220.181.13.100")
+	time.Sleep(1 * time.Second)
+}
+
+func TestParseQueue_ParseIP(t *testing.T) {
+	var queue = agents.NewQueue()
+	for _, ip := range []string{
+		"192.168.1.100",
+		"42.120.160.1",
+		"42.236.10.98",
+		"124.115.0.100",
+	} {
+		ptr, err := queue.ParseIP(ip)
+		if err != nil {
+			t.Log(ip, "=>", err)
+			continue
+		}
+		t.Log(ip, "=>", ptr)
+	}
+}
+
+func TestParseQueue_ParsePtr(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var queue = agents.NewQueue()
+	for _, s := range [][]string{
+		{"baiduspider-220-181-108-101.crawl.baidu.com.", "baidu"},
+		{"crawl-66-249-71-219.googlebot.com.", "google"},
+		{"msnbot-40-77-167-31.search.msn.com.", "bing"},
+		{"sogouspider-49-7-20-129.crawl.sogou.com.", "sogou"},
+		{"m13102.mail.163.com.", "youdao"},
+		{"yeurosport.pat1.tc2.yahoo.com.", "yahoo"},
+		{"shenmaspider-42-120-160-1.crawl.sm.cn.", "sm"},
+		{"93-158-161-39.spider.yandex.com.", "yandex"},
+		{"25.bl.bot.semrush.com.", "semrush"},
+	} {
+		a.IsTrue(queue.ParsePtr(s[0]) == s[1])
+	}
+}
+
+func BenchmarkQueue_ParsePtr(b *testing.B) {
+	var queue = agents.NewQueue()
+
+	for i := 0; i < b.N; i++ {
+		for _, s := range [][]string{
+			{"baiduspider-220-181-108-101.crawl.baidu.com.", "baidu"},
+			{"crawl-66-249-71-219.googlebot.com.", "google"},
+			{"msnbot-40-77-167-31.search.msn.com.", "bing"},
+			{"sogouspider-49-7-20-129.crawl.sogou.com.", "sogou"},
+			{"m13102.mail.163.com.", "youdao"},
+			{"yeurosport.pat1.tc2.yahoo.com.", "yahoo"},
+			{"shenmaspider-42-120-160-1.crawl.sm.cn.", "sm"},
+			{"93-158-161-39.spider.yandex.com.", "yandex"},
+			{"93.158.164.218-red.dhcp.yndx.net.", "yandex"},
+			{"25.bl.bot.semrush.com.", "semrush"},
+		} {
+			queue.ParsePtr(s[0])
+		}
+	}
+}