Initial commit (code only without large binaries)

EdgeAPI/internal/tasks/dns_task_executor.go

@@ -0,0 +1,788 @@
+package tasks
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	dnsmodels "github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
+	"github.com/TeaOSLab/EdgeAPI/internal/dnsclients"
+	"github.com/TeaOSLab/EdgeAPI/internal/dnsclients/dnstypes"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iputils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"net"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewDNSTaskExecutor(20 * time.Second).Start()
+		})
+	})
+}
+
+// DNSTaskExecutor DNS任务执行器
+type DNSTaskExecutor struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewDNSTaskExecutor(duration time.Duration) *DNSTaskExecutor {
+	return &DNSTaskExecutor{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *DNSTaskExecutor) Start() {
+	for {
+		select {
+		case <-this.ticker.C:
+		case <-dnsmodels.DNSTasksNotifier:
+			time.Sleep(3 * time.Second) // 人为延长N秒，等待可能的几个任务合并
+		}
+
+		err := this.Loop()
+		if err != nil {
+			this.logErr("DNSTaskExecutor", err.Error())
+		}
+	}
+}
+
+func (this *DNSTaskExecutor) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	return this.loop()
+}
+
+func (this *DNSTaskExecutor) loop() error {
+	tasks, err := dnsmodels.SharedDNSTaskDAO.FindAllDoingTasks(nil)
+	if err != nil {
+		return err
+	}
+
+	for _, task := range tasks {
+		var taskId = int64(task.Id)
+		var taskVersion = int64(task.Version)
+		switch task.Type {
+		case dnsmodels.DNSTaskTypeServerChange:
+			err = this.doServer(taskId, int64(task.Version), int64(task.ClusterId), int64(task.ServerId))
+			if err != nil {
+				err = dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskError(nil, taskId, err.Error())
+				if err != nil {
+					return err
+				}
+			}
+		case dnsmodels.DNSTaskTypeNodeChange:
+			err = this.doNode(taskId, taskVersion, int64(task.ClusterId), int64(task.NodeId))
+			if err != nil {
+				err = dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskError(nil, taskId, err.Error())
+				if err != nil {
+					return err
+				}
+			}
+		case dnsmodels.DNSTaskTypeClusterChange, dnsmodels.DNSTaskTypeClusterNodesChange:
+			err = this.doCluster(taskId, taskVersion, int64(task.ClusterId), task.Type == dnsmodels.DNSTaskTypeClusterNodesChange)
+			if err != nil {
+				err = dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskError(nil, taskId, err.Error())
+				if err != nil {
+					return err
+				}
+			}
+		case dnsmodels.DNSTaskTypeClusterRemoveDomain:
+			err = this.doClusterRemove(taskId, taskVersion, int64(task.ClusterId), int64(task.DomainId), task.RecordName)
+			if err != nil {
+				err = dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskError(nil, taskId, err.Error())
+				if err != nil {
+					return err
+				}
+			}
+		case dnsmodels.DNSTaskTypeDomainChange:
+			err = this.doDomainWithTask(taskId, taskVersion, int64(task.DomainId))
+			if err != nil {
+				err = dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskError(nil, taskId, err.Error())
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// 修改服务相关记录
+func (this *DNSTaskExecutor) doServer(taskId int64, taskVersion int64, oldClusterId int64, serverId int64) error {
+	var tx *dbs.Tx
+
+	var isOk = false
+	defer func() {
+		if isOk {
+			err := dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskDone(tx, taskId, taskVersion)
+			if err != nil {
+				this.logErr("DNSTaskExecutor", err.Error())
+			}
+		}
+	}()
+
+	// 检查是否已通过审核
+	serverDNS, err := models.SharedServerDAO.FindStatelessServerDNS(tx, serverId)
+	if err != nil {
+		return err
+	}
+	if serverDNS == nil {
+		isOk = true
+		return nil
+	}
+	if len(serverDNS.DnsName) == 0 {
+		isOk = true
+		return nil
+	}
+
+	var recordName = serverDNS.DnsName
+	var recordType = dnstypes.RecordTypeCNAME
+
+	// 新的DNS设置
+	manager, newDomainId, domain, clusterDNSName, dnsConfig, err := this.findDNSManagerWithClusterId(tx, int64(serverDNS.ClusterId))
+	if err != nil {
+		return err
+	}
+
+	// 如果集群发生了变化，则从老的集群中删除
+	if oldClusterId > 0 && int64(serverDNS.ClusterId) != oldClusterId {
+		oldManager, oldDomainId, oldDomain, _, _, err := this.findDNSManagerWithClusterId(tx, oldClusterId)
+		if err != nil {
+			return err
+		}
+
+		// 如果域名发生了变化
+		if oldDomainId != newDomainId {
+			if oldManager != nil {
+				oldRecord, err := oldManager.QueryRecord(oldDomain, recordName, recordType)
+				if err != nil {
+					return err
+				}
+				if oldRecord != nil {
+					// 删除记录
+					err = oldManager.DeleteRecord(oldDomain, oldRecord)
+					if err != nil {
+						return err
+					}
+
+					// 更新域名中记录缓存
+					// 这里不创建域名更新任务，而是直接更新，避免影响其他任务的执行
+					err = this.doDomain(oldDomainId)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+
+		isOk = true
+		return nil
+	}
+
+	// 处理新的集群
+	if manager == nil {
+		isOk = true
+		return nil
+	}
+	var ttl int32 = 0
+	if dnsConfig != nil {
+		ttl = dnsConfig.TTL
+	}
+
+	recordValue := clusterDNSName + "." + domain + "."
+	recordRoute := manager.DefaultRoute()
+	if serverDNS.State == models.ServerStateDisabled || !serverDNS.IsOn {
+		// 检查记录是否已经存在
+		record, err := manager.QueryRecord(domain, recordName, recordType)
+		if err != nil {
+			return err
+		}
+		if record != nil {
+			// 删除
+			err = manager.DeleteRecord(domain, record)
+			if err != nil {
+				return err
+			}
+			err = dnsmodels.SharedDNSTaskDAO.CreateDomainTask(tx, newDomainId, dnsmodels.DNSTaskTypeDomainChange)
+			if err != nil {
+				return err
+			}
+		}
+
+		isOk = true
+	} else {
+		// 是否已存在
+		exist, err := dnsmodels.SharedDNSDomainDAO.ExistDomainRecord(tx, newDomainId, recordName, recordType, recordRoute, recordValue)
+		if err != nil {
+			return err
+		}
+		if exist {
+			isOk = true
+			return nil
+		}
+
+		// 检查记录是否已经存在
+		record, err := manager.QueryRecord(domain, recordName, recordType)
+		if err != nil {
+			return err
+		}
+		if record != nil {
+			if record.Value == recordValue || record.Value == strings.TrimRight(recordValue, ".") {
+				isOk = true
+				return nil
+			}
+
+			// 删除
+			err = manager.DeleteRecord(domain, record)
+			if err != nil {
+				return err
+			}
+			err = dnsmodels.SharedDNSTaskDAO.CreateDomainTask(tx, newDomainId, dnsmodels.DNSTaskTypeDomainChange)
+			if err != nil {
+				return err
+			}
+		}
+
+		err = manager.AddRecord(domain, &dnstypes.Record{
+			Id:    "",
+			Name:  recordName,
+			Type:  recordType,
+			Value: recordValue,
+			Route: recordRoute,
+			TTL:   ttl,
+		})
+		if err != nil {
+			return err
+		}
+
+		err = dnsmodels.SharedDNSTaskDAO.CreateDomainTask(tx, newDomainId, dnsmodels.DNSTaskTypeDomainChange)
+		if err != nil {
+			return err
+		}
+
+		isOk = true
+	}
+
+	return nil
+}
+
+// 修改节点相关记录
+func (this *DNSTaskExecutor) doNode(taskId int64, taskVersion int64, nodeClusterId int64, nodeId int64) error {
+	var isOk = false
+	defer func() {
+		if isOk {
+			err := dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskDone(nil, taskId, taskVersion)
+			if err != nil {
+				this.logErr("DNSTaskExecutor", err.Error())
+			}
+		}
+	}()
+
+	var tx *dbs.Tx
+	node, err := models.SharedNodeDAO.FindStatelessNodeDNS(tx, nodeId)
+	if err != nil {
+		return err
+	}
+	if node == nil {
+		isOk = true
+		return nil
+	}
+
+	// 转交给cluster统一处理
+	if nodeClusterId > 0 {
+		err = dnsmodels.SharedDNSTaskDAO.CreateClusterTask(tx, nodeClusterId, dnsmodels.DNSTaskTypeClusterNodesChange)
+		if err != nil {
+			return err
+		}
+	} else {
+		clusterIds, err := models.SharedNodeDAO.FindEnabledAndOnNodeClusterIds(tx, nodeId)
+		if err != nil {
+			return err
+		}
+		for _, clusterId := range clusterIds {
+			err = dnsmodels.SharedDNSTaskDAO.CreateClusterTask(tx, clusterId, dnsmodels.DNSTaskTypeClusterNodesChange)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	isOk = true
+
+	return nil
+}
+
+// 修改集群相关记录
+func (this *DNSTaskExecutor) doCluster(taskId int64, taskVersion int64, clusterId int64, nodesOnly bool) error {
+	var isOk = false
+	defer func() {
+		if isOk {
+			err := dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskDone(nil, taskId, taskVersion)
+			if err != nil {
+				this.logErr("DNSTaskExecutor", err.Error())
+			}
+		}
+	}()
+
+	var tx *dbs.Tx
+	manager, domainId, domain, clusterDNSName, dnsConfig, err := this.findDNSManagerWithClusterId(tx, clusterId)
+	if err != nil {
+		return err
+	}
+	if manager == nil {
+		isOk = true
+		return nil
+	}
+
+	var clusterDomain = clusterDNSName + "." + domain
+
+	var ttl int32 = 0
+	if dnsConfig != nil {
+		ttl = dnsConfig.TTL
+	}
+
+	// 以前的节点记录
+	records, err := manager.GetRecords(domain)
+	if err != nil {
+		return err
+	}
+	var oldRecordsMap = map[string]*dnstypes.Record{}      // route@value => record
+	var oldCnameRecordsMap = map[string]*dnstypes.Record{} // cname => record
+	for _, record := range records {
+		if (record.Type == dnstypes.RecordTypeA || record.Type == dnstypes.RecordTypeAAAA) && record.Name == clusterDNSName {
+			key := record.Route + "@" + record.Value
+			oldRecordsMap[key] = record
+		}
+
+		if record.Type == dnstypes.RecordTypeCNAME {
+			oldCnameRecordsMap[record.Name] = record
+		}
+	}
+
+	// 当前的节点记录
+	var newRecordKeys = []string{}
+	nodes, err := models.SharedNodeDAO.FindAllEnabledNodesDNSWithClusterId(tx, clusterId, true, dnsConfig != nil && dnsConfig.IncludingLnNodes, true)
+	if err != nil {
+		return err
+	}
+	var isChanged = false
+	var addingNodeRecordKeysMap = map[string]bool{} // clusterDnsName_type_ip_route
+	for _, node := range nodes {
+		shouldSkip, shouldOverwrite, ipAddressesStrings, err := models.SharedNodeDAO.CheckNodeIPAddresses(tx, node)
+		if err != nil {
+			return err
+		}
+		if shouldSkip {
+			continue
+		}
+
+		routes, err := node.DNSRouteCodesForDomainId(domainId)
+		if err != nil {
+			return err
+		}
+		if len(routes) == 0 {
+			routes = []string{manager.DefaultRoute()}
+		}
+
+		// 所有的IP记录
+		if !shouldOverwrite {
+			ipAddresses, err := models.SharedNodeIPAddressDAO.FindAllEnabledAddressesWithNode(tx, int64(node.Id), nodeconfigs.NodeRoleNode)
+			if err != nil {
+				return err
+			}
+			if len(ipAddresses) == 0 {
+				continue
+			}
+			for _, ipAddress := range ipAddresses {
+				// 检查专属节点
+				if !ipAddress.IsValidInCluster(clusterId) {
+					continue
+				}
+
+				var ip = ipAddress.DNSIP()
+				if len(ip) == 0 || !ipAddress.CanAccess || !ipAddress.IsUp || !ipAddress.IsOn {
+					continue
+				}
+				if net.ParseIP(ip) == nil {
+					continue
+				}
+				ipAddressesStrings = append(ipAddressesStrings, ip)
+			}
+		}
+
+		if len(ipAddressesStrings) == 0 {
+			continue
+		}
+
+		for _, ip := range ipAddressesStrings {
+			for _, route := range routes {
+				var key = route + "@" + ip
+				_, ok := oldRecordsMap[key]
+				if ok {
+					newRecordKeys = append(newRecordKeys, key)
+					continue
+				}
+
+				var recordType = dnstypes.RecordTypeA
+				if iputils.IsIPv6(ip) {
+					recordType = dnstypes.RecordTypeAAAA
+				}
+
+				// 避免添加重复的记录
+				var fullKey = clusterDNSName + "_" + recordType + "_" + ip + "_" + route
+				if addingNodeRecordKeysMap[fullKey] {
+					continue
+				}
+				addingNodeRecordKeysMap[fullKey] = true
+
+				err = manager.AddRecord(domain, &dnstypes.Record{
+					Id:    "",
+					Name:  clusterDNSName,
+					Type:  recordType,
+					Value: ip,
+					Route: route,
+					TTL:   ttl,
+				})
+				if err != nil {
+					return err
+				}
+				isChanged = true
+				newRecordKeys = append(newRecordKeys, key)
+			}
+		}
+	}
+
+	// 删除多余的节点解析记录
+	for key, record := range oldRecordsMap {
+		if !lists.ContainsString(newRecordKeys, key) {
+			isChanged = true
+			err = manager.DeleteRecord(domain, record)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// 服务域名
+	if !nodesOnly {
+		servers, err := models.SharedServerDAO.FindAllServersDNSWithClusterId(tx, clusterId)
+		if err != nil {
+			return err
+		}
+		serverRecords := []*dnstypes.Record{}             // 之所以用数组再存一遍，是因为dnsName可能会重复
+		serverRecordsMap := map[string]*dnstypes.Record{} // dnsName => *Record
+		for _, record := range records {
+			if record.Type == dnstypes.RecordTypeCNAME && record.Value == clusterDomain+"." {
+				serverRecords = append(serverRecords, record)
+				serverRecordsMap[record.Name] = record
+			}
+		}
+
+		// 新增的域名
+		var serverDNSNames = []string{}
+		for _, server := range servers {
+			var dnsName = server.DnsName
+			if len(dnsName) == 0 {
+				continue
+			}
+			serverDNSNames = append(serverDNSNames, dnsName)
+			_, ok := serverRecordsMap[dnsName]
+			if !ok {
+				isChanged = true
+				err = manager.AddRecord(domain, &dnstypes.Record{
+					Id:    "",
+					Name:  dnsName,
+					Type:  dnstypes.RecordTypeCNAME,
+					Value: clusterDomain + ".",
+					Route: "", // 注意这里为空，需要在执行过程中获取默认值
+					TTL:   ttl,
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 自动设置的CNAME
+		var cnameRecords = []string{}
+		if dnsConfig != nil {
+			cnameRecords = dnsConfig.CNAMERecords
+		}
+		for _, cnameRecord := range cnameRecords {
+			// 如果记录已存在，则跳过
+			if lists.ContainsString(serverDNSNames, cnameRecord) {
+				continue
+			}
+
+			serverDNSNames = append(serverDNSNames, cnameRecord)
+			_, ok := serverRecordsMap[cnameRecord]
+			if !ok {
+				isChanged = true
+				err = manager.AddRecord(domain, &dnstypes.Record{
+					Id:    "",
+					Name:  cnameRecord,
+					Type:  dnstypes.RecordTypeCNAME,
+					Value: clusterDomain + ".",
+					Route: "", // 注意这里为空，需要在执行过程中获取默认值
+					TTL:   ttl,
+				})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 多余的域名
+		for _, record := range serverRecords {
+			if !lists.ContainsString(serverDNSNames, record.Name) {
+				isChanged = true
+				err = manager.DeleteRecord(domain, record)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	// 通知更新域名
+	if isChanged {
+		err = dnsmodels.SharedDNSTaskDAO.CreateDomainTask(tx, domainId, dnsmodels.DNSTaskTypeDomainChange)
+		if err != nil {
+			return err
+		}
+	}
+
+	isOk = true
+
+	return nil
+}
+
+func (this *DNSTaskExecutor) doClusterRemove(taskId int64, taskVersion int64, clusterId int64, domainId int64, dnsName string) error {
+	var isOk = false
+	defer func() {
+		if isOk {
+			err := dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskDone(nil, taskId, taskVersion)
+			if err != nil {
+				this.logErr("DNSTaskExecutor", err.Error())
+			}
+		}
+	}()
+
+	var tx *dbs.Tx
+
+	dnsInfo, err := models.SharedNodeClusterDAO.FindClusterDNSInfo(tx, clusterId, nil)
+	if err != nil {
+		return err
+	}
+
+	if len(dnsName) == 0 {
+		if dnsInfo == nil {
+			isOk = true
+			return nil
+		}
+		dnsName = dnsInfo.DnsName
+		if len(dnsName) == 0 {
+			isOk = true
+			return nil
+		}
+	}
+
+	// 再次检查是否正在使用，如果正在使用，则直接返回
+	if dnsInfo != nil && dnsInfo.State == models.NodeClusterStateEnabled /** 尚未被删除 **/ && int64(dnsInfo.DnsDomainId) == domainId && dnsInfo.DnsName == dnsName {
+		isOk = true
+		return nil
+	}
+
+	domain, manager, err := this.findDNSManagerWithDomainId(tx, domainId)
+	if err != nil {
+		return err
+	}
+	if domain == nil {
+		isOk = true
+		return nil
+	}
+	var fullName = dnsName + "." + domain.Name
+
+	records, err := domain.DecodeRecords()
+	if err != nil {
+		return err
+	}
+
+	var isChanged bool
+
+	for _, record := range records {
+		// node A
+		if (record.Type == dnstypes.RecordTypeA || record.Type == dnstypes.RecordTypeAAAA) && record.Name == dnsName {
+			err = manager.DeleteRecord(domain.Name, record)
+			if err != nil {
+				return err
+			}
+			isChanged = true
+		}
+
+		// server CNAME
+		if record.Type == dnstypes.RecordTypeCNAME && strings.TrimRight(record.Value, ".") == fullName {
+			err = manager.DeleteRecord(domain.Name, record)
+			if err != nil {
+				return err
+			}
+			isChanged = true
+		}
+	}
+
+	if isChanged {
+		err = dnsmodels.SharedDNSTaskDAO.CreateDomainTask(tx, domainId, dnsmodels.DNSTaskTypeDomainChange)
+		if err != nil {
+			return err
+		}
+	}
+
+	isOk = true
+
+	return nil
+}
+
+func (this *DNSTaskExecutor) doDomain(domainId int64) error {
+	return this.doDomainWithTask(0, 0, domainId)
+}
+
+func (this *DNSTaskExecutor) doDomainWithTask(taskId int64, taskVersion int64, domainId int64) error {
+	var tx *dbs.Tx
+
+	var isOk = false
+	defer func() {
+		if isOk {
+			if taskId > 0 {
+				err := dnsmodels.SharedDNSTaskDAO.UpdateDNSTaskDone(tx, taskId, taskVersion)
+				if err != nil {
+					this.logErr("DNSTaskExecutor", err.Error())
+				}
+			}
+		}
+	}()
+
+	dnsDomain, err := dnsmodels.SharedDNSDomainDAO.FindEnabledDNSDomain(tx, domainId, nil)
+	if err != nil {
+		return err
+	}
+	if dnsDomain == nil {
+		isOk = true
+		return nil
+	}
+	var providerId = int64(dnsDomain.ProviderId)
+	if providerId <= 0 {
+		isOk = true
+		return nil
+	}
+
+	provider, err := dnsmodels.SharedDNSProviderDAO.FindEnabledDNSProvider(tx, providerId)
+	if err != nil {
+		return err
+	}
+	if provider == nil {
+		isOk = true
+		return nil
+	}
+
+	var manager = dnsclients.FindProvider(provider.Type, int64(provider.Id))
+	if manager == nil {
+		this.logErr("DNSTaskExecutor", "unsupported dns provider type '"+provider.Type+"'")
+		isOk = true
+		return nil
+	}
+	params, err := provider.DecodeAPIParams()
+	if err != nil {
+		return err
+	}
+	err = manager.Auth(params)
+	if err != nil {
+		return err
+	}
+	records, err := manager.GetRecords(dnsDomain.Name)
+	if err != nil {
+		return err
+	}
+	recordsJSON, err := json.Marshal(records)
+	if err != nil {
+		return err
+	}
+	err = dnsmodels.SharedDNSDomainDAO.UpdateDomainRecords(tx, domainId, recordsJSON)
+	if err != nil {
+		return err
+	}
+	isOk = true
+	return nil
+}
+
+func (this *DNSTaskExecutor) findDNSManagerWithClusterId(tx *dbs.Tx, clusterId int64) (manager dnsclients.ProviderInterface, domainId int64, domain string, clusterDNSName string, dnsConfig *dnsconfigs.ClusterDNSConfig, err error) {
+	clusterDNS, err := models.SharedNodeClusterDAO.FindClusterDNSInfo(tx, clusterId, nil)
+	if err != nil {
+		return nil, 0, "", "", nil, err
+	}
+	if clusterDNS == nil || len(clusterDNS.DnsName) == 0 || clusterDNS.DnsDomainId <= 0 {
+		return nil, 0, "", "", nil, nil
+	}
+
+	dnsConfig, err = clusterDNS.DecodeDNSConfig()
+	if err != nil {
+		return nil, 0, "", "", nil, err
+	}
+
+	dnsDomain, manager, err := this.findDNSManagerWithDomainId(tx, int64(clusterDNS.DnsDomainId))
+	if err != nil {
+		return nil, 0, "", "", nil, err
+	}
+
+	if dnsDomain == nil {
+		return nil, 0, "", clusterDNS.DnsName, dnsConfig, nil
+	}
+
+	return manager, int64(dnsDomain.Id), dnsDomain.Name, clusterDNS.DnsName, dnsConfig, nil
+}
+
+func (this *DNSTaskExecutor) findDNSManagerWithDomainId(tx *dbs.Tx, domainId int64) (*dnsmodels.DNSDomain, dnsclients.ProviderInterface, error) {
+	dnsDomain, err := dnsmodels.SharedDNSDomainDAO.FindEnabledDNSDomain(tx, domainId, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	if dnsDomain == nil {
+		return nil, nil, nil
+	}
+	providerId := int64(dnsDomain.ProviderId)
+	if providerId <= 0 {
+		return nil, nil, nil
+	}
+
+	provider, err := dnsmodels.SharedDNSProviderDAO.FindEnabledDNSProvider(tx, providerId)
+	if err != nil {
+		return nil, nil, err
+	}
+	if provider == nil {
+		return nil, nil, nil
+	}
+
+	var manager = dnsclients.FindProvider(provider.Type, int64(provider.Id))
+	if manager == nil {
+		this.logErr("DNSTaskExecutor", "unsupported dns provider type '"+provider.Type+"'")
+		return nil, nil, nil
+	}
+	params, err := provider.DecodeAPIParams()
+	if err != nil {
+		return nil, nil, err
+	}
+	err = manager.Auth(params)
+	if err != nil {
+		return nil, nil, err
+	}
+	return dnsDomain, manager, nil
+}

EdgeAPI/internal/tasks/dns_task_executor_plus_test.go

@@ -0,0 +1,21 @@
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestDNSTaskExecutor_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewDNSTaskExecutor(10 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/dns_task_executor_test.go

@@ -0,0 +1,21 @@
+//go:build !plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestDNSTaskExecutor_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewDNSTaskExecutor(10 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/event_looper.go

@@ -0,0 +1,67 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewEventLooper(2 * time.Second).Start()
+		})
+	})
+}
+
+// EventLooper 事件相关处理程序
+type EventLooper struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewEventLooper(duration time.Duration) *EventLooper {
+	return &EventLooper{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *EventLooper) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("EventLooper", err.Error())
+		}
+	}
+}
+
+func (this *EventLooper) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	events, err := models.SharedSysEventDAO.FindEvents(nil, 100)
+	if err != nil {
+		return err
+	}
+	for _, eventOne := range events {
+		event, err := eventOne.DecodeEvent()
+		if err != nil {
+			this.logErr("EventLooper", err.Error())
+			continue
+		}
+		err = event.Run()
+		if err != nil {
+			this.logErr("EventLooper", err.Error())
+			continue
+		}
+		err = models.SharedSysEventDAO.DeleteEvent(nil, int64(eventOne.Id))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/health_check_cluster_task.go

@@ -0,0 +1,170 @@
+package tasks
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"time"
+)
+
+// HealthCheckClusterTask 单个集群的健康检查任务
+type HealthCheckClusterTask struct {
+	BaseTask
+
+	clusterId int64
+	config    *serverconfigs.HealthCheckConfig
+	ticker    *utils.Ticker
+
+	notifiedTime time.Time
+}
+
+// NewHealthCheckClusterTask 创建新任务
+func NewHealthCheckClusterTask(clusterId int64, config *serverconfigs.HealthCheckConfig) *HealthCheckClusterTask {
+	return &HealthCheckClusterTask{
+		clusterId: clusterId,
+		config:    config,
+	}
+}
+
+// Reset 重置配置
+func (this *HealthCheckClusterTask) Reset(config *serverconfigs.HealthCheckConfig) {
+	// 检查是否有变化
+	oldJSON, err := json.Marshal(this.config)
+	if err != nil {
+		this.logErr("HealthCheckClusterTask", err.Error())
+		return
+	}
+	newJSON, err := json.Marshal(config)
+	if err != nil {
+		this.logErr("HealthCheckClusterTask", err.Error())
+		return
+	}
+	if !bytes.Equal(oldJSON, newJSON) {
+		this.config = config
+		this.Run()
+	}
+}
+
+// Run 执行
+func (this *HealthCheckClusterTask) Run() {
+	this.Stop()
+
+	if this.config == nil {
+		return
+	}
+	if !this.config.IsOn {
+		return
+	}
+	if this.config.Interval == nil {
+		return
+	}
+	var duration = this.config.Interval.Duration()
+	if duration <= 0 {
+		return
+	}
+	var ticker = utils.NewTicker(duration)
+	goman.New(func() {
+		for ticker.Wait() {
+			err := this.Loop()
+			if err != nil {
+				this.logErr("HealthCheckClusterTask", err.Error())
+			}
+		}
+	})
+	this.ticker = ticker
+}
+
+// Stop 停止
+func (this *HealthCheckClusterTask) Stop() {
+	if this.ticker == nil {
+		return
+	}
+	this.ticker.Stop()
+	this.ticker = nil
+}
+
+// Loop 单个循环任务
+func (this *HealthCheckClusterTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 开始运行
+	var executor = NewHealthCheckExecutor(this.clusterId)
+	results, err := executor.Run()
+	if err != nil {
+		return err
+	}
+
+	var failedResults = []maps.Map{}
+	for _, result := range results {
+		if !result.IsOk {
+			failedResults = append(failedResults, maps.Map{
+				"node": maps.Map{
+					"id":   result.Node.Id,
+					"name": result.Node.Name,
+				},
+				"isOk":     false,
+				"error":    result.Error,
+				"nodeAddr": result.NodeAddr,
+			})
+		}
+	}
+
+	if len(failedResults) > 0 {
+		// 10分钟内不重复提醒
+		if time.Since(this.notifiedTime) > 10*time.Minute {
+			this.notifiedTime = time.Now()
+
+			failedResultsJSON, err := json.Marshal(failedResults)
+			if err != nil {
+				return err
+			}
+			var subject = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题"
+			var message = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题："
+			var failedDescriptions = []string{}
+			var failedIndex int
+			for _, result := range results {
+				if result.IsOk || result.Node == nil {
+					continue
+				}
+				failedIndex++
+				failedDescriptions = append(failedDescriptions, "节点"+types.String(failedIndex)+"："+result.Node.Name+"，IP："+result.NodeAddr)
+			}
+
+			const maxNodeDescriptions = 10
+			var isOverMax = false
+			if len(failedDescriptions) > maxNodeDescriptions {
+				failedDescriptions = failedDescriptions[:maxNodeDescriptions]
+				isOverMax = true
+			}
+			message += strings.Join(failedDescriptions, "；")
+			if isOverMax {
+				message += " ..."
+			} else {
+				message += "。"
+			}
+
+			err = models.NewMessageDAO().CreateClusterMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, models.MessageTypeHealthCheckFailed, models.MessageLevelError, subject, subject, message, failedResultsJSON)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Config 获取当前配置
+func (this *HealthCheckClusterTask) Config() *serverconfigs.HealthCheckConfig {
+	return this.config
+}

EdgeAPI/internal/tasks/health_check_cluster_task_test.go

@@ -0,0 +1,17 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+)
+
+func TestHealthCheckClusterTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+	var task = tasks.NewHealthCheckClusterTask(10, nil)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/health_check_executor.go

@@ -0,0 +1,335 @@
+package tasks
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iputils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type HealthCheckExecutor struct {
+	BaseTask
+
+	clusterId int64
+}
+
+func NewHealthCheckExecutor(clusterId int64) *HealthCheckExecutor {
+	return &HealthCheckExecutor{clusterId: clusterId}
+}
+
+func (this *HealthCheckExecutor) Run() ([]*HealthCheckResult, error) {
+	cluster, err := models.NewNodeClusterDAO().FindEnabledNodeCluster(nil, this.clusterId)
+	if err != nil {
+		return nil, err
+	}
+	if cluster == nil || !cluster.IsOn {
+		// 如果节点已经被删除，则不提示错误
+		return nil, nil
+	}
+	if !cluster.HealthCheck.IsNotNull() {
+		return nil, errors.New("health check config is not found")
+	}
+
+	var healthCheckConfig = &serverconfigs.HealthCheckConfig{}
+	err = json.Unmarshal(cluster.HealthCheck, healthCheckConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	var results = []*HealthCheckResult{}
+
+	// 查询集群下的节点
+	nodes, err := models.NewNodeDAO().FindAllEnabledNodesWithClusterId(nil, this.clusterId, false)
+	if err != nil {
+		return nil, err
+	}
+	if len(nodes) == 0 {
+		return results, nil
+	}
+
+	var tx *dbs.Tx
+	for _, node := range nodes {
+		if !node.IsOn || node.IsBackupForCluster || node.IsBackupForGroup || (len(node.OfflineDay) > 0 && node.OfflineDay < timeutil.Format("Ymd")) {
+			continue
+		}
+
+		ipAddrs, err := models.SharedNodeIPAddressDAO.FindNodeAccessIPAddresses(tx, int64(node.Id), nodeconfigs.NodeRoleNode)
+		if err != nil {
+			return nil, err
+		}
+		for _, ipAddr := range ipAddrs {
+			var ipClusterIds = ipAddr.DecodeClusterIds()
+			if len(ipClusterIds) > 0 && !lists.ContainsInt64(ipClusterIds, this.clusterId) {
+				continue
+			}
+
+			// TODO 支持备用IP
+			var result = &HealthCheckResult{
+				Node:       node,
+				NodeAddrId: int64(ipAddr.Id),
+				NodeAddr:   ipAddr.Ip,
+			}
+
+			results = append(results, result)
+		}
+	}
+
+	// 并行检查
+	var preparedResults = []*HealthCheckResult{}
+	for _, result := range results {
+		if len(result.NodeAddr) > 0 {
+			preparedResults = append(preparedResults, result)
+		}
+	}
+
+	if len(preparedResults) == 0 {
+		return results, nil
+	}
+
+	var countResults = len(preparedResults)
+	var queue = make(chan *HealthCheckResult, countResults)
+	for _, result := range preparedResults {
+		queue <- result
+	}
+
+	var countTries = types.Int(healthCheckConfig.CountTries)
+	if countTries > 10 { // 限定最多尝试10次 TODO 应该在管理界面提示用户
+		countTries = 10
+	}
+	if countTries < 1 {
+		countTries = 3
+	}
+
+	var tryDelay = 1 * time.Second
+	if healthCheckConfig.TryDelay != nil {
+		tryDelay = healthCheckConfig.TryDelay.Duration()
+
+		if tryDelay > 1*time.Minute { // 最多不能超过1分钟 TODO 应该在管理界面提示用户
+			tryDelay = 1 * time.Minute
+		}
+	}
+
+	var concurrent = 128
+
+	var wg = sync.WaitGroup{}
+	wg.Add(countResults)
+	for i := 0; i < concurrent; i++ {
+		go func() {
+			for {
+				select {
+				case result := <-queue:
+					this.runNode(healthCheckConfig, result, countTries, tryDelay)
+					wg.Done()
+				default:
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+
+	return results, nil
+}
+
+func (this *HealthCheckExecutor) runNode(healthCheckConfig *serverconfigs.HealthCheckConfig, result *HealthCheckResult, countTries int, tryDelay time.Duration) {
+	for i := 1; i <= countTries; i++ {
+		var before = time.Now()
+		err := this.runNodeOnce(healthCheckConfig, result)
+		result.CostMs = time.Since(before).Seconds() * 1000
+		if err != nil {
+			result.Error = err.Error()
+		}
+		if result.IsOk {
+			break
+		}
+		if tryDelay > 0 {
+			time.Sleep(tryDelay)
+		}
+	}
+
+	// 修改节点IP状态
+	if teaconst.IsPlus {
+		isChanged, err := models.SharedNodeIPAddressDAO.UpdateAddressHealthCount(nil, result.NodeAddrId, result.IsOk, healthCheckConfig.CountUp, healthCheckConfig.CountDown, healthCheckConfig.AutoDown)
+		if err != nil {
+			this.logErr("HealthCheckExecutor", err.Error())
+			return
+		}
+
+		if isChanged {
+			// 在线状态发生变化
+			if healthCheckConfig.AutoDown {
+				// 发送消息
+				var message string
+				var messageType string
+				var messageLevel string
+				if result.IsOk {
+					message = "健康检查成功，节点\"" + result.Node.Name + "\"，IP\"" + result.NodeAddr + "\"已恢复上线"
+					messageType = models.MessageTypeHealthCheckNodeUp
+					messageLevel = models.MessageLevelSuccess
+				} else {
+					message = "健康检查失败，节点\"" + result.Node.Name + "\"，IP\"" + result.NodeAddr + "\"已自动下线"
+					messageType = models.MessageTypeHealthCheckNodeDown
+					messageLevel = models.MessageLevelError
+				}
+
+				err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), messageType, messageLevel, message, message, nil, false)
+				if err != nil {
+					this.logErr("HealthCheckExecutor", err.Error())
+					return
+				}
+
+				// 触发节点动作
+				if !result.IsOk {
+					err := this.fireNodeActions(int64(result.Node.Id))
+					if err != nil {
+						this.logErr("HealthCheckExecutor", err.Error())
+					}
+					return
+				}
+			}
+
+			// 触发阈值
+			err = models.SharedNodeIPAddressDAO.FireThresholds(nil, nodeconfigs.NodeRoleNode, int64(result.Node.Id))
+			if err != nil {
+				this.logErr("HealthCheckExecutor", err.Error())
+				return
+			}
+		}
+
+		// 结束处理 ，因为我们只处理IP的上下线，不修改节点的状态
+		if healthCheckConfig.AutoDown {
+			return
+		}
+	}
+
+	// 修改节点状态
+	if healthCheckConfig.AutoDown {
+		isChanged, err := models.SharedNodeDAO.UpdateNodeUpCount(nil, int64(result.Node.Id), result.IsOk, healthCheckConfig.CountUp, healthCheckConfig.CountDown)
+		if err != nil {
+			this.logErr("HealthCheckExecutor", err.Error())
+		} else if isChanged {
+			// 通知恢复或下线
+			if result.IsOk {
+				var message = "健康检查成功，节点\"" + result.Node.Name + "\"已恢复上线"
+				err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeUp, models.MessageLevelSuccess, message, message, nil, false)
+			} else {
+				var message = "健康检查失败，节点\"" + result.Node.Name + "\"已自动下线"
+				err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeDown, models.MessageLevelError, message, message, nil, false)
+			}
+			if err != nil {
+				this.logErr("HealthCheckExecutor", err.Error())
+				return
+			}
+		}
+	} else {
+		// 通知健康检查结果
+		var err error
+		if result.IsOk {
+			message := "节点\"" + result.Node.Name + "\"健康检查成功"
+			err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeUp, models.MessageLevelSuccess, message, message, nil, false)
+		} else {
+			message := "节点\"" + result.Node.Name + "\"健康检查失败"
+			err = models.NewMessageDAO().CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, int64(result.Node.Id), models.MessageTypeHealthCheckNodeDown, models.MessageLevelError, message, message, nil, false)
+		}
+		if err != nil {
+			this.logErr("HealthCheckExecutor", err.Error())
+			return
+		}
+	}
+}
+
+// 检查单个节点
+func (this *HealthCheckExecutor) runNodeOnce(healthCheckConfig *serverconfigs.HealthCheckConfig, result *HealthCheckResult) error {
+	// 支持IPv6
+	if iputils.IsIPv6(result.NodeAddr) {
+		result.NodeAddr = configutils.QuoteIP(result.NodeAddr)
+	}
+
+	if len(healthCheckConfig.URL) == 0 {
+		healthCheckConfig.URL = "http://${host}/"
+	}
+
+	var url = strings.ReplaceAll(healthCheckConfig.URL, "${host}", result.NodeAddr)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	req.Close = true
+	if len(healthCheckConfig.UserAgent) > 0 {
+		req.Header.Set("User-Agent", healthCheckConfig.UserAgent)
+	} else {
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36")
+	}
+
+	key, err := nodeutils.Base64EncodeMap(maps.Map{
+		"onlyBasicRequest": healthCheckConfig.OnlyBasicRequest,
+		"accessLogIsOn":    healthCheckConfig.AccessLogIsOn,
+	})
+	if err != nil {
+		return err
+	}
+	req.Header.Set(serverconfigs.HealthCheckHeaderName, key)
+
+	var timeout = 5 * time.Second
+	if healthCheckConfig.Timeout != nil {
+		timeout = healthCheckConfig.Timeout.Duration()
+	}
+
+	var client = &http.Client{
+		Timeout: timeout,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				_, port, err := net.SplitHostPort(addr)
+				if err != nil {
+					return nil, err
+				}
+				return net.DialTimeout(network, configutils.QuoteIP(result.NodeAddr)+":"+port, timeout)
+			},
+			MaxIdleConns:          3,
+			MaxIdleConnsPerHost:   3,
+			MaxConnsPerHost:       3,
+			IdleConnTimeout:       10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSHandshakeTimeout:   0,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	defer func() {
+		client.CloseIdleConnections()
+	}()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	_ = resp.Body.Close()
+
+	if len(healthCheckConfig.StatusCodes) > 0 && !lists.ContainsInt(healthCheckConfig.StatusCodes, resp.StatusCode) {
+		result.Error = "invalid response status code '" + strconv.Itoa(resp.StatusCode) + "'"
+		return nil
+	}
+
+	result.IsOk = true
+
+	return nil
+}

EdgeAPI/internal/tasks/health_check_executor_ext.go

@@ -0,0 +1,9 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build !plus
+
+package tasks
+
+// 触发节点动作
+func (this *HealthCheckExecutor) fireNodeActions(nodeId int64) error {
+	return nil
+}

EdgeAPI/internal/tasks/health_check_executor_ext_plus.go

@@ -0,0 +1,14 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+)
+
+// 触发节点动作
+func (this *HealthCheckExecutor) fireNodeActions(nodeId int64) error {
+	return models.SharedNodeDAO.FireNodeActions(nil, nodeId, nodeconfigs.NodeActionParamHealthCheckFailure, nil)
+}

EdgeAPI/internal/tasks/health_check_executor_test.go

@@ -0,0 +1,25 @@
+//go:build plus
+// +build plus
+
+package tasks_test
+
+import (
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+)
+
+func TestHealthCheckExecutor_Run(t *testing.T) {
+	teaconst.IsPlus = true
+	dbs.NotifyReady()
+
+	var executor = tasks.NewHealthCheckExecutor(42)
+	results, err := executor.Run()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, result := range results {
+		t.Log(result.Node.Name, "addr:", result.NodeAddr, "isOk:", result.IsOk, "error:", result.Error)
+	}
+}

EdgeAPI/internal/tasks/health_check_result.go

@@ -0,0 +1,13 @@
+package tasks
+
+import "github.com/TeaOSLab/EdgeAPI/internal/db/models"
+
+// HealthCheckResult 健康检查结果
+type HealthCheckResult struct {
+	Node       *models.Node
+	NodeAddr   string // 节点IP地址
+	NodeAddrId int64  // 节点IP地址ID
+	IsOk       bool
+	Error      string
+	CostMs     float64
+}

EdgeAPI/internal/tasks/health_check_task.go

@@ -0,0 +1,136 @@
+package tasks
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewHealthCheckTask(1 * time.Minute).Start()
+		})
+	})
+}
+
+// HealthCheckTask 节点健康检查任务
+type HealthCheckTask struct {
+	BaseTask
+
+	ticker   *time.Ticker
+	tasksMap map[int64]*HealthCheckClusterTask // taskId => task
+}
+
+func NewHealthCheckTask(duration time.Duration) *HealthCheckTask {
+	return &HealthCheckTask{
+		ticker:   time.NewTicker(duration),
+		tasksMap: map[int64]*HealthCheckClusterTask{},
+	}
+}
+
+func (this *HealthCheckTask) Start() {
+	err := this.Loop()
+	if err != nil {
+		this.logErr("HealthCheckTask", err.Error())
+	}
+
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("HealthCheckTask", err.Error())
+		}
+	}
+}
+
+func (this *HealthCheckTask) Loop() error {
+	clusters, err := models.NewNodeClusterDAO().FindAllEnableClusters(nil)
+	if err != nil {
+		return err
+	}
+	clusterIds := []int64{}
+	for _, cluster := range clusters {
+		clusterIds = append(clusterIds, int64(cluster.Id))
+	}
+
+	// 停掉删除的
+	for clusterId, task := range this.tasksMap {
+		if !lists.ContainsInt64(clusterIds, clusterId) {
+			task.Stop()
+			delete(this.tasksMap, clusterId)
+		}
+	}
+
+	// 启动新的或更新老的
+	for _, cluster := range clusters {
+		var clusterId = int64(cluster.Id)
+
+		if !cluster.IsOn {
+			this.stopClusterTask(clusterId)
+			continue
+		}
+
+		// 检查当前集群上是否有服务，如果尚没有部署服务，则直接跳过
+		countServers, err := models.SharedServerDAO.CountAllEnabledServersWithNodeClusterId(nil, clusterId)
+		if err != nil {
+			return err
+		}
+		if countServers == 0 {
+			this.stopClusterTask(clusterId)
+			continue
+		}
+
+		var config = &serverconfigs.HealthCheckConfig{}
+		if len(cluster.HealthCheck) > 0 {
+			err = json.Unmarshal(cluster.HealthCheck, config)
+			if err != nil {
+				this.logErr("HealthCheckTask", err.Error())
+				this.stopClusterTask(clusterId)
+				continue
+			}
+			if !config.IsOn {
+				this.stopClusterTask(clusterId)
+				continue
+			}
+		} else {
+			this.stopClusterTask(clusterId)
+			continue
+		}
+
+		task, ok := this.tasksMap[clusterId]
+		if ok {
+			// 检查是否有变化
+			newJSON, _ := json.Marshal(config)
+			oldJSON, _ := json.Marshal(task.Config())
+			if !bytes.Equal(oldJSON, newJSON) {
+				remotelogs.Println("TASK", "[HealthCheckTask]update cluster '"+numberutils.FormatInt64(clusterId)+"'")
+				goman.New(func() {
+					task.Reset(config)
+				})
+			}
+		} else {
+			task = NewHealthCheckClusterTask(clusterId, config)
+			this.tasksMap[clusterId] = task
+			goman.New(func() {
+				task.Run()
+			})
+		}
+	}
+
+	return nil
+}
+
+func (this *HealthCheckTask) stopClusterTask(clusterId int64) {
+	var task = this.tasksMap[clusterId]
+	if task != nil {
+		task.Stop()
+		delete(this.tasksMap, clusterId)
+	}
+}

EdgeAPI/internal/tasks/health_check_task_test.go

@@ -0,0 +1,17 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"testing"
+	"time"
+)
+
+func TestNewHealthCheckTask(t *testing.T) {
+	var task = tasks.NewHealthCheckTask(1 * time.Minute)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/log_task.go

@@ -0,0 +1,139 @@
+package tasks
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewLogTask(24*time.Hour, 1*time.Minute).Start()
+		})
+	})
+}
+
+type LogTask struct {
+	BaseTask
+
+	cleanTicker   *time.Ticker
+	monitorTicker *time.Ticker
+}
+
+func NewLogTask(cleanDuration time.Duration, monitorDuration time.Duration) *LogTask {
+	return &LogTask{
+		cleanTicker:   time.NewTicker(cleanDuration),
+		monitorTicker: time.NewTicker(monitorDuration),
+	}
+}
+
+func (this *LogTask) Start() {
+	goman.New(func() {
+		this.RunClean()
+	})
+	goman.New(func() {
+		this.RunMonitor()
+	})
+}
+
+func (this *LogTask) RunClean() {
+	for range this.cleanTicker.C {
+		err := this.LoopClean()
+		if err != nil {
+			this.logErr("LogTask", err.Error())
+		}
+	}
+}
+
+func (this *LogTask) LoopClean() error {
+	var configKey = "adminLogConfig"
+	valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
+	if err != nil {
+		return err
+	}
+	if len(valueJSON) == 0 {
+		return nil
+	}
+
+	var config = &systemconfigs.LogConfig{}
+	err = json.Unmarshal(valueJSON, config)
+	if err != nil {
+		return err
+	}
+	if config.Days > 0 {
+		err = models.SharedLogDAO.DeleteLogsPermanentlyBeforeDays(nil, config.Days)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (this *LogTask) RunMonitor() {
+	for range this.monitorTicker.C {
+		err := this.LoopMonitor()
+		if err != nil {
+			this.logErr("LogTask", err.Error())
+		}
+	}
+}
+
+func (this *LogTask) LoopMonitor() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var configKey = "adminLogConfig"
+	valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
+	if err != nil {
+		return err
+	}
+	if len(valueJSON) == 0 {
+		return nil
+	}
+
+	var config = &systemconfigs.LogConfig{}
+	err = json.Unmarshal(valueJSON, config)
+	if err != nil {
+		return err
+	}
+
+	if config.Capacity != nil {
+		capacityBytes := config.Capacity.Bytes()
+		if capacityBytes > 0 {
+			sumBytes, err := models.SharedLogDAO.SumLogsSize()
+			if err != nil {
+				return err
+			}
+			if sumBytes > capacityBytes {
+				err := models.SharedMessageDAO.CreateMessage(nil, 0, 0, models.MessageTypeLogCapacityOverflow, models.MessageLevelError, "日志用量已经超出最大限制", "日志用量已经超出最大限制，当前的用量为"+this.formatBytes(sumBytes)+"，而设置的最大容量为"+this.formatBytes(capacityBytes)+"。", nil)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *LogTask) formatBytes(bytes int64) string {
+	sizeHuman := ""
+	if bytes < 1024 {
+		sizeHuman = numberutils.FormatInt64(bytes) + "字节"
+	} else if bytes < 1024*1024 {
+		sizeHuman = fmt.Sprintf("%.2fK", float64(bytes)/1024)
+	} else if bytes < 1024*1024*1024 {
+		sizeHuman = fmt.Sprintf("%.2fM", float64(bytes)/1024/1024)
+	} else {
+		sizeHuman = fmt.Sprintf("%.2fG", float64(bytes)/1024/1024/1024)
+	}
+	return sizeHuman
+}

EdgeAPI/internal/tasks/log_task_test.go

@@ -0,0 +1,30 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestLogTask_LoopClean(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewLogTask(24*time.Hour, 1*time.Minute)
+	err := task.LoopClean()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestLogTask_LoopMonitor(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewLogTask(24*time.Hour, 1*time.Minute)
+	err := task.LoopMonitor()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/message_task.go

@@ -0,0 +1,46 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewMessageTask(24 * time.Hour).Start()
+		})
+	})
+}
+
+// MessageTask 消息相关任务
+type MessageTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+// NewMessageTask 获取新对象
+func NewMessageTask(duration time.Duration) *MessageTask {
+	return &MessageTask{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+// Start 开始运行
+func (this *MessageTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("MessageTask", err.Error())
+		}
+	}
+}
+
+// Loop 单次运行
+func (this *MessageTask) Loop() error {
+	dayTime := time.Now().AddDate(0, 0, -30) // TODO 这个30天应该可以在界面上设置
+	return models.NewMessageDAO().DeleteMessagesBeforeDay(nil, dayTime)
+}

EdgeAPI/internal/tasks/monitor_item_value_task.go

@@ -0,0 +1,51 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewMonitorItemValueTask(1 * time.Hour).Start()
+		})
+	})
+}
+
+// MonitorItemValueTask 节点监控数值任务
+type MonitorItemValueTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+// NewMonitorItemValueTask 获取新对象
+func NewMonitorItemValueTask(duration time.Duration) *MonitorItemValueTask {
+	var ticker = time.NewTicker(duration)
+	if Tea.IsTesting() {
+		ticker = time.NewTicker(1 * time.Minute)
+	}
+
+	return &MonitorItemValueTask{
+		ticker: ticker,
+	}
+}
+
+func (this *MonitorItemValueTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("MonitorItemValueTask", err.Error())
+		}
+	}
+}
+
+func (this *MonitorItemValueTask) Loop() error {
+	return models.SharedNodeValueDAO.Clean(nil)
+}

EdgeAPI/internal/tasks/monitor_item_value_task_test.go

@@ -0,0 +1,21 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestMonitorItemValueTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewMonitorItemValueTask(1 * time.Minute)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/node_log_cleaner_task.go

@@ -0,0 +1,49 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewNodeLogCleanerTask(24 * time.Hour).Start()
+		})
+	})
+}
+
+// NodeLogCleanerTask 清理节点日志的任务
+type NodeLogCleanerTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewNodeLogCleanerTask(duration time.Duration) *NodeLogCleanerTask {
+	return &NodeLogCleanerTask{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *NodeLogCleanerTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("NodeLogCleanerTask", err.Error())
+		}
+	}
+}
+
+func (this *NodeLogCleanerTask) Loop() error {
+	// 删除 N天 以前的info日志
+	err := models.SharedNodeLogDAO.DeleteExpiredLogsWithLevel(nil, "info", 3)
+	if err != nil {
+		return err
+	}
+
+	// TODO 7天这个数值改成可以设置
+	return models.SharedNodeLogDAO.DeleteExpiredLogs(nil, 7)
+}

EdgeAPI/internal/tasks/node_log_cleaner_task_test.go

@@ -0,0 +1,19 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNodeLogCleaner_loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var cleaner = tasks.NewNodeLogCleanerTask(24 * time.Hour)
+	err := cleaner.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("OK")
+}

EdgeAPI/internal/tasks/node_monitor_task.go

@@ -0,0 +1,168 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/installers"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewNodeMonitorTask(1 * time.Minute).Start()
+		})
+	})
+}
+
+// 节点启动尝试
+type nodeStartingTry struct {
+	count     int
+	timestamp int64
+}
+
+// NodeMonitorTask 边缘节点监控任务
+type NodeMonitorTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+
+	inactiveMap map[string]int  // cluster@nodeId => count
+	notifiedMap map[int64]int64 // nodeId => timestamp
+
+	recoverMap map[int64]*nodeStartingTry // nodeId => *nodeStartingTry
+}
+
+func NewNodeMonitorTask(duration time.Duration) *NodeMonitorTask {
+	return &NodeMonitorTask{
+		ticker:      time.NewTicker(duration),
+		inactiveMap: map[string]int{},
+		notifiedMap: map[int64]int64{},
+		recoverMap:  map[int64]*nodeStartingTry{},
+	}
+}
+
+func (this *NodeMonitorTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("NodeMonitorTask", err.Error())
+		}
+	}
+}
+
+func (this *NodeMonitorTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	clusters, err := models.SharedNodeClusterDAO.FindAllEnableClusters(nil)
+	if err != nil {
+		return err
+	}
+	for _, cluster := range clusters {
+		err := this.MonitorCluster(cluster)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *NodeMonitorTask) MonitorCluster(cluster *models.NodeCluster) error {
+	var clusterId = int64(cluster.Id)
+
+	// 检查离线节点
+	inactiveNodes, err := models.SharedNodeDAO.FindAllInactiveNodesWithClusterId(nil, clusterId)
+	if err != nil {
+		return err
+	}
+
+	// 尝试自动远程启动
+	if cluster.AutoRemoteStart {
+		var nodeQueue = installers.NewNodeQueue()
+		for _, node := range inactiveNodes {
+			var nodeId = int64(node.Id)
+			tryInfo, ok := this.recoverMap[nodeId]
+			if !ok {
+				tryInfo = &nodeStartingTry{
+					count:     1,
+					timestamp: time.Now().Unix(),
+				}
+				this.recoverMap[nodeId] = tryInfo
+			} else {
+				if tryInfo.count >= 3 /** 3次 **/ { // N 秒内超过 M 次就暂时不再重新尝试，防止阻塞当前任务
+					if tryInfo.timestamp+10*60 /** 10 分钟 **/ > time.Now().Unix() {
+						continue
+					}
+					tryInfo.timestamp = time.Now().Unix()
+					tryInfo.count = 0
+				}
+				tryInfo.count++
+			}
+
+			// TODO 如果用户手工安装的位置不在标准位置，需要节点自身记住最近启动的位置
+			err = nodeQueue.StartNode(nodeId)
+			if err != nil {
+				if !installers.IsGrantError(err) {
+					_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleNode, nodeId, 0, 0, models.LevelInfo, "NODE", "start node from remote API failed: "+err.Error(), time.Now().Unix(), "", nil)
+				}
+			} else {
+				_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleNode, nodeId, 0, 0, models.LevelSuccess, "NODE", "start node from remote API successfully", time.Now().Unix(), "", nil)
+			}
+		}
+	}
+
+	var nodeMap = map[int64]*models.Node{} // nodeId => Node
+	for _, node := range inactiveNodes {
+		var nodeId = int64(node.Id)
+		nodeMap[nodeId] = node
+		this.inactiveMap[types.String(clusterId)+"@"+types.String(nodeId)]++
+	}
+
+	const maxInactiveTries = 5
+
+	// 处理现有的离线状态
+	for key, count := range this.inactiveMap {
+		var pieces = strings.Split(key, "@")
+		if pieces[0] != types.String(clusterId) {
+			continue
+		}
+		var nodeId = types.Int64(pieces[1])
+		node, ok := nodeMap[nodeId]
+		if ok {
+			// 连续 N 次离线发送通知
+			// 同时也要确保两次发送通知的时间不会过近
+			if count >= maxInactiveTries && time.Now().Unix()-this.notifiedMap[nodeId] > 3600 {
+				this.inactiveMap[key] = 0
+				this.notifiedMap[nodeId] = time.Now().Unix()
+
+				var subject = "节点\"" + node.Name + "\"已处于离线状态"
+				var msg = "集群 \"" + cluster.Name + "\" 节点 \"" + node.Name + "\" 已处于离线状态，请检查节点是否异常"
+				err = models.SharedMessageDAO.CreateNodeMessage(nil, nodeconfigs.NodeRoleNode, clusterId, int64(node.Id), models.MessageTypeNodeInactive, models.LevelError, subject, msg, nil, false)
+				if err != nil {
+					return err
+				}
+
+				// 设置通知时间
+				err = models.SharedNodeDAO.UpdateNodeInactiveNotifiedAt(nil, nodeId, time.Now().Unix())
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			delete(this.inactiveMap, key)
+		}
+	}
+
+	// 检查CPU、内存、硬盘不足节点
+	// TODO 需要实现
+
+	return nil
+}

EdgeAPI/internal/tasks/node_monitor_task_test.go

@@ -0,0 +1,33 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNodeMonitorTask_loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewNodeMonitorTask(60 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestNodeMonitorTask_Monitor(t *testing.T) {
+	dbs.NotifyReady()
+	var task = tasks.NewNodeMonitorTask(60 * time.Second)
+	for i := 0; i < 5; i++ {
+		err := task.MonitorCluster(&models.NodeCluster{
+			Id: 42,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}

EdgeAPI/internal/tasks/node_schedule_task_plus.go

@@ -0,0 +1,77 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewNodeScheduleTask(1 * time.Minute).Start()
+		})
+	})
+}
+
+// NodeScheduleTask 节点调度任务
+type NodeScheduleTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewNodeScheduleTask(duration time.Duration) *NodeScheduleTask {
+	return &NodeScheduleTask{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *NodeScheduleTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("NodeScheduleTask", err.Error())
+		}
+	}
+}
+
+func (this *NodeScheduleTask) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var tx *dbs.Tx
+
+	// 恢复动作过期的节点
+	err := this.recoverNodes(tx)
+	if err != nil {
+		return err
+	}
+
+	// 下线租期到期的节点
+	err = models.SharedNodeDAO.NotifyNodesWithOfflineDay(tx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *NodeScheduleTask) recoverNodes(tx *dbs.Tx) error {
+	nodeIds, err := models.SharedNodeDAO.FindNodeIdsWithExpiredActions(tx)
+	if err != nil {
+		return err
+	}
+	for _, nodeId := range nodeIds {
+		err = models.SharedNodeDAO.ResetNodeActionStatus(tx, nodeId, true)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

EdgeAPI/internal/tasks/node_schedule_task_plus_test.go

@@ -0,0 +1,20 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNewNodeScheduleTask(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = NewNodeScheduleTask(1 * time.Minute)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/node_task_extractor.go

@@ -0,0 +1,57 @@
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewNodeTaskExtractor(10 * time.Second).Start()
+		})
+	})
+}
+
+// NodeTaskExtractor 节点任务
+type NodeTaskExtractor struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewNodeTaskExtractor(duration time.Duration) *NodeTaskExtractor {
+	return &NodeTaskExtractor{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *NodeTaskExtractor) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("NodeTaskExtractor", err.Error())
+		}
+	}
+}
+
+func (this *NodeTaskExtractor) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 这里不解锁，是为了让任务N秒钟之内只运行一次
+
+	for _, role := range []string{nodeconfigs.NodeRoleNode, nodeconfigs.NodeRoleDNS} {
+		err := models.SharedNodeTaskDAO.ExtractAllClusterTasks(nil, role)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/ns_node_monitor_task_plus.go

@@ -0,0 +1,174 @@
+//go:build plus
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/installers"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewNSNodeMonitorTask(1 * time.Minute).Start()
+		})
+	})
+}
+
+// 节点启动尝试
+type nsNodeStartingTry struct {
+	count     int
+	timestamp int64
+}
+
+// NSNodeMonitorTask 边缘节点监控任务
+type NSNodeMonitorTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+
+	inactiveMap map[string]int  // cluster@nodeId => count
+	notifiedMap map[int64]int64 // nodeId => timestamp
+
+	recoverMap map[int64]*nsNodeStartingTry // nodeId => *nsNodeStartingTry
+}
+
+func NewNSNodeMonitorTask(duration time.Duration) *NSNodeMonitorTask {
+	return &NSNodeMonitorTask{
+		ticker:      time.NewTicker(duration),
+		inactiveMap: map[string]int{},
+		notifiedMap: map[int64]int64{},
+		recoverMap:  map[int64]*nsNodeStartingTry{},
+	}
+}
+
+func (this *NSNodeMonitorTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("NS_NODE_MONITOR", err.Error())
+		}
+	}
+}
+
+func (this *NSNodeMonitorTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	clusters, err := models.SharedNSClusterDAO.FindAllEnabledClusters(nil)
+	if err != nil {
+		return err
+	}
+	for _, cluster := range clusters {
+		err := this.monitorCluster(cluster)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *NSNodeMonitorTask) monitorCluster(cluster *models.NSCluster) error {
+	var clusterId = int64(cluster.Id)
+
+	// 检查离线节点
+	inactiveNodes, err := models.SharedNSNodeDAO.FindAllNotifyingInactiveNodesWithClusterId(nil, clusterId)
+	if err != nil {
+		return err
+	}
+
+	// 尝试自动远程启动
+	if cluster.AutoRemoteStart {
+		var nodeQueue = installers.NewNSNodeQueue()
+		for _, node := range inactiveNodes {
+			var nodeId = int64(node.Id)
+			tryInfo, ok := this.recoverMap[nodeId]
+			if !ok {
+				tryInfo = &nsNodeStartingTry{
+					count:     1,
+					timestamp: time.Now().Unix(),
+				}
+				this.recoverMap[nodeId] = tryInfo
+			} else {
+				if tryInfo.count >= 3 /** 3次 **/ { // N 秒内超过 M 次就暂时不再重新尝试，防止阻塞当前任务
+					if tryInfo.timestamp+10*60 /** 10 分钟 **/ > time.Now().Unix() {
+						continue
+					}
+					tryInfo.timestamp = time.Now().Unix()
+					tryInfo.count = 0
+				}
+				tryInfo.count++
+			}
+
+			// TODO 如果用户手工安装的位置不在标准位置，需要节点自身记住最近启动的位置
+			err = nodeQueue.StartNode(nodeId)
+			if err != nil {
+				if !installers.IsGrantError(err) {
+					_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDNS, nodeId, 0, 0, models.LevelError, "NODE", "start node from remote API failed: "+err.Error(), time.Now().Unix(), "", nil)
+				}
+			} else {
+				_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDNS, nodeId, 0, 0, models.LevelSuccess, "NODE", "start node from remote API successfully", time.Now().Unix(), "", nil)
+			}
+		}
+	}
+
+	var nodeMap = map[int64]*models.NSNode{}
+	for _, node := range inactiveNodes {
+		var nodeId = int64(node.Id)
+		nodeMap[nodeId] = node
+		this.inactiveMap[types.String(clusterId)+"@"+types.String(nodeId)]++
+	}
+
+	const maxInactiveTries = 5
+
+	// 处理现有的离线状态
+	for key, count := range this.inactiveMap {
+		var pieces = strings.Split(key, "@")
+		if pieces[0] != types.String(clusterId) {
+			continue
+		}
+		var nodeId = types.Int64(pieces[1])
+		node, ok := nodeMap[nodeId]
+		if ok {
+			// 连续 N 次离线发送通知
+			// 同时也要确保两次发送通知的时间不会过近
+			if count >= maxInactiveTries && time.Now().Unix()-this.notifiedMap[nodeId] > 3600 {
+				this.inactiveMap[key] = 0
+				this.notifiedMap[nodeId] = time.Now().Unix()
+
+				var subject = "DNS节点\"" + node.Name + "\"已处于离线状态"
+				var msg = "DNS节点\"" + node.Name + "\"已处于离线状态"
+				err = models.SharedMessageDAO.CreateNodeMessage(nil, nodeconfigs.NodeRoleDNS, clusterId, int64(node.Id), models.MessageTypeNSNodeInactive, models.LevelError, subject, msg, nil, false)
+				if err != nil {
+					return err
+				}
+
+				// 修改在线状态
+				err = models.SharedNSNodeDAO.UpdateNodeStatusIsNotified(nil, int64(node.Id))
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			delete(this.inactiveMap, key)
+		}
+	}
+
+	// TODO 检查恢复连接
+
+	// 检查CPU、内存、硬盘不足节点，而且离线的节点不再重复提示
+	// TODO 需要实现
+
+	// TODO 检查53/tcp、53/udp是否能够访问
+
+	return nil
+}

EdgeAPI/internal/tasks/ns_records_health_check_task_plus.go

@@ -0,0 +1,298 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/nameservers"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	"net"
+	"runtime"
+	"sync"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			var duration = 3 * time.Minute
+			if Tea.IsTesting() { // 测试专用
+				duration = 1 * time.Minute
+			}
+			NewNSRecordsHealthCheckTask(duration).Start()
+		})
+	})
+}
+
+type nsRecordHealthCheckTask struct {
+	domainId int64
+
+	recordName      string
+	recordType      string
+	recordId        int64
+	recordIP        string
+	recordCountUp   int
+	recordCountDown int
+	recordIsUp      bool
+
+	domainConfig *dnsconfigs.NSRecordsHealthCheckConfig
+	recordConfig *dnsconfigs.NSRecordHealthCheckConfig
+}
+
+// NSRecordsHealthCheckTask NS记录健康检查任务
+type NSRecordsHealthCheckTask struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewNSRecordsHealthCheckTask(duration time.Duration) *NSRecordsHealthCheckTask {
+	return &NSRecordsHealthCheckTask{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *NSRecordsHealthCheckTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("NS_RECORDS_HEALTH_CHECK_TASK", err.Error())
+		}
+	}
+}
+
+func (this *NSRecordsHealthCheckTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 基础用户设置
+	var tx *dbs.Tx
+	userSetting, err := models.SharedSysSettingDAO.ReadNSUserConfig(tx)
+	if err != nil {
+		return err
+	}
+	var canCheckForBasicUser = userSetting.DefaultPlanConfig != nil && userSetting.DefaultPlanConfig.SupportHealthCheck
+
+	// 套餐设置
+	plans, err := nameservers.SharedNSPlanDAO.FindAllEnabledPlans(tx)
+	if err != nil {
+		return err
+	}
+	var canCheckPlanMap = map[int64]bool{} // planId => canCheck
+	var queryPlanIds = []int64{}
+	for _, plan := range plans {
+		if !plan.IsOn || len(plan.Config) == 0 {
+			continue
+		}
+
+		var planConfig = dnsconfigs.DefaultNSPlanConfig()
+		err = json.Unmarshal(plan.Config, planConfig)
+		if err != nil {
+			return err
+		}
+		if planConfig.SupportHealthCheck {
+			canCheckPlanMap[int64(plan.Id)] = true
+
+			if !canCheckForBasicUser {
+				queryPlanIds = append(queryPlanIds, int64(plan.Id))
+			}
+		}
+	}
+	if !canCheckForBasicUser && len(canCheckPlanMap) == 0 {
+		// 没有开放任何健康检查，则直接返回
+		return nil
+	}
+
+	// 查询域名
+	domains, err := nameservers.SharedNSDomainDAO.FindDomainsWithHealthCheckOn(tx, queryPlanIds)
+	if err != nil {
+		return err
+	}
+
+	// 记录队列
+	var recordTasks = []*nsRecordHealthCheckTask{}
+
+	for _, domain := range domains {
+		var domainHealthCheckConfig = dnsconfigs.NewNSRecordsHealthCheckConfig()
+		err = json.Unmarshal(domain.RecordsHealthCheck, domainHealthCheckConfig)
+		if err != nil {
+			return err
+		}
+		err = domainHealthCheckConfig.Init()
+		if err != nil {
+			return err
+		}
+
+		// 列出记录
+		records, err := nameservers.SharedNSRecordDAO.FindAllDomainRecordsWithHealthCheckOn(tx, int64(domain.Id))
+		if err != nil {
+			return err
+		}
+		for _, record := range records {
+			var recordHealthCheckConfig = dnsconfigs.NewNSRecordHealthCheckConfig()
+			err = json.Unmarshal(record.HealthCheck, recordHealthCheckConfig)
+			if err != nil {
+				return err
+			}
+			err = recordHealthCheckConfig.Init()
+			if err != nil {
+				return err
+			}
+
+			recordTasks = append(recordTasks, &nsRecordHealthCheckTask{
+				domainId:        int64(domain.Id),
+				recordId:        int64(record.Id),
+				recordName:      record.Name,
+				recordType:      record.Type,
+				recordIP:        record.Value,
+				recordCountUp:   int(record.CountUp),
+				recordCountDown: int(record.CountDown),
+				recordIsUp:      record.IsUp,
+				domainConfig:    domainHealthCheckConfig,
+				recordConfig:    recordHealthCheckConfig,
+			})
+		}
+	}
+
+	if len(recordTasks) == 0 {
+		return nil
+	}
+
+	var recordTaskChan = make(chan *nsRecordHealthCheckTask, len(recordTasks))
+	for _, recordTask := range recordTasks {
+		recordTaskChan <- recordTask
+	}
+	var concurrent = runtime.NumCPU() * 8 // 并发数
+	if concurrent >= 128 {
+		concurrent = 128
+	}
+	var wg = sync.WaitGroup{}
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func() {
+			defer wg.Done()
+
+			for {
+				select {
+				case recordTask := <-recordTaskChan:
+					taskErr := this.processRecordTask(recordTask)
+					if taskErr != nil {
+						remotelogs.Error("NS_RECORDS_HEALTH_CHECK_TASK", "check '"+types.String(recordTask.recordId)+" "+recordTask.recordType+" "+recordTask.recordIP+"' failed: "+err.Error())
+					}
+				default:
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+
+	return nil
+}
+
+// 处理单个记录任务
+func (this *NSRecordsHealthCheckTask) processRecordTask(recordTask *nsRecordHealthCheckTask) error {
+	var tx *dbs.Tx
+
+	var timeoutSeconds = recordTask.domainConfig.TimeoutSeconds
+	var port = recordTask.domainConfig.Port
+	var minUp = recordTask.domainConfig.CountUp
+	var minDown = recordTask.domainConfig.CountDown
+
+	if recordTask.recordConfig.TimeoutSeconds > 0 {
+		timeoutSeconds = recordTask.recordConfig.TimeoutSeconds
+	}
+	if recordTask.recordConfig.Port > 0 {
+		port = recordTask.recordConfig.Port
+	}
+	if recordTask.recordConfig.CountUp > 0 {
+		minUp = recordTask.recordConfig.CountUp
+	}
+	if recordTask.recordConfig.CountDown > 0 {
+		minDown = recordTask.recordConfig.CountDown
+	}
+
+	if timeoutSeconds <= 0 {
+		timeoutSeconds = 5
+	}
+	if port <= 0 {
+		port = 80
+	}
+	if minUp <= 0 {
+		minUp = 1
+	}
+	if minDown <= 0 {
+		minDown = 3
+	}
+
+	conn, err := net.DialTimeout("tcp", configutils.QuoteIP(recordTask.recordIP)+":"+types.String(port), time.Duration(timeoutSeconds)*time.Second)
+	var beforeIsUp = recordTask.recordIsUp
+	var beforeCountUp = recordTask.recordCountUp
+	var beforeCountDown = recordTask.recordCountDown
+
+	if err != nil {
+		err = nil // 不需要返回连接端口错误
+
+		recordTask.recordCountUp = 0
+		recordTask.recordCountDown++
+		if recordTask.recordCountDown >= minDown {
+			recordTask.recordCountDown = minDown
+			recordTask.recordIsUp = false
+		}
+
+		if beforeIsUp && !recordTask.recordIsUp {
+			// 改变状态
+			err = nameservers.SharedNSRecordDAO.UpdateRecordIsUp(tx, recordTask.recordId, recordTask.recordIsUp, recordTask.recordCountUp, recordTask.recordCountDown)
+			if err != nil {
+				return err
+			}
+
+			// TODO 发送通知，但需要避免发送过于频繁
+		} else if recordTask.recordCountDown != beforeCountDown {
+			// 计数
+			err = nameservers.SharedNSRecordDAO.UpdateRecordIsUp(tx, recordTask.recordId, recordTask.recordIsUp, recordTask.recordCountUp, recordTask.recordCountDown)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+	_ = conn.(*net.TCPConn).SetLinger(0)
+	_ = conn.Close()
+
+	recordTask.recordCountDown = 0
+	recordTask.recordCountUp++
+	if recordTask.recordCountUp >= minUp {
+		recordTask.recordCountUp = minUp
+		recordTask.recordIsUp = true
+	}
+
+	if !beforeIsUp && recordTask.recordIsUp {
+		// 改变状态
+		err = nameservers.SharedNSRecordDAO.UpdateRecordIsUp(tx, recordTask.recordId, recordTask.recordIsUp, recordTask.recordCountUp, recordTask.recordCountDown)
+		if err != nil {
+			return err
+		}
+
+		// TODO 发送通知，但需要避免发送过于频繁
+	} else if recordTask.recordCountUp != beforeCountUp {
+		// 计数
+		err = nameservers.SharedNSRecordDAO.UpdateRecordIsUp(tx, recordTask.recordId, recordTask.recordIsUp, recordTask.recordCountUp, recordTask.recordCountDown)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/ns_records_health_check_task_plus_test.go

@@ -0,0 +1,21 @@
+// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNewNSRecordsHealthCheckTask(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewNSRecordsHealthCheckTask(60 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/server_access_log_cleaner.go

@@ -0,0 +1,137 @@
+package tasks
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"regexp"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewServerAccessLogCleaner(6 * time.Hour).Start()
+		})
+	})
+}
+
+// ServerAccessLogCleaner 服务访问日志自动清理
+type ServerAccessLogCleaner struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewServerAccessLogCleaner(duration time.Duration) *ServerAccessLogCleaner {
+	return &ServerAccessLogCleaner{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+func (this *ServerAccessLogCleaner) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("[TASK][ServerAccessLogCleaner]", err.Error())
+		}
+	}
+}
+
+func (this *ServerAccessLogCleaner) Loop() error {
+	// 当前设置
+	configJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, systemconfigs.SettingCodeDatabaseConfigSetting)
+	if err != nil {
+		return err
+	}
+	if len(configJSON) == 0 {
+		return nil
+	}
+	var config = systemconfigs.NewDatabaseConfig()
+	err = json.Unmarshal(configJSON, config)
+	if err != nil {
+		return err
+	}
+	if config.ServerAccessLog.Clean.Days <= 0 {
+		return nil
+	}
+	var days = config.ServerAccessLog.Clean.Days
+	var endDay = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days+1))
+
+	// 当前连接的数据库
+	db, err := dbs.Default()
+	if err != nil {
+		return err
+	}
+	err = this.cleanDB(db, endDay)
+	if err != nil {
+		return err
+	}
+
+	// 日志数据库节点
+	nodes, err := models.SharedDBNodeDAO.FindAllEnabledAndOnDBNodes(nil)
+	if err != nil {
+		return err
+	}
+	for _, node := range nodes {
+		err := func(node *models.DBNode) error {
+			var dbConfig = node.DBConfig()
+			nodeDB, err := dbs.NewInstanceFromConfig(dbConfig)
+			if err != nil {
+				return err
+			}
+
+			defer func() {
+				_ = nodeDB.Close()
+			}()
+
+			err = this.cleanDB(nodeDB, endDay)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}(node)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (this *ServerAccessLogCleaner) cleanDB(db *dbs.DB, endDay string) error {
+	ones, columnNames, err := db.FindPreparedOnes("SHOW TABLES")
+	if err != nil {
+		return err
+	}
+	if len(columnNames) != 1 {
+		return errors.New("invalid column names: " + strings.Join(columnNames, ", "))
+	}
+	var columnName = columnNames[0]
+	var reg = regexp.MustCompile(`^(?i)(edgeHTTPAccessLogs|edgeNSAccessLogs)_(\d{8})(_\d{4})?$`)
+	for _, one := range ones {
+		var tableName = one.GetString(columnName)
+		if len(tableName) == 0 {
+			continue
+		}
+		if !reg.MatchString(tableName) {
+			continue
+		}
+		var matches = reg.FindStringSubmatch(tableName)
+		var day = matches[2]
+
+		if day < endDay {
+			_, err = db.Exec("DROP TABLE " + tableName)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

EdgeAPI/internal/tasks/server_access_log_cleaner_test.go

@@ -0,0 +1,19 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestServerAccessLogCleaner_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewServerAccessLogCleaner(24 * time.Hour)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/ssl_cert_expire_check_executor.go

@@ -0,0 +1,237 @@
+package tasks
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/acme"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewSSLCertExpireCheckExecutor(1 * time.Hour).Start()
+		})
+	})
+}
+
+// SSLCertExpireCheckExecutor 证书检查任务
+type SSLCertExpireCheckExecutor struct {
+	BaseTask
+
+	ticker *time.Ticker
+}
+
+func NewSSLCertExpireCheckExecutor(duration time.Duration) *SSLCertExpireCheckExecutor {
+	return &SSLCertExpireCheckExecutor{
+		ticker: time.NewTicker(duration),
+	}
+}
+
+// Start 启动任务
+func (this *SSLCertExpireCheckExecutor) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("SSLCertExpireCheckExecutor", err.Error())
+		}
+	}
+}
+
+// Loop 单次执行
+func (this *SSLCertExpireCheckExecutor) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 查找需要自动更新的证书
+	// 30, 14 ... 是到期的天数
+	for _, days := range []int{30, 14, 7} {
+		certs, err := models.SharedSSLCertDAO.FindAllExpiringCerts(nil, days)
+		if err != nil {
+			return err
+		}
+		for _, cert := range certs {
+			// 发送消息
+			var subject = "SSL证书\"" + cert.Name + "\"在" + strconv.Itoa(days) + "天后将到期，"
+			var msg = "SSL证书\"" + cert.Name + "\"（" + this.summaryDNSNames(cert.DnsNames) + "）在" + strconv.Itoa(days) + "天后将到期，"
+
+			// 是否有自动更新任务
+			if cert.AcmeTaskId > 0 {
+				task, err := acme.SharedACMETaskDAO.FindEnabledACMETask(nil, int64(cert.AcmeTaskId))
+				if err != nil {
+					return err
+				}
+				if task != nil {
+					if task.AutoRenew == 1 {
+						msg += "此证书是免费申请的证书，且已设置了自动续期，将会在到期前三天自动尝试续期。"
+					} else {
+						msg += "此证书是免费申请的证书，没有设置自动续期，请在到期前手动执行续期任务。"
+					}
+				}
+			} else {
+				msg += "请及时更新证书。"
+			}
+
+			err = models.SharedMessageDAO.CreateMessage(nil, int64(cert.AdminId), int64(cert.UserId), models.MessageTypeSSLCertExpiring, models.MessageLevelWarning, subject, msg, maps.Map{
+				"certId":     cert.Id,
+				"acmeTaskId": cert.AcmeTaskId,
+			}.AsJSON())
+			if err != nil {
+				return err
+			}
+
+			// 设置最后通知时间
+			err = models.SharedSSLCertDAO.UpdateCertNotifiedAt(nil, int64(cert.Id))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// 自动续期
+	for _, days := range []int{3, 2, 1} {
+		certs, err := models.SharedSSLCertDAO.FindAllExpiringCerts(nil, days)
+		if err != nil {
+			return err
+		}
+		for _, cert := range certs {
+			// 发送消息
+			var subject = "SSL证书\"" + cert.Name + "\"在" + strconv.Itoa(days) + "天后将到期，"
+			var msg = "SSL证书\"" + cert.Name + "\"（" + this.summaryDNSNames(cert.DnsNames) + "）在" + strconv.Itoa(days) + "天后将到期，"
+
+			// 是否有自动更新任务
+			if cert.AcmeTaskId > 0 {
+				task, err := acme.SharedACMETaskDAO.FindEnabledACMETask(nil, int64(cert.AcmeTaskId))
+				if err != nil {
+					return err
+				}
+				if task != nil {
+					if task.AutoRenew == 1 {
+						isOk, errMsg, _ := acme.SharedACMETaskDAO.RunTask(nil, int64(cert.AcmeTaskId))
+						if isOk {
+							// 发送成功通知
+							subject = "系统已成功为你自动更新了证书\"" + cert.Name + "\""
+							msg = "系统已成功为你自动更新了证书\"" + cert.Name + "\"（" + this.summaryDNSNames(cert.DnsNames) + "）。"
+							err = models.SharedMessageDAO.CreateMessage(nil, int64(cert.AdminId), int64(cert.UserId), models.MessageTypeSSLCertACMETaskSuccess, models.MessageLevelSuccess, subject, msg, maps.Map{
+								"certId":     cert.Id,
+								"acmeTaskId": cert.AcmeTaskId,
+							}.AsJSON())
+							if err != nil {
+								return err
+							}
+
+							// 更新通知时间
+							err = models.SharedSSLCertDAO.UpdateCertNotifiedAt(nil, int64(cert.Id))
+							if err != nil {
+								return err
+							}
+						} else {
+							// 发送失败通知
+							subject = "系统在尝试自动更新证书\"" + cert.Name + "\"时发生错误"
+							msg = "系统在尝试自动更新证书\"" + cert.Name + "\"（" + this.summaryDNSNames(cert.DnsNames) + "）时发生错误：" + errMsg + "。请检查系统设置并修复错误。"
+							err = models.SharedMessageDAO.CreateMessage(nil, int64(cert.AdminId), int64(cert.UserId), models.MessageTypeSSLCertACMETaskFailed, models.MessageLevelError, subject, msg, maps.Map{
+								"certId":     cert.Id,
+								"acmeTaskId": cert.AcmeTaskId,
+							}.AsJSON())
+							if err != nil {
+								return err
+							}
+
+							// 更新通知时间
+							err = models.SharedSSLCertDAO.UpdateCertNotifiedAt(nil, int64(cert.Id))
+							if err != nil {
+								return err
+							}
+						}
+
+						// 中止不发送消息
+						continue
+
+					} else {
+						msg += "此证书是免费申请的证书，没有设置自动续期，请在到期前手动执行续期任务。"
+					}
+				}
+			} else {
+				msg += "请及时更新证书。"
+			}
+
+			err = models.SharedMessageDAO.CreateMessage(nil, int64(cert.AdminId), int64(cert.UserId), models.MessageTypeSSLCertExpiring, models.MessageLevelWarning, subject, msg, maps.Map{
+				"certId":     cert.Id,
+				"acmeTaskId": cert.AcmeTaskId,
+			}.AsJSON())
+			if err != nil {
+				return err
+			}
+
+			// 设置最后通知时间
+			err = models.SharedSSLCertDAO.UpdateCertNotifiedAt(nil, int64(cert.Id))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// 当天过期
+	for _, days := range []int{0} {
+		certs, err := models.SharedSSLCertDAO.FindAllExpiringCerts(nil, days)
+		if err != nil {
+			return err
+		}
+		for _, cert := range certs {
+			// 发送消息
+			var today = timeutil.Format("Y-m-d")
+			var subject = "SSL证书\"" + cert.Name + "\"在今天（" + today + "）过期"
+			var msg = "SSL证书\"" + cert.Name + "\"（" + this.summaryDNSNames(cert.DnsNames) + "）在今天（" + today + "）过期，请及时更新证书，之后将不再重复提醒。"
+			err = models.SharedMessageDAO.CreateMessage(nil, int64(cert.AdminId), int64(cert.UserId), models.MessageTypeSSLCertExpiring, models.MessageLevelWarning, subject, msg, maps.Map{
+				"certId":     cert.Id,
+				"acmeTaskId": cert.AcmeTaskId,
+			}.AsJSON())
+			if err != nil {
+				return err
+			}
+
+			// 设置最后通知时间
+			err = models.SharedSSLCertDAO.UpdateCertNotifiedAt(nil, int64(cert.Id))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// 对证书中DNS域名的描述
+func (this *SSLCertExpireCheckExecutor) summaryDNSNames(dnsNamesJSON []byte) string {
+	if len(dnsNamesJSON) == 0 {
+		return ""
+	}
+	var dnsNames = []string{}
+	err := json.Unmarshal(dnsNamesJSON, &dnsNames)
+	if err != nil {
+		// ignore error
+		return ""
+	}
+
+	var count = len(dnsNames)
+	if count == 0 {
+		return ""
+	}
+	if count == 1 {
+		return "包含" + dnsNames[0] + "域名"
+	}
+	if count <= 10 {
+		return "包含" + strings.Join(dnsNames, "、") + "等域名"
+	}
+
+	return "包含" + strings.Join(dnsNames[:10], "、") + "等" + types.String(count) + "个域名"
+}

EdgeAPI/internal/tasks/ssl_cert_expire_check_executor_test.go

@@ -0,0 +1,26 @@
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"testing"
+	"time"
+)
+
+func TestSSLCertExpireCheckExecutor_loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	t.Log("30 days later: ", timeutil.FormatTime("Y-m-d", time.Now().Unix()+30*86400), time.Now().Unix()+30*86400)
+	t.Log("14 days later: ", timeutil.FormatTime("Y-m-d", time.Now().Unix()+14*86400), time.Now().Unix()+14*86400)
+	t.Log("7 days later: ", timeutil.FormatTime("Y-m-d", time.Now().Unix()+7*86400), time.Now().Unix()+7*86400)
+	t.Log("3 days later: ", timeutil.FormatTime("Y-m-d", time.Now().Unix()+3*86400), time.Now().Unix()+3*86400)
+	t.Log("today: ", timeutil.FormatTime("Y-m-d", time.Now().Unix()), time.Now().Unix())
+
+	var task = tasks.NewSSLCertExpireCheckExecutor(1 * time.Hour)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

EdgeAPI/internal/tasks/ssl_cert_update_ocsp_task.go

@@ -0,0 +1,197 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/iwind/TeaGo/dbs"
+	"golang.org/x/crypto/ocsp"
+	"io"
+	"net/http"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewSSLCertUpdateOCSPTask(1 * time.Minute).Start()
+		})
+	})
+}
+
+type SSLCertUpdateOCSPTask struct {
+	BaseTask
+
+	ticker     *time.Ticker
+	httpClient *http.Client
+}
+
+func NewSSLCertUpdateOCSPTask(duration time.Duration) *SSLCertUpdateOCSPTask {
+	return &SSLCertUpdateOCSPTask{
+		ticker:     time.NewTicker(duration),
+		httpClient: utils.SharedHttpClient(5 * time.Second),
+	}
+}
+
+func (this *SSLCertUpdateOCSPTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			this.logErr("SSLCertUpdateOCSPTask", err.Error())
+		}
+	}
+}
+
+func (this *SSLCertUpdateOCSPTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var tx *dbs.Tx
+	// TODO 将来可以设置单次任务条数
+	var size int64 = 60
+	var maxTries = 5
+	certs, err := models.SharedSSLCertDAO.ListCertsToUpdateOCSP(tx, maxTries, size)
+	if err != nil {
+		return fmt.Errorf("list certs failed: %w", err)
+	}
+
+	if len(certs) == 0 {
+		return nil
+	}
+
+	// 锁定
+	for _, cert := range certs {
+		err := models.SharedSSLCertDAO.PrepareCertOCSPUpdating(tx, int64(cert.Id))
+		if err != nil {
+			return fmt.Errorf("prepare cert ocsp updating failed: %w", err)
+		}
+	}
+
+	for _, cert := range certs {
+		ocspData, expiresAt, err := this.UpdateCertOCSP(cert)
+		var errString = ""
+		var hasErr = false
+		if err != nil {
+			errString = err.Error()
+			hasErr = true
+
+			remotelogs.Warn("SSLCertUpdateOCSPTask", "update ocsp failed: "+errString)
+		}
+		err = models.SharedSSLCertDAO.UpdateCertOCSP(tx, int64(cert.Id), ocspData, expiresAt, hasErr, errString)
+		if err != nil {
+			return fmt.Errorf("update ocsp failed: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// UpdateCertOCSP 更新单个证书OCSP
+func (this *SSLCertUpdateOCSPTask) UpdateCertOCSP(certOne *models.SSLCert) (ocspData []byte, expiresAt int64, err error) {
+	if certOne.IsCA || len(certOne.CertData) == 0 || len(certOne.KeyData) == 0 {
+		return
+	}
+
+	keyPair, err := tls.X509KeyPair(certOne.CertData, certOne.KeyData)
+	if err != nil {
+		return nil, 0, errors.New("parse certificate failed: " + err.Error())
+	}
+	if len(keyPair.Certificate) == 0 {
+		return nil, 0, nil
+	}
+
+	var certData = keyPair.Certificate[0]
+	cert, err := x509.ParseCertificate(certData)
+	if err != nil {
+		return nil, 0, errors.New("parse certificate block failed: " + err.Error())
+	}
+
+	// 是否已过期
+	var now = time.Now()
+	if cert.NotBefore.After(now) || cert.NotAfter.Before(now) {
+		return nil, 0, nil
+	}
+
+	if len(cert.IssuingCertificateURL) == 0 || len(cert.OCSPServer) == 0 {
+		return nil, 0, nil
+	}
+
+	if len(cert.DNSNames) == 0 {
+		return nil, 0, nil
+	}
+
+	var issuerURL = cert.IssuingCertificateURL[0]
+	var ocspServerURL = cert.OCSPServer[0]
+
+	issuerReq, err := http.NewRequest(http.MethodGet, issuerURL, nil)
+	if err != nil {
+		return nil, 0, errors.New("request issuer certificate failed: " + err.Error())
+	}
+	issuerReq.Header.Set("User-Agent", teaconst.ProductName+"/"+teaconst.Version)
+	issuerResp, err := this.httpClient.Do(issuerReq)
+	if err != nil {
+		return nil, 0, errors.New("request issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+	defer func() {
+		_ = issuerResp.Body.Close()
+	}()
+
+	issuerData, err := io.ReadAll(issuerResp.Body)
+	if err != nil {
+		return nil, 0, errors.New("read issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+	issuerCert, err := x509.ParseCertificate(issuerData)
+	if err != nil {
+		return nil, 0, errors.New("parse issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+
+	buf, err := ocsp.CreateRequest(cert, issuerCert, &ocsp.RequestOptions{
+		Hash: crypto.SHA1,
+	})
+	if err != nil {
+		return nil, 0, errors.New("create ocsp request failed: " + err.Error())
+	}
+	ocspReq, err := http.NewRequest(http.MethodPost, ocspServerURL, bytes.NewBuffer(buf))
+	if err != nil {
+		return nil, 0, errors.New("request ocsp failed: " + err.Error())
+	}
+	ocspReq.Header.Set("Content-Type", "application/ocsp-request")
+	ocspReq.Header.Set("Accept", "application/ocsp-response")
+
+	ocspResp, err := this.httpClient.Do(ocspReq)
+	if err != nil {
+		return nil, 0, errors.New("request ocsp failed: '" + ocspServerURL + "': " + err.Error())
+	}
+
+	defer func() {
+		_ = ocspResp.Body.Close()
+	}()
+
+	respData, err := io.ReadAll(ocspResp.Body)
+	if err != nil {
+		return nil, 0, errors.New("read ocsp failed: '" + ocspServerURL + "': " + err.Error())
+	}
+
+	ocspResult, err := ocsp.ParseResponse(respData, issuerCert)
+	if err != nil {
+		return nil, 0, errors.New("decode ocsp failed: " + err.Error())
+	}
+
+	// 只返回Good的ocsp
+	if ocspResult.Status == ocsp.Good {
+		return respData, ocspResult.NextUpdate.Unix(), nil
+	}
+	return nil, 0, nil
+}

EdgeAPI/internal/tasks/ssl_cert_update_ocsp_task_test.go

@@ -0,0 +1,20 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestSSLCertUpdateOCSPTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewSSLCertUpdateOCSPTask(1 * time.Minute)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/task_base.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+)
+
+type BaseTask struct {
+}
+
+func (this *BaseTask) logErr(taskType string, errString string) {
+	remotelogs.Error("TASK", "run '"+taskType+"' failed: "+errString)
+}
+
+func (this *BaseTask) IsPrimaryNode() bool {
+	return models.SharedAPINodeDAO.CheckAPINodeIsPrimaryWithoutErr()
+}

EdgeAPI/internal/tasks/task_interface.go

@@ -0,0 +1,9 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package tasks
+
+type TaskInterface interface {
+	Start() error
+	Loop() error
+	Stop() error
+}

EdgeAPI/internal/tasks/task_send_messages_plus.go

@@ -0,0 +1,220 @@
+//go:build plus
+
+package tasks
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/senders/mediasenders"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/monitorconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewSendMessagesTask(30 * time.Second).Start()
+		})
+	})
+}
+
+type MessageSendingStat struct {
+	Timestamp int64
+	Count     int32
+}
+
+// SendMessagesTask 发送消息任务
+type SendMessagesTask struct {
+	BaseTask
+
+	statMap    map[int64]*MessageSendingStat // instanceId => *Stat
+	statLocker sync.Mutex
+
+	ticker *time.Ticker
+}
+
+// NewSendMessagesTask 获取一个实例
+func NewSendMessagesTask(duration time.Duration) *SendMessagesTask {
+	return &SendMessagesTask{
+		statMap: map[int64]*MessageSendingStat{},
+		ticker:  time.NewTicker(duration),
+	}
+}
+
+// Start 启动任务
+func (this *SendMessagesTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("SEND_MESSAGE_TASK", err.Error())
+		}
+	}
+}
+
+// Loop 单次循环
+func (this *SendMessagesTask) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var cacheMap = utils.NewCacheMap()
+	var tx *dbs.Tx
+	messageTasks, err := models.SharedMessageTaskDAO.FindSendingMessageTasks(tx, 1000)
+	if err != nil {
+		return err
+	}
+
+	// 产品名称
+	productName, err := models.SharedSysSettingDAO.ReadProductName(tx)
+	if err != nil {
+		return err
+	}
+	if len(productName) == 0 {
+		productName = teaconst.GlobalProductName
+	}
+
+	for _, task := range messageTasks {
+		resp, shouldSkip, sendErr := this.send(tx, task, productName, cacheMap)
+		if shouldSkip {
+			continue
+		}
+		var isOk = sendErr == nil
+		var resultMap = maps.Map{
+			"isOk":     isOk,
+			"response": resp,
+			"error":    "",
+		}
+		if sendErr != nil {
+			resultMap["error"] = sendErr.Error()
+		}
+		var newStatus = models.MessageTaskStatusSuccess
+		if !isOk {
+			newStatus = models.MessageTaskStatusFailed
+		}
+
+		sendErr = models.SharedMessageTaskDAO.UpdateMessageTaskStatus(tx, int64(task.Id), newStatus, resultMap.AsJSON())
+		if sendErr != nil {
+			remotelogs.Error("SEND_MESSAGE_TASK", sendErr.Error())
+		}
+
+		// 创建发送记录
+		if newStatus == models.MessageTaskStatusSuccess || newStatus == models.MessageTaskStatusFailed {
+			createLogErr := models.SharedMessageTaskLogDAO.CreateLog(tx, int64(task.Id), isOk, resultMap.GetString("error"), resp)
+			if createLogErr != nil {
+				remotelogs.Error("SEND_MESSAGE_TASK", createLogErr.Error())
+			}
+		}
+	}
+
+	return nil
+}
+
+// CheckRate 检查发送频率
+func (this *SendMessagesTask) CheckRate(instanceId int64, minutes int32, count int32) bool {
+	if minutes <= 0 || count <= 0 {
+		return true
+	}
+
+	this.statLocker.Lock()
+	defer this.statLocker.Unlock()
+
+	var nowTime = time.Now().Unix()
+
+	stat, ok := this.statMap[instanceId]
+	if ok {
+		if stat.Timestamp < nowTime-int64(minutes*60) { // 时间不够
+			stat.Count = 1
+			stat.Timestamp = nowTime
+			return true
+		} else if stat.Count < count { // 次数不足
+			stat.Count++
+			return true
+		} else {
+			return false
+		}
+	} else {
+		this.statMap[instanceId] = &MessageSendingStat{
+			Timestamp: nowTime,
+			Count:     1,
+		}
+		return true
+	}
+}
+
+func (this *SendMessagesTask) StatMap() map[int64]*MessageSendingStat {
+	return this.statMap
+}
+
+// 发送
+func (this *SendMessagesTask) send(tx *dbs.Tx, task *models.MessageTask, productName string, cacheMap *utils.CacheMap) (response string, shouldSkip bool, sendErr error) {
+	// recipient
+	recipient, err := models.SharedMessageRecipientDAO.FindEnabledMessageRecipient(tx, int64(task.RecipientId), cacheMap)
+	if err != nil {
+		return "", false, err
+	}
+	if recipient == nil {
+		return "", false, errors.New("could not find recipient with id '" + types.String(task.RecipientId) + "'")
+	}
+
+	// instance
+	var instanceId = int64(task.InstanceId)
+	if instanceId <= 0 {
+		instanceId = int64(recipient.InstanceId)
+	}
+	if instanceId <= 0 {
+		return "", false, errors.New("could not find instance with id '" + types.String(instanceId) + "'")
+	}
+
+	var instance *models.MessageMediaInstance
+	instance, err = models.SharedMessageMediaInstanceDAO.FindEnabledMessageMediaInstance(tx, instanceId, cacheMap)
+	if err != nil {
+		return "", false, err
+	}
+	if instance == nil {
+		return "", false, errors.New("could not find instance with id '" + types.String(instanceId) + "'")
+	}
+
+	var rateConfig = &monitorconfigs.RateConfig{}
+	if len(instance.Rate) > 0 {
+		err = json.Unmarshal(instance.Rate, rateConfig)
+		if err != nil {
+			return "", false, fmt.Errorf("decode instance rate json failed: %w", err)
+		} else if rateConfig.Count > 0 && rateConfig.Minutes > 0 && !this.CheckRate(instanceId, rateConfig.Minutes, rateConfig.Count) {
+			return "", true, nil
+		}
+	}
+
+	// user
+	var toUser = task.User
+	if len(toUser) == 0 && len(recipient.User) > 0 {
+		toUser = recipient.User
+	}
+
+	// TODO 考虑重用？
+	media, err := mediasenders.NewMediaInstance(instance.MediaType, instance.Params)
+	if err != nil {
+		return "", false, err
+	}
+
+	if media == nil {
+		return "", false, errors.New("could not create media instance for '" + instance.MediaType + "'")
+	}
+
+	respBytes, err := media.Send(toUser, task.Subject, task.Body, productName, timeutil.FormatTime("Y-m-d H:i:s", int64(task.CreatedAt)))
+	if err != nil {
+		return "", false, err
+	}
+
+	return string(respBytes), false, nil
+}

EdgeAPI/internal/tasks/task_send_messages_plus_test.go

@@ -0,0 +1,37 @@
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
+	"testing"
+	"time"
+)
+
+func TestSendMessagesTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewSendMessagesTask(5 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestSendMessagesTask_checkRate(t *testing.T) {
+	task := tasks.NewSendMessagesTask(5 * time.Second)
+	t.Log(task.CheckRate(1, 0, 0))
+	t.Log(task.CheckRate(1, 1, 2))
+	t.Log(task.CheckRate(1, 1, 2))
+	t.Log(task.CheckRate(1, 1, 2))
+
+	var statMap = task.StatMap()
+	logs.PrintAsJSON(statMap, t)
+	statMap[1].Timestamp = time.Now().Unix() - 61
+	t.Log(task.CheckRate(1, 1, 2))
+	logs.PrintAsJSON(statMap, t)
+}

EdgeAPI/internal/tasks/user_email_notification_task_plus.go

@@ -0,0 +1,159 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"fmt"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/senders/mediasenders"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
+	"github.com/iwind/TeaGo/types"
+	"net/mail"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewUserEmailNotificationTask(30 * time.Second).Start()
+		})
+	})
+}
+
+type UserEmailNotificationTask struct {
+	BaseTask
+
+	duration time.Duration
+}
+
+func NewUserEmailNotificationTask(duration time.Duration) *UserEmailNotificationTask {
+	return &UserEmailNotificationTask{
+		duration: duration,
+	}
+}
+
+func (this *UserEmailNotificationTask) Start() {
+	var ticker = time.NewTicker(this.duration)
+	for range ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("UserEmailNotificationTask", err.Error())
+		}
+	}
+}
+
+func (this *UserEmailNotificationTask) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var tx *dbs.Tx
+
+	// 删除过期的通知
+	err := models.SharedUserEmailNotificationDAO.DeleteExpiredNotifications(tx, 2)
+	if err != nil {
+		return err
+	}
+
+	// 通知邮件设置
+	senderConfig, err := models.SharedSysSettingDAO.ReadUserSenderConfig(tx)
+	if err != nil {
+		return err
+	}
+	var notifyEmail = senderConfig.NotifyEmail
+	if notifyEmail == nil {
+		return nil
+	}
+	if !notifyEmail.IsOn {
+		return nil
+	}
+
+	// 准备邮件中的参数
+	productName, err := models.SharedSysSettingDAO.ReadProductName(tx)
+	if err != nil {
+		return err
+	}
+	if len(productName) == 0 {
+		productName = teaconst.ProductNameZH
+	}
+
+	userNodeAddr, err := models.SharedUserNodeDAO.FindUserNodeAccessAddr(tx)
+	if err != nil {
+		return err
+	}
+
+	// 查询待发送
+	notifications, err := models.SharedUserEmailNotificationDAO.ListSendingNotifications(tx, 100)
+	if err != nil {
+		return err
+	}
+	if len(notifications) == 0 {
+		return nil
+	}
+
+	var mailInfos = []*mediasenders.MailInfo{}
+	var notificationIds = []int64{}
+	for _, notification := range notifications {
+		_, err = mail.ParseAddress(notification.Email)
+		if err != nil {
+			// 直接删除
+			err = models.SharedUserEmailNotificationDAO.DeleteNotification(tx, int64(notification.Id))
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// 标题和内容
+		var subject = strings.ReplaceAll(notification.Subject, "${product.name}", productName)
+
+		var body = notification.Body
+		body = strings.ReplaceAll(body, "${product.name}", productName)
+		body = strings.ReplaceAll(body, "${url.home}", userNodeAddr)
+
+		mailInfos = append(mailInfos, &mediasenders.MailInfo{
+			To:      notification.Email,
+			Subject: subject,
+			Body:    body,
+		})
+
+		if Tea.IsTesting() {
+			logs.Println("sending notification mail to :", notification.Email)
+		}
+
+		notificationIds = append(notificationIds, int64(notification.Id))
+	}
+	if len(mailInfos) == 0 {
+		return nil
+	}
+
+	// 发送
+	var mailSender = mediasenders.NewEmailMedia()
+	mailSender.Protocol = notifyEmail.Protocol
+	mailSender.SMTP = notifyEmail.SMTPHost + ":" + types.String(notifyEmail.SMTPPort)
+	mailSender.Username = notifyEmail.Username
+	mailSender.Password = notifyEmail.Password
+	mailSender.From = notifyEmail.FromEmail
+	mailSender.FromName = notifyEmail.FromName
+	err = mailSender.SendMails(mailInfos, productName)
+	if err != nil {
+		return fmt.Errorf("send notification mail failed: %w", err)
+	}
+
+	// 设置为已发送
+	for _, notificationId := range notificationIds {
+		err = models.SharedUserEmailNotificationDAO.DeleteNotification(tx, notificationId)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/user_email_notification_task_plus_test.go

@@ -0,0 +1,21 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestUserEmailNotificationTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewUserEmailNotificationTask(30 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/user_email_verification_task_plus.go

@@ -0,0 +1,192 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"fmt"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/senders/mediasenders"
+	"github.com/TeaOSLab/EdgeCommon/pkg/userconfigs"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
+	"github.com/iwind/TeaGo/types"
+	"net/mail"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewUserEmailVerificationTask(30 * time.Second).Start()
+		})
+	})
+}
+
+// UserEmailVerificationTask 用户邮件验证任务
+type UserEmailVerificationTask struct {
+	BaseTask
+
+	duration time.Duration
+}
+
+func NewUserEmailVerificationTask(duration time.Duration) *UserEmailVerificationTask {
+	return &UserEmailVerificationTask{
+		duration: duration,
+	}
+}
+
+func (this *UserEmailVerificationTask) Start() {
+	var ticker = time.NewTicker(this.duration)
+	for range ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("UserEmailVerificationTask", err.Error())
+		}
+	}
+}
+
+func (this *UserEmailVerificationTask) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	var tx *dbs.Tx
+
+	// 注册设置
+	registerConfig, err := models.SharedSysSettingDAO.ReadUserRegisterConfig(tx)
+	if err != nil {
+		return err
+	}
+	if !registerConfig.EmailVerification.IsOn {
+		return nil
+	}
+
+	// 删除过期的认证
+	var life = registerConfig.EmailVerification.Life
+	if life <= 0 {
+		life = userconfigs.EmailVerificationDefaultLife
+	}
+	var lifeDays = life / 86400
+	if lifeDays <= 0 {
+		lifeDays = 1
+	}
+	err = models.SharedUserEmailVerificationDAO.DeleteExpiredVerifications(tx, lifeDays)
+	if err != nil {
+		return err
+	}
+
+	// 检查邮件设置
+	senderConfig, err := models.SharedSysSettingDAO.ReadUserSenderConfig(tx)
+	if err != nil {
+		return err
+	}
+	var verifyEmail = senderConfig.VerifyEmail
+	if verifyEmail == nil {
+		// TODO 思考是否需要用户没有设置
+		return nil
+	}
+	if !verifyEmail.IsOn {
+		return nil
+	}
+
+	// 查找等待发送的认证
+	verifications, err := models.SharedUserEmailVerificationDAO.ListSendingVerifications(tx, 10 /** 单次10封 **/)
+	if err != nil {
+		return err
+	}
+	if len(verifications) == 0 {
+		return nil
+	}
+
+	// 准备邮件中的参数
+	productName, err := models.SharedSysSettingDAO.ReadProductName(tx)
+	if err != nil {
+		return err
+	}
+	if len(productName) == 0 {
+		productName = teaconst.ProductNameZH
+	}
+
+	userNodeAddr, err := models.SharedUserNodeDAO.FindUserNodeAccessAddr(tx)
+	if err != nil {
+		return err
+	}
+
+	var mailInfos = []*mediasenders.MailInfo{}
+	var verificationIds = []int64{}
+	for _, verification := range verifications {
+		// 检查时间
+		if int64(verification.CreatedAt) < time.Now().Unix()-int64(life) {
+			// 已过期，设置为已发送
+			err = models.SharedUserEmailVerificationDAO.DeleteVerification(tx, int64(verification.Id))
+			if err != nil {
+				return err
+			}
+
+			continue
+		}
+
+		_, err = mail.ParseAddress(verification.Email)
+		if err != nil {
+			// 邮件已发送，设置为已发送
+			err = models.SharedUserEmailVerificationDAO.DeleteVerification(tx, int64(verification.Id))
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// 标题和内容
+		var subject = registerConfig.EmailVerification.Subject
+		subject = strings.ReplaceAll(subject, "${product.name}", productName)
+
+		var body = registerConfig.EmailVerification.Body
+		body = strings.ReplaceAll(body, "${product.name}", productName)
+		body = strings.ReplaceAll(body, "${url.home}", userNodeAddr)
+		body = strings.ReplaceAll(body, "${url.verify}", userNodeAddr+"/email/verify/"+verification.Code)
+
+		mailInfos = append(mailInfos, &mediasenders.MailInfo{
+			To:      verification.Email,
+			Subject: subject,
+			Body:    body,
+		})
+
+		if Tea.IsTesting() {
+			logs.Println("sending verification mail to :", verification.Email)
+		}
+
+		verificationIds = append(verificationIds, int64(verification.Id))
+	}
+	if len(mailInfos) == 0 {
+		return nil
+	}
+
+	// 发送
+	var mailSender = mediasenders.NewEmailMedia()
+	mailSender.Protocol = verifyEmail.Protocol
+	mailSender.SMTP = verifyEmail.SMTPHost + ":" + types.String(verifyEmail.SMTPPort)
+	mailSender.Username = verifyEmail.Username
+	mailSender.Password = verifyEmail.Password
+	mailSender.From = verifyEmail.FromEmail
+	mailSender.FromName = verifyEmail.FromName
+	err = mailSender.SendMails(mailInfos, productName)
+	if err != nil {
+		return fmt.Errorf("send mail failed: %w", err)
+	}
+
+	// 设置为已发送
+	for _, verificationId := range verificationIds {
+		err = models.SharedUserEmailVerificationDAO.UpdateVerificationIsSent(tx, verificationId)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/user_email_verification_task_plus_test.go

@@ -0,0 +1,21 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNewUserEmailVerificationTask(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewUserEmailVerificationTask(30 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

EdgeAPI/internal/tasks/user_verify_code_task_plus.go

@@ -0,0 +1,160 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks
+
+import (
+	"fmt"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/senders/mediasenders"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewUserVerifyCodeTask(30 * time.Second).Start()
+		})
+	})
+}
+
+// UserVerifyCodeTask 验证码相关任务
+type UserVerifyCodeTask struct {
+	BaseTask
+
+	duration time.Duration
+}
+
+func NewUserVerifyCodeTask(duration time.Duration) *UserVerifyCodeTask {
+	return &UserVerifyCodeTask{
+		duration: duration,
+	}
+}
+
+func (this *UserVerifyCodeTask) Start() {
+	var ticker = time.NewTicker(this.duration)
+	for range ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("UserVerifyCodeTask", err.Error())
+		}
+	}
+}
+
+func (this *UserVerifyCodeTask) Loop() error {
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 删除过期
+	var tx *dbs.Tx
+	err := models.SharedUserVerifyCodeDAO.DeleteExpiredCodes(tx)
+	if err != nil {
+		return err
+	}
+
+	// 注册信息
+	registerConfig, _ := models.SharedSysSettingDAO.ReadUserRegisterConfig(tx)
+	if registerConfig == nil {
+		return nil
+	}
+
+	// 邮件设置
+	senderConfig, err := models.SharedSysSettingDAO.ReadUserSenderConfig(tx)
+	if err != nil {
+		return err
+	}
+	var verifyEmail = senderConfig.VerifyEmail
+
+	// 准备邮件、短信中的参数
+	productName, err := models.SharedSysSettingDAO.ReadProductName(tx)
+	if err != nil {
+		return err
+	}
+	if len(productName) == 0 {
+		productName = teaconst.ProductNameZH
+	}
+
+	userNodeAddr, err := models.SharedUserNodeDAO.FindUserNodeAccessAddr(tx)
+	if err != nil {
+		return err
+	}
+
+	// 待发送
+	verifyCodes, err := models.SharedUserVerifyCodeDAO.ListSendingCodes(tx, 10)
+	if err != nil {
+		return err
+	}
+
+	var mailInfos = []*mediasenders.MailInfo{}
+	var verifyIds = []int64{}
+
+	for _, verifyCode := range verifyCodes {
+		// 已过期
+		if int64(verifyCode.ExpiresAt) < time.Now().Unix() {
+			continue
+		}
+
+		switch verifyCode.Type {
+		case models.UserVerifyCodeTypeResetPassword:
+			// 通过邮件
+			if len(verifyCode.Email) > 0 && verifyEmail != nil && verifyEmail.IsOn {
+				if registerConfig.EmailResetPassword.IsOn {
+					var subject = registerConfig.EmailResetPassword.Subject
+					subject = strings.ReplaceAll(subject, "${product.name}", productName)
+
+					var body = registerConfig.EmailResetPassword.Body
+					body = strings.ReplaceAll(body, "${product.name}", productName)
+					body = strings.ReplaceAll(body, "${url.home}", userNodeAddr)
+					body = strings.ReplaceAll(body, "${code}", verifyCode.Code)
+
+					mailInfos = append(mailInfos, &mediasenders.MailInfo{
+						To:      verifyCode.Email,
+						Subject: subject,
+						Body:    body,
+					})
+
+					verifyIds = append(verifyIds, int64(verifyCode.Id))
+				}
+			}
+
+			// 通过手机号
+			if len(verifyCode.Mobile) > 0 {
+				// TODO 需要实现
+
+				verifyIds = append(verifyIds, int64(verifyCode.Id))
+			}
+		}
+	}
+
+	// 发送邮件
+	if len(mailInfos) > 0 && verifyEmail != nil {
+		var mailSender = mediasenders.NewEmailMedia()
+		mailSender.Protocol = verifyEmail.Protocol
+		mailSender.SMTP = verifyEmail.SMTPHost + ":" + types.String(verifyEmail.SMTPPort)
+		mailSender.Username = verifyEmail.Username
+		mailSender.Password = verifyEmail.Password
+		mailSender.From = verifyEmail.FromEmail
+		mailSender.FromName = verifyEmail.FromName
+		err = mailSender.SendMails(mailInfos, productName)
+		if err != nil {
+			return fmt.Errorf("send mail failed: %w", err)
+		}
+	}
+
+	// 设置为已发送
+	for _, verifyId := range verifyIds {
+		err = models.SharedUserVerifyCodeDAO.UpdateCodeIsSent(tx, verifyId)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

EdgeAPI/internal/tasks/user_verify_code_task_plus_test.go

@@ -0,0 +1,21 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build plus
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNewUserVerifyCodeTask(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewUserVerifyCodeTask(30 * time.Second)
+	err := task.Loop()
+	if err != nil {
+		t.Fatal(err)
+	}
+}