package tasks import ( "time" "github.com/TeaOSLab/EdgeAPI/internal/db/models" "github.com/TeaOSLab/EdgeAPI/internal/goman" "github.com/TeaOSLab/EdgeAPI/internal/installers" "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs" "github.com/iwind/TeaGo/dbs" ) func init() { dbs.OnReadyDone(func() { goman.New(func() { NewHTTPDNSNodeMonitorTask(1 * time.Minute).Start() }) }) } type httpdnsNodeStartingTry struct { count int timestamp int64 } // HTTPDNSNodeMonitorTask monitors HTTPDNS node activity and optionally tries to start offline nodes. type HTTPDNSNodeMonitorTask struct { BaseTask ticker *time.Ticker recoverMap map[int64]*httpdnsNodeStartingTry // nodeId => retry info } func NewHTTPDNSNodeMonitorTask(duration time.Duration) *HTTPDNSNodeMonitorTask { return &HTTPDNSNodeMonitorTask{ ticker: time.NewTicker(duration), recoverMap: map[int64]*httpdnsNodeStartingTry{}, } } func (t *HTTPDNSNodeMonitorTask) Start() { for range t.ticker.C { if err := t.Loop(); err != nil { t.logErr("HTTPDNS_NODE_MONITOR", err.Error()) } } } func (t *HTTPDNSNodeMonitorTask) Loop() error { // only run on primary api node if !t.IsPrimaryNode() { return nil } clusters, err := models.SharedHTTPDNSClusterDAO.FindAllEnabledClusters(nil) if err != nil { return err } for _, cluster := range clusters { if cluster == nil || cluster.IsOn == 0 || cluster.AutoRemoteStart == 0 { continue } clusterID := int64(cluster.Id) inactiveNodes, err := models.SharedHTTPDNSNodeDAO.FindAllInactiveNodesWithClusterId(nil, clusterID) if err != nil { return err } if len(inactiveNodes) == 0 { continue } nodeQueue := installers.NewHTTPDNSNodeQueue() for _, node := range inactiveNodes { nodeID := int64(node.Id) tryInfo, ok := t.recoverMap[nodeID] if !ok { tryInfo = &httpdnsNodeStartingTry{ count: 1, timestamp: time.Now().Unix(), } t.recoverMap[nodeID] = tryInfo } else { if tryInfo.count >= 3 { if tryInfo.timestamp+10*60 > time.Now().Unix() { continue } tryInfo.timestamp = time.Now().Unix() tryInfo.count = 0 } tryInfo.count++ } err = nodeQueue.StartNode(nodeID) if err != nil { if !installers.IsGrantError(err) { _ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelError, "NODE", "start node from remote API failed: "+err.Error(), time.Now().Unix(), "", nil) } continue } _ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelSuccess, "NODE", "start node from remote API successfully", time.Now().Unix(), "", nil) } } return nil }