108 lines
2.6 KiB
Go
108 lines
2.6 KiB
Go
package tasks
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
|
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
|
"github.com/TeaOSLab/EdgeAPI/internal/installers"
|
|
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
|
"github.com/iwind/TeaGo/dbs"
|
|
)
|
|
|
|
func init() {
|
|
dbs.OnReadyDone(func() {
|
|
goman.New(func() {
|
|
NewHTTPDNSNodeMonitorTask(1 * time.Minute).Start()
|
|
})
|
|
})
|
|
}
|
|
|
|
type httpdnsNodeStartingTry struct {
|
|
count int
|
|
timestamp int64
|
|
}
|
|
|
|
// HTTPDNSNodeMonitorTask monitors HTTPDNS node activity and optionally tries to start offline nodes.
|
|
type HTTPDNSNodeMonitorTask struct {
|
|
BaseTask
|
|
|
|
ticker *time.Ticker
|
|
|
|
recoverMap map[int64]*httpdnsNodeStartingTry // nodeId => retry info
|
|
}
|
|
|
|
func NewHTTPDNSNodeMonitorTask(duration time.Duration) *HTTPDNSNodeMonitorTask {
|
|
return &HTTPDNSNodeMonitorTask{
|
|
ticker: time.NewTicker(duration),
|
|
recoverMap: map[int64]*httpdnsNodeStartingTry{},
|
|
}
|
|
}
|
|
|
|
func (t *HTTPDNSNodeMonitorTask) Start() {
|
|
for range t.ticker.C {
|
|
if err := t.Loop(); err != nil {
|
|
t.logErr("HTTPDNS_NODE_MONITOR", err.Error())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (t *HTTPDNSNodeMonitorTask) Loop() error {
|
|
// only run on primary api node
|
|
if !t.IsPrimaryNode() {
|
|
return nil
|
|
}
|
|
|
|
clusters, err := models.SharedHTTPDNSClusterDAO.FindAllEnabledClusters(nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, cluster := range clusters {
|
|
if cluster == nil || cluster.IsOn == 0 || cluster.AutoRemoteStart == 0 {
|
|
continue
|
|
}
|
|
clusterID := int64(cluster.Id)
|
|
inactiveNodes, err := models.SharedHTTPDNSNodeDAO.FindAllInactiveNodesWithClusterId(nil, clusterID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(inactiveNodes) == 0 {
|
|
continue
|
|
}
|
|
|
|
nodeQueue := installers.NewHTTPDNSNodeQueue()
|
|
for _, node := range inactiveNodes {
|
|
nodeID := int64(node.Id)
|
|
tryInfo, ok := t.recoverMap[nodeID]
|
|
if !ok {
|
|
tryInfo = &httpdnsNodeStartingTry{
|
|
count: 1,
|
|
timestamp: time.Now().Unix(),
|
|
}
|
|
t.recoverMap[nodeID] = tryInfo
|
|
} else {
|
|
if tryInfo.count >= 3 {
|
|
if tryInfo.timestamp+10*60 > time.Now().Unix() {
|
|
continue
|
|
}
|
|
tryInfo.timestamp = time.Now().Unix()
|
|
tryInfo.count = 0
|
|
}
|
|
tryInfo.count++
|
|
}
|
|
|
|
err = nodeQueue.StartNode(nodeID)
|
|
if err != nil {
|
|
if !installers.IsGrantError(err) {
|
|
_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelError, "NODE", "start node from remote API failed: "+err.Error(), time.Now().Unix(), "", nil)
|
|
}
|
|
continue
|
|
}
|
|
_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelSuccess, "NODE", "start node from remote API successfully", time.Now().Unix(), "", nil)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|