1.4.5.2

EdgeAPI/internal/tasks/health_check_cluster_task.go

@@ -0,0 +1,170 @@
+package tasks
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"time"
+)
+
+// HealthCheckClusterTask 单个集群的健康检查任务
+type HealthCheckClusterTask struct {
+	BaseTask
+
+	clusterId int64
+	config    *serverconfigs.HealthCheckConfig
+	ticker    *utils.Ticker
+
+	notifiedTime time.Time
+}
+
+// NewHealthCheckClusterTask 创建新任务
+func NewHealthCheckClusterTask(clusterId int64, config *serverconfigs.HealthCheckConfig) *HealthCheckClusterTask {
+	return &HealthCheckClusterTask{
+		clusterId: clusterId,
+		config:    config,
+	}
+}
+
+// Reset 重置配置
+func (this *HealthCheckClusterTask) Reset(config *serverconfigs.HealthCheckConfig) {
+	// 检查是否有变化
+	oldJSON, err := json.Marshal(this.config)
+	if err != nil {
+		this.logErr("HealthCheckClusterTask", err.Error())
+		return
+	}
+	newJSON, err := json.Marshal(config)
+	if err != nil {
+		this.logErr("HealthCheckClusterTask", err.Error())
+		return
+	}
+	if !bytes.Equal(oldJSON, newJSON) {
+		this.config = config
+		this.Run()
+	}
+}
+
+// Run 执行
+func (this *HealthCheckClusterTask) Run() {
+	this.Stop()
+
+	if this.config == nil {
+		return
+	}
+	if !this.config.IsOn {
+		return
+	}
+	if this.config.Interval == nil {
+		return
+	}
+	var duration = this.config.Interval.Duration()
+	if duration <= 0 {
+		return
+	}
+	var ticker = utils.NewTicker(duration)
+	goman.New(func() {
+		for ticker.Wait() {
+			err := this.Loop()
+			if err != nil {
+				this.logErr("HealthCheckClusterTask", err.Error())
+			}
+		}
+	})
+	this.ticker = ticker
+}
+
+// Stop 停止
+func (this *HealthCheckClusterTask) Stop() {
+	if this.ticker == nil {
+		return
+	}
+	this.ticker.Stop()
+	this.ticker = nil
+}
+
+// Loop 单个循环任务
+func (this *HealthCheckClusterTask) Loop() error {
+	// 检查是否为主节点
+	if !this.IsPrimaryNode() {
+		return nil
+	}
+
+	// 开始运行
+	var executor = NewHealthCheckExecutor(this.clusterId)
+	results, err := executor.Run()
+	if err != nil {
+		return err
+	}
+
+	var failedResults = []maps.Map{}
+	for _, result := range results {
+		if !result.IsOk {
+			failedResults = append(failedResults, maps.Map{
+				"node": maps.Map{
+					"id":   result.Node.Id,
+					"name": result.Node.Name,
+				},
+				"isOk":     false,
+				"error":    result.Error,
+				"nodeAddr": result.NodeAddr,
+			})
+		}
+	}
+
+	if len(failedResults) > 0 {
+		// 10分钟内不重复提醒
+		if time.Since(this.notifiedTime) > 10*time.Minute {
+			this.notifiedTime = time.Now()
+
+			failedResultsJSON, err := json.Marshal(failedResults)
+			if err != nil {
+				return err
+			}
+			var subject = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题"
+			var message = "有" + numberutils.FormatInt(len(failedResults)) + "个节点IP在健康检查中出现问题："
+			var failedDescriptions = []string{}
+			var failedIndex int
+			for _, result := range results {
+				if result.IsOk || result.Node == nil {
+					continue
+				}
+				failedIndex++
+				failedDescriptions = append(failedDescriptions, "节点"+types.String(failedIndex)+"："+result.Node.Name+"，IP："+result.NodeAddr)
+			}
+
+			const maxNodeDescriptions = 10
+			var isOverMax = false
+			if len(failedDescriptions) > maxNodeDescriptions {
+				failedDescriptions = failedDescriptions[:maxNodeDescriptions]
+				isOverMax = true
+			}
+			message += strings.Join(failedDescriptions, "；")
+			if isOverMax {
+				message += " ..."
+			} else {
+				message += "。"
+			}
+
+			err = models.NewMessageDAO().CreateClusterMessage(nil, nodeconfigs.NodeRoleNode, this.clusterId, models.MessageTypeHealthCheckFailed, models.MessageLevelError, subject, subject, message, failedResultsJSON)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Config 获取当前配置
+func (this *HealthCheckClusterTask) Config() *serverconfigs.HealthCheckConfig {
+	return this.config
+}