waf-platform/EdgeAPI/internal/rpc/services/httpdns/service_httpdns_board.go

package httpdns

import (
	"context"
	"time"

	"github.com/TeaOSLab/EdgeAPI/internal/clickhouse"
	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
	"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
	"github.com/TeaOSLab/EdgeAPI/internal/utils"
	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
	"github.com/iwind/TeaGo/dbs"
	timeutil "github.com/iwind/TeaGo/utils/time"
)

// HTTPDNSBoardService HTTPDNS 仪表盘服务
type HTTPDNSBoardService struct {
	services.BaseService
}

// ComposeHTTPDNSBoard 组合看板数据
func (this *HTTPDNSBoardService) ComposeHTTPDNSBoard(ctx context.Context, req *pb.ComposeHTTPDNSBoardRequest) (*pb.ComposeHTTPDNSBoardResponse, error) {
	_, err := this.ValidateAdmin(ctx)
	if err != nil {
		return nil, err
	}

	tx := this.NullTx()
	result := &pb.ComposeHTTPDNSBoardResponse{}

	countApps, err := models.SharedHTTPDNSAppDAO.CountEnabledApps(tx, "")
	if err != nil {
		return nil, err
	}
	result.CountApps = countApps

	countDomains, err := models.SharedHTTPDNSDomainDAO.CountEnabledDomains(tx, 0)
	if err != nil {
		return nil, err
	}
	result.CountDomains = countDomains

	countClusters, err := models.SharedHTTPDNSClusterDAO.CountEnabledClusters(tx, "")
	if err != nil {
		return nil, err
	}
	result.CountClusters = countClusters

	allNodes, err := models.SharedHTTPDNSNodeDAO.ListEnabledNodes(tx, 0)
	if err != nil {
		return nil, err
	}
	result.CountNodes = int64(len(allNodes))

	var countOffline int64
	for _, node := range allNodes {
		if !node.IsActive {
			countOffline++
		}
	}
	result.CountOfflineNodes = countOffline

	hourFrom := timeutil.Format("YmdH", time.Now().Add(-23*time.Hour))
	hourTo := timeutil.Format("YmdH")
	dayFrom := timeutil.Format("Ymd", time.Now().AddDate(0, 0, -14))
	dayTo := timeutil.Format("Ymd")
	todayFrom := timeutil.Format("Ymd", time.Now().Add(-24*time.Hour))

	store := clickhouse.NewHTTPDNSAccessLogsStore()
	if store.Client().IsConfigured() {
		err = this.composeTrafficAndRanksFromClickHouse(ctx, tx, store, result, hourFrom, hourTo, dayFrom, dayTo, todayFrom)
	}
	if err != nil || !store.Client().IsConfigured() {
		err = this.composeTrafficAndRanksFromMySQL(tx, result, hourFrom, hourTo, dayFrom, dayTo, todayFrom)
		if err != nil {
			return nil, err
		}
	}

	err = this.fillNodeValues(tx, result)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func (this *HTTPDNSBoardService) composeTrafficAndRanksFromClickHouse(ctx context.Context, tx *dbs.Tx, store *clickhouse.HTTPDNSAccessLogsStore, result *pb.ComposeHTTPDNSBoardResponse, hourFrom string, hourTo string, dayFrom string, dayTo string, todayFrom string) error {
	hourlyStats, err := store.FindHourlyStats(ctx, hourFrom, hourTo)
	if err != nil {
		return err
	}
	hourlyMap := map[string]*clickhouse.HTTPDNSAccessLogHourlyStat{}
	for _, stat := range hourlyStats {
		hourlyMap[stat.Hour] = stat
	}
	hours, err := utils.RangeHours(hourFrom, hourTo)
	if err != nil {
		return err
	}
	for _, hour := range hours {
		stat, ok := hourlyMap[hour]
		if ok {
			result.HourlyTrafficStats = append(result.HourlyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_HourlyTrafficStat{
				Hour:          hour,
				CountRequests: stat.CountRequests,
			})
		} else {
			result.HourlyTrafficStats = append(result.HourlyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_HourlyTrafficStat{Hour: hour})
		}
	}

	dailyStats, err := store.FindDailyStats(ctx, dayFrom, dayTo)
	if err != nil {
		return err
	}
	dailyMap := map[string]*clickhouse.HTTPDNSAccessLogDailyStat{}
	for _, stat := range dailyStats {
		dailyMap[stat.Day] = stat
	}
	days, err := utils.RangeDays(dayFrom, dayTo)
	if err != nil {
		return err
	}
	for _, day := range days {
		stat, ok := dailyMap[day]
		if ok {
			result.DailyTrafficStats = append(result.DailyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_DailyTrafficStat{
				Day:           day,
				CountRequests: stat.CountRequests,
			})
		} else {
			result.DailyTrafficStats = append(result.DailyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_DailyTrafficStat{Day: day})
		}
	}

	topAppStats, err := store.ListTopApps(ctx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topAppStats {
		appName := stat.AppName
		if len(appName) == 0 {
			appName = stat.AppId
		}
		result.TopAppStats = append(result.TopAppStats, &pb.ComposeHTTPDNSBoardResponse_TopAppStat{
			AppId:         0,
			AppName:       appName,
			CountRequests: stat.CountRequests,
		})
	}

	topDomainStats, err := store.ListTopDomains(ctx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topDomainStats {
		result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeHTTPDNSBoardResponse_TopDomainStat{
			DomainName:    stat.Domain,
			CountRequests: stat.CountRequests,
		})
	}

	topNodeStats, err := store.ListTopNodes(ctx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topNodeStats {
		nodeName := ""
		node, nodeErr := models.SharedHTTPDNSNodeDAO.FindEnabledNode(tx, int64(stat.NodeId))
		if nodeErr == nil && node != nil {
			nodeName = node.Name
		}
		if len(nodeName) == 0 {
			continue
		}
		result.TopNodeStats = append(result.TopNodeStats, &pb.ComposeHTTPDNSBoardResponse_TopNodeStat{
			ClusterId:     int64(stat.ClusterId),
			NodeId:        int64(stat.NodeId),
			NodeName:      nodeName,
			CountRequests: stat.CountRequests,
		})
	}

	return nil
}

func (this *HTTPDNSBoardService) composeTrafficAndRanksFromMySQL(tx *dbs.Tx, result *pb.ComposeHTTPDNSBoardResponse, hourFrom string, hourTo string, dayFrom string, dayTo string, todayFrom string) error {
	hourlyStats, err := models.SharedHTTPDNSAccessLogDAO.FindHourlyStats(tx, hourFrom, hourTo)
	if err != nil {
		return err
	}
	hourlyMap := map[string]*models.HTTPDNSAccessLogHourlyStat{}
	for _, stat := range hourlyStats {
		hourlyMap[stat.Hour] = stat
	}
	hours, err := utils.RangeHours(hourFrom, hourTo)
	if err != nil {
		return err
	}
	for _, hour := range hours {
		stat, ok := hourlyMap[hour]
		if ok {
			result.HourlyTrafficStats = append(result.HourlyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_HourlyTrafficStat{
				Hour:          hour,
				CountRequests: stat.CountRequests,
			})
		} else {
			result.HourlyTrafficStats = append(result.HourlyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_HourlyTrafficStat{Hour: hour})
		}
	}

	dailyStats, err := models.SharedHTTPDNSAccessLogDAO.FindDailyStats(tx, dayFrom, dayTo)
	if err != nil {
		return err
	}
	dailyMap := map[string]*models.HTTPDNSAccessLogDailyStat{}
	for _, stat := range dailyStats {
		dailyMap[stat.Day] = stat
	}
	days, err := utils.RangeDays(dayFrom, dayTo)
	if err != nil {
		return err
	}
	for _, day := range days {
		stat, ok := dailyMap[day]
		if ok {
			result.DailyTrafficStats = append(result.DailyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_DailyTrafficStat{
				Day:           day,
				CountRequests: stat.CountRequests,
			})
		} else {
			result.DailyTrafficStats = append(result.DailyTrafficStats, &pb.ComposeHTTPDNSBoardResponse_DailyTrafficStat{Day: day})
		}
	}

	topAppStats, err := models.SharedHTTPDNSAccessLogDAO.ListTopApps(tx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topAppStats {
		appName := stat.AppName
		if len(appName) == 0 {
			appName = stat.AppId
		}
		result.TopAppStats = append(result.TopAppStats, &pb.ComposeHTTPDNSBoardResponse_TopAppStat{
			AppId:         0,
			AppName:       appName,
			CountRequests: stat.CountRequests,
		})
	}

	topDomainStats, err := models.SharedHTTPDNSAccessLogDAO.ListTopDomains(tx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topDomainStats {
		result.TopDomainStats = append(result.TopDomainStats, &pb.ComposeHTTPDNSBoardResponse_TopDomainStat{
			DomainName:    stat.Domain,
			CountRequests: stat.CountRequests,
		})
	}

	topNodeStats, err := models.SharedHTTPDNSAccessLogDAO.ListTopNodes(tx, todayFrom, dayTo, 10)
	if err != nil {
		return err
	}
	for _, stat := range topNodeStats {
		nodeName := ""
		node, nodeErr := models.SharedHTTPDNSNodeDAO.FindEnabledNode(tx, int64(stat.NodeId))
		if nodeErr == nil && node != nil {
			nodeName = node.Name
		}
		if len(nodeName) == 0 {
			continue
		}
		result.TopNodeStats = append(result.TopNodeStats, &pb.ComposeHTTPDNSBoardResponse_TopNodeStat{
			ClusterId:     int64(stat.ClusterId),
			NodeId:        int64(stat.NodeId),
			NodeName:      nodeName,
			CountRequests: stat.CountRequests,
		})
	}

	return nil
}

func (this *HTTPDNSBoardService) fillNodeValues(tx *dbs.Tx, result *pb.ComposeHTTPDNSBoardResponse) error {
	cpuValues, err := models.SharedNodeValueDAO.ListValuesForHTTPDNSNodes(tx, nodeconfigs.NodeValueItemCPU, "usage", nodeconfigs.NodeValueRangeMinute)
	if err != nil {
		return err
	}
	for _, v := range cpuValues {
		result.CpuNodeValues = append(result.CpuNodeValues, &pb.NodeValue{
			ValueJSON: v.Value,
			CreatedAt: int64(v.CreatedAt),
		})
	}

	memoryValues, err := models.SharedNodeValueDAO.ListValuesForHTTPDNSNodes(tx, nodeconfigs.NodeValueItemMemory, "usage", nodeconfigs.NodeValueRangeMinute)
	if err != nil {
		return err
	}
	for _, v := range memoryValues {
		result.MemoryNodeValues = append(result.MemoryNodeValues, &pb.NodeValue{
			ValueJSON: v.Value,
			CreatedAt: int64(v.CreatedAt),
		})
	}

	loadValues, err := models.SharedNodeValueDAO.ListValuesForHTTPDNSNodes(tx, nodeconfigs.NodeValueItemLoad, "load1m", nodeconfigs.NodeValueRangeMinute)
	if err != nil {
		return err
	}
	for _, v := range loadValues {
		result.LoadNodeValues = append(result.LoadNodeValues, &pb.NodeValue{
			ValueJSON: v.Value,
			CreatedAt: int64(v.CreatedAt),
		})
	}

	return nil
}