Initial commit (code only without large binaries)

EdgeAdmin/internal/web/actions/default/servers/logs/checkLog.go

@@ -0,0 +1,55 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+)
+
+// CheckLogAction 检查是否有日志
+type CheckLogAction struct {
+	actionutils.ParentAction
+}
+
+func (this *CheckLogAction) RunPost(params struct {
+	Day       string
+	Partition int32
+
+	ServerId          int64
+	RequestId         string
+	ClusterId         int64
+	NodeId            int64
+	HasError          bool
+	HasFirewallPolicy bool
+	Keyword           string
+	Ip                string
+	Domain            string
+
+	Hour string
+}) {
+	resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+		Partition:         params.Partition,
+		RequestId:         params.RequestId,
+		NodeClusterId:     params.ClusterId,
+		NodeId:            params.NodeId,
+		ServerId:          params.ServerId,
+		HasError:          params.HasError,
+		HasFirewallPolicy: params.HasFirewallPolicy,
+		Day:               params.Day,
+		HourFrom:          params.Hour,
+		HourTo:            params.Hour,
+		Keyword:           params.Keyword,
+		Ip:                params.Ip,
+		Domain:            params.Domain,
+		Size:              1,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	this.Data["hasLogs"] = len(resp.HttpAccessLogs) > 0
+
+	this.Success()
+}

EdgeAdmin/internal/web/actions/default/servers/logs/hasLogs.go

@@ -0,0 +1,67 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"regexp"
+	"strings"
+)
+
+// HasLogsAction 检查某个分区是否有日志
+type HasLogsAction struct {
+	actionutils.ParentAction
+}
+
+func (this *HasLogsAction) RunPost(params struct {
+	ClusterId int64
+	NodeId    int64
+	Day       string
+	Hour      string
+	Keyword   string
+	Ip        string
+	Domain    string
+	HasError  int
+	HasWAF    int
+	Partition int32 `default:"-1"`
+
+	RequestId string
+	ServerId  int64
+}) {
+	if len(params.Day) == 0 {
+		params.Day = timeutil.Format("Y-m-d")
+	}
+
+	var day = params.Day
+
+	if len(day) > 0 && regexp.MustCompile(`\d{4}-\d{2}-\d{2}`).MatchString(day) {
+		day = strings.ReplaceAll(day, "-", "")
+	}
+
+	resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+		Partition:         params.Partition,
+		RequestId:         params.RequestId,
+		NodeClusterId:     params.ClusterId,
+		NodeId:            params.NodeId,
+		ServerId:          params.ServerId,
+		HasError:          params.HasError > 0,
+		HasFirewallPolicy: params.HasWAF > 0,
+		Day:               day,
+		HourFrom:          params.Hour,
+		HourTo:            params.Hour,
+		Keyword:           params.Keyword,
+		Ip:                params.Ip,
+		Domain:            params.Domain,
+		Size:              1,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	this.Data["hasLogs"] = len(resp.HttpAccessLogs) > 0
+
+	this.Success()
+}

EdgeAdmin/internal/web/actions/default/servers/logs/index.go

@@ -0,0 +1,250 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/maps"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type IndexAction struct {
+	actionutils.ParentAction
+}
+
+func (this *IndexAction) Init() {
+	this.Nav("", "", "")
+}
+
+func (this *IndexAction) RunGet(params struct {
+	ClusterId int64
+	NodeId    int64
+	Day       string
+	Hour      string
+	Keyword   string
+	Ip        string
+	Domain    string
+	HasError  int
+	HasWAF    int
+	Partition int32 `default:"-1"`
+
+	RequestId string
+	ServerId  int64
+
+	PageSize int64
+}) {
+	if len(params.Day) == 0 {
+		params.Day = timeutil.Format("Y-m-d")
+	}
+
+	this.Data["clusterId"] = params.ClusterId
+	this.Data["nodeId"] = params.NodeId
+	this.Data["serverId"] = 0
+	this.Data["path"] = this.Request.URL.Path
+	this.Data["day"] = params.Day
+	this.Data["hour"] = params.Hour
+	this.Data["keyword"] = params.Keyword
+	this.Data["ip"] = params.Ip
+	this.Data["domain"] = params.Domain
+	this.Data["accessLogs"] = []interface{}{}
+	this.Data["hasError"] = params.HasError
+	this.Data["hasWAF"] = params.HasWAF
+	this.Data["pageSize"] = params.PageSize
+	this.Data["isSlowQuery"] = false
+	this.Data["slowQueryCost"] = ""
+	this.Data["partition"] = params.Partition
+
+	var day = params.Day
+	var ipList = []string{}
+	var wafMaps = []maps.Map{}
+
+	if len(day) > 0 && regexp.MustCompile(`\d{4}-\d{2}-\d{2}`).MatchString(day) {
+		day = strings.ReplaceAll(day, "-", "")
+		size := params.PageSize
+		if size < 1 {
+			size = 20
+		}
+
+		this.Data["hasError"] = params.HasError
+
+		var before = time.Now()
+		resp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+			Partition:         params.Partition,
+			RequestId:         params.RequestId,
+			NodeClusterId:     params.ClusterId,
+			NodeId:            params.NodeId,
+			ServerId:          params.ServerId,
+			HasError:          params.HasError > 0,
+			HasFirewallPolicy: params.HasWAF > 0,
+			Day:               day,
+			HourFrom:          params.Hour,
+			HourTo:            params.Hour,
+			Keyword:           params.Keyword,
+			Ip:                params.Ip,
+			Domain:            params.Domain,
+			Size:              size,
+		})
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+
+		var cost = time.Since(before).Seconds()
+		if cost > 5 {
+			this.Data["slowQueryCost"] = fmt.Sprintf("%.2f", cost)
+			this.Data["isSlowQuery"] = true
+		}
+
+		if len(resp.HttpAccessLogs) == 0 {
+			this.Data["accessLogs"] = []interface{}{}
+		} else {
+			this.Data["accessLogs"] = resp.HttpAccessLogs
+			for _, accessLog := range resp.HttpAccessLogs {
+				// IP信息集合
+				if len(accessLog.RemoteAddr) > 0 {
+					if !lists.ContainsString(ipList, accessLog.RemoteAddr) {
+						ipList = append(ipList, accessLog.RemoteAddr)
+					}
+				}
+
+				// WAF信息集合
+				if accessLog.FirewallPolicyId > 0 && accessLog.FirewallRuleGroupId > 0 && accessLog.FirewallRuleSetId > 0 {
+					// 检查Set是否已经存在
+					var existSet = false
+					for _, wafMap := range wafMaps {
+						if wafMap.GetInt64("setId") == accessLog.FirewallRuleSetId {
+							existSet = true
+							break
+						}
+					}
+					if !existSet {
+						wafMaps = append(wafMaps, maps.Map{
+							"policyId": accessLog.FirewallPolicyId,
+							"groupId":  accessLog.FirewallRuleGroupId,
+							"setId":    accessLog.FirewallRuleSetId,
+						})
+					}
+				}
+			}
+		}
+		this.Data["hasMore"] = resp.HasMore
+		this.Data["nextRequestId"] = resp.RequestId
+
+		// 上一个requestId
+		this.Data["hasPrev"] = false
+		this.Data["lastRequestId"] = ""
+		if len(params.RequestId) > 0 {
+			this.Data["hasPrev"] = true
+			prevResp, err := this.RPC().HTTPAccessLogRPC().ListHTTPAccessLogs(this.AdminContext(), &pb.ListHTTPAccessLogsRequest{
+				Partition:         params.Partition,
+				RequestId:         params.RequestId,
+				NodeClusterId:     params.ClusterId,
+				NodeId:            params.NodeId,
+				ServerId:          params.ServerId,
+				HasError:          params.HasError > 0,
+				HasFirewallPolicy: params.HasWAF > 0,
+				Day:               day,
+				HourFrom:          params.Hour,
+				HourTo:            params.Hour,
+				Keyword:           params.Keyword,
+				Ip:                params.Ip,
+				Domain:            params.Domain,
+				Size:              size,
+				Reverse:           true,
+			})
+			if err != nil {
+				this.ErrorPage(err)
+				return
+			}
+			if int64(len(prevResp.HttpAccessLogs)) == size {
+				this.Data["lastRequestId"] = prevResp.RequestId
+			}
+		}
+	}
+
+	// 根据IP查询区域
+	this.Data["regions"] = iplibrary.LookupIPSummaries(ipList)
+
+	// WAF相关
+	var wafInfos = map[int64]maps.Map{}                          // set id => WAF Map
+	var wafPolicyCacheMap = map[int64]*pb.HTTPFirewallPolicy{}   // id => *pb.HTTPFirewallPolicy
+	var wafGroupCacheMap = map[int64]*pb.HTTPFirewallRuleGroup{} // id => *pb.HTTPFirewallRuleGroup
+	var wafSetCacheMap = map[int64]*pb.HTTPFirewallRuleSet{}     // id => *pb.HTTPFirewallRuleSet
+	for _, wafMap := range wafMaps {
+		var policyId = wafMap.GetInt64("policyId")
+		var groupId = wafMap.GetInt64("groupId")
+		var setId = wafMap.GetInt64("setId")
+		if policyId > 0 {
+			pbPolicy, ok := wafPolicyCacheMap[policyId]
+			if !ok {
+				policyResp, err := this.RPC().HTTPFirewallPolicyRPC().FindEnabledHTTPFirewallPolicy(this.AdminContext(), &pb.FindEnabledHTTPFirewallPolicyRequest{HttpFirewallPolicyId: policyId})
+				if err != nil {
+					this.ErrorPage(err)
+					return
+				}
+				pbPolicy = policyResp.HttpFirewallPolicy
+				wafPolicyCacheMap[policyId] = pbPolicy
+			}
+			if pbPolicy != nil {
+				wafMap = maps.Map{
+					"policy": maps.Map{
+						"id":       pbPolicy.Id,
+						"name":     pbPolicy.Name,
+						"serverId": pbPolicy.ServerId,
+					},
+				}
+				if groupId > 0 {
+					pbGroup, ok := wafGroupCacheMap[groupId]
+					if !ok {
+						groupResp, err := this.RPC().HTTPFirewallRuleGroupRPC().FindEnabledHTTPFirewallRuleGroup(this.AdminContext(), &pb.FindEnabledHTTPFirewallRuleGroupRequest{FirewallRuleGroupId: groupId})
+						if err != nil {
+							this.ErrorPage(err)
+							return
+						}
+						pbGroup = groupResp.FirewallRuleGroup
+						wafGroupCacheMap[groupId] = pbGroup
+					}
+
+					if pbGroup != nil {
+						wafMap["group"] = maps.Map{
+							"id":   pbGroup.Id,
+							"name": pbGroup.Name,
+						}
+
+						if setId > 0 {
+							pbSet, ok := wafSetCacheMap[setId]
+							if !ok {
+								setResp, err := this.RPC().HTTPFirewallRuleSetRPC().FindEnabledHTTPFirewallRuleSet(this.AdminContext(), &pb.FindEnabledHTTPFirewallRuleSetRequest{FirewallRuleSetId: setId})
+								if err != nil {
+									this.ErrorPage(err)
+									return
+								}
+								pbSet = setResp.FirewallRuleSet
+								wafSetCacheMap[setId] = pbSet
+							}
+
+							if pbSet != nil {
+								wafMap["set"] = maps.Map{
+									"id":   pbSet.Id,
+									"name": pbSet.Name,
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+
+		wafInfos[setId] = wafMap
+	}
+	this.Data["wafInfos"] = wafInfos
+
+	this.Show()
+}

EdgeAdmin/internal/web/actions/default/servers/logs/init.go

@@ -0,0 +1,24 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/configloaders"
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/helpers"
+	"github.com/iwind/TeaGo"
+)
+
+func init() {
+	TeaGo.BeforeStart(func(server *TeaGo.Server) {
+		server.
+			Helper(helpers.NewUserMustAuth(configloaders.AdminModuleCodeServer)).
+			Data("teaMenu", "servers").
+			Data("teaSubMenu", "log").
+			Prefix("/servers/logs").
+			Get("", new(IndexAction)).
+			GetPost("/settings", new(SettingsAction)).
+			Post("/partitionData", new(PartitionDataAction)).
+			Post("/hasLogs", new(HasLogsAction)).
+			EndAll()
+	})
+}

EdgeAdmin/internal/web/actions/default/servers/logs/partitionData.go

@@ -0,0 +1,37 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"strings"
+)
+
+// PartitionDataAction 读取分区表
+type PartitionDataAction struct {
+	actionutils.ParentAction
+}
+
+func (this *PartitionDataAction) RunPost(params struct {
+	Day string
+}) {
+	var day = params.Day
+	day = strings.ReplaceAll(day, "-", "")
+
+	resp, err := this.RPC().HTTPAccessLogRPC().FindHTTPAccessLogPartitions(this.AdminContext(), &pb.FindHTTPAccessLogPartitionsRequest{
+		Day: day,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	if len(resp.Partitions) > 0 {
+		this.Data["partitions"] = resp.Partitions
+	} else {
+		this.Data["partitions"] = []int32{}
+	}
+
+	this.Success()
+}

EdgeAdmin/internal/web/actions/default/servers/logs/settings.go

@@ -0,0 +1,100 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package logs
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAdmin/internal/web/actions/actionutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
+	"github.com/iwind/TeaGo/actions"
+)
+
+type SettingsAction struct {
+	actionutils.ParentAction
+}
+
+func (this *SettingsAction) Init() {
+	this.Nav("", "", "settings")
+}
+
+func (this *SettingsAction) RunGet(params struct{}) {
+	settingsResp, err := this.RPC().SysSettingRPC().ReadSysSetting(this.AdminContext(), &pb.ReadSysSettingRequest{Code: systemconfigs.SettingCodeAccessLogQueue})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	var config = &serverconfigs.AccessLogQueueConfig{
+		MaxLength:         0,
+		CountPerSecond:    0,
+		Percent:           100,
+		EnableAutoPartial: true,
+		RowsPerTable:      500_000,
+	}
+	if len(settingsResp.ValueJSON) > 0 {
+		err = json.Unmarshal(settingsResp.ValueJSON, config)
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+	} else {
+		configJSON, err := json.Marshal(config)
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+		_, err = this.RPC().SysSettingRPC().UpdateSysSetting(this.AdminContext(), &pb.UpdateSysSettingRequest{
+			Code:      systemconfigs.SettingCodeAccessLogQueue,
+			ValueJSON: configJSON,
+		})
+		if err != nil {
+			this.ErrorPage(err)
+			return
+		}
+	}
+
+	this.Data["config"] = config
+
+	this.Show()
+}
+
+func (this *SettingsAction) RunPost(params struct {
+	Percent           int
+	CountPerSecond    int
+	MaxLength         int
+	EnableAutoPartial bool
+	RowsPerTable      int64
+
+	Must *actions.Must
+	CSRF *actionutils.CSRF
+}) {
+	params.Must.
+		Field("percent", params.Percent).
+		Gte(0, "请输入大于0的整数").
+		Lte(100, "请输入小于100的整数")
+
+	var config = &serverconfigs.AccessLogQueueConfig{
+		MaxLength:         params.MaxLength,
+		CountPerSecond:    params.CountPerSecond,
+		Percent:           params.Percent,
+		EnableAutoPartial: params.EnableAutoPartial,
+		RowsPerTable:      params.RowsPerTable,
+	}
+	configJSON, err := json.Marshal(config)
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+	_, err = this.RPC().SysSettingRPC().UpdateSysSetting(this.AdminContext(), &pb.UpdateSysSettingRequest{
+		Code:      systemconfigs.SettingCodeAccessLogQueue,
+		ValueJSON: configJSON,
+	})
+	if err != nil {
+		this.ErrorPage(err)
+		return
+	}
+
+	this.Success()
+}