feat: sync httpdns sdk/platform updates without large binaries

This commit is contained in:
robin
2026-03-04 17:59:14 +08:00
parent 853897a6f8
commit 532891fad0
700 changed files with 6096 additions and 2712 deletions

View File

@@ -2,9 +2,12 @@ package httpdns
import (
"encoding/json"
"log"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
"github.com/iwind/TeaGo/dbs"
)
func toPBCluster(cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
@@ -25,9 +28,94 @@ func toPBCluster(cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
UpdatedAt: int64(cluster.UpdatedAt),
AutoRemoteStart: cluster.AutoRemoteStart,
AccessLogIsOn: cluster.AccessLogIsOn,
TimeZone: cluster.TimeZone,
}
}
// toPBClusterWithResolvedCerts 转换集群并解析证书引用为实际 PEM 数据
// 供节点调用的 RPC 使用,确保节点能拿到完整的证书内容
func toPBClusterWithResolvedCerts(tx *dbs.Tx, cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
pbCluster := toPBCluster(cluster)
if pbCluster == nil {
return nil
}
resolved := resolveTLSPolicyCerts(tx, cluster.TLSPolicy)
if resolved != nil {
pbCluster.TlsPolicyJSON = resolved
}
return pbCluster
}
// resolveTLSPolicyCerts 将 tlsPolicyJSON 中的 certRefs 解析为带实际 PEM 数据的 certs
func resolveTLSPolicyCerts(tx *dbs.Tx, tlsPolicyJSON []byte) []byte {
if len(tlsPolicyJSON) == 0 {
return nil
}
// 解析外层结构: {"listen": [...], "sslPolicy": {...}}
var tlsConfig map[string]json.RawMessage
if err := json.Unmarshal(tlsPolicyJSON, &tlsConfig); err != nil {
return nil
}
sslPolicyData, ok := tlsConfig["sslPolicy"]
if !ok || len(sslPolicyData) == 0 {
return nil
}
var sslPolicy sslconfigs.SSLPolicy
if err := json.Unmarshal(sslPolicyData, &sslPolicy); err != nil {
return nil
}
// 检查 certs 是否已经有实际数据
for _, cert := range sslPolicy.Certs {
if cert != nil && len(cert.CertData) > 128 && len(cert.KeyData) > 128 {
return nil // 已有完整 PEM 数据,无需处理
}
}
// 从 certRefs 解析实际证书数据
if len(sslPolicy.CertRefs) == 0 {
return nil
}
var resolvedCerts []*sslconfigs.SSLCertConfig
for _, ref := range sslPolicy.CertRefs {
if ref == nil || ref.CertId <= 0 {
continue
}
certConfig, err := models.SharedSSLCertDAO.ComposeCertConfig(tx, ref.CertId, false, nil, nil)
if err != nil {
log.Println("[HTTPDNS]resolve cert", ref.CertId, "failed:", err.Error())
continue
}
if certConfig == nil || len(certConfig.CertData) == 0 || len(certConfig.KeyData) == 0 {
continue
}
resolvedCerts = append(resolvedCerts, certConfig)
}
if len(resolvedCerts) == 0 {
return nil
}
// 把解析后的证书写回 sslPolicy.Certs
sslPolicy.Certs = resolvedCerts
newPolicyData, err := json.Marshal(&sslPolicy)
if err != nil {
return nil
}
tlsConfig["sslPolicy"] = newPolicyData
result, err := json.Marshal(tlsConfig)
if err != nil {
return nil
}
return result
}
func toPBNode(node *models.HTTPDNSNode) *pb.HTTPDNSNode {
if node == nil {
return nil

View File

@@ -67,10 +67,10 @@ func (this *HTTPDNSAppService) CreateHTTPDNSApp(ctx context.Context, req *pb.Cre
return errors.New("appId already exists")
}
// 使用 clusterIdsJSON若为空则优先从用户关联集群获取,再 fallback 到全局默认
// 使用 clusterIdsJSON若为空则从用户关联集群获取
clusterIdsJSON := req.ClusterIdsJSON
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
// 优先读取用户关联的 HTTPDNS 集群
// 读取用户关联的 HTTPDNS 集群
if req.UserId > 0 {
user, userErr := models.SharedUserDAO.FindEnabledUser(tx, req.UserId, nil)
if userErr != nil {
@@ -83,16 +83,11 @@ func (this *HTTPDNSAppService) CreateHTTPDNSApp(ctx context.Context, req *pb.Cre
}
}
}
// fallback 到全局默认
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
defaultClusterIds, defaultErr := readHTTPDNSDefaultClusterIdList(tx)
if defaultErr != nil {
return defaultErr
}
if len(defaultClusterIds) > 0 {
clusterIdsJSON, _ = json.Marshal(defaultClusterIds)
}
}
}
// 如果仍然没有集群,则不允许创建
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
return errors.New("用户尚未分配 HTTPDNS 集群,无法创建应用")
}
appDbId, err = models.SharedHTTPDNSAppDAO.CreateApp(tx, appName, appId, clusterIdsJSON, req.IsOn, req.UserId)
@@ -143,14 +138,6 @@ func readHTTPDNSDefaultClusterIdList(tx *dbs.Tx) ([]int64, error) {
}
}
// fallback默认主集群
primaryClusterId, err := models.SharedHTTPDNSClusterDAO.FindDefaultPrimaryClusterId(tx)
if err != nil {
return nil, err
}
if primaryClusterId > 0 {
return []int64{primaryClusterId}, nil
}
return nil, nil
}

View File

@@ -3,10 +3,14 @@ package httpdns
import (
"context"
"errors"
"fmt"
"strings"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/dbs"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
// HTTPDNSClusterService HTTPDNS集群服务
@@ -25,7 +29,7 @@ func (this *HTTPDNSClusterService) CreateHTTPDNSCluster(ctx context.Context, req
}
var clusterId int64
err = this.RunTx(func(tx *dbs.Tx) error {
clusterId, err = models.SharedHTTPDNSClusterDAO.CreateCluster(tx, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn)
clusterId, err = models.SharedHTTPDNSClusterDAO.CreateCluster(tx, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn, req.TimeZone)
if err != nil {
return err
}
@@ -42,13 +46,41 @@ func (this *HTTPDNSClusterService) UpdateHTTPDNSCluster(ctx context.Context, req
if err != nil {
return nil, err
}
// Compatibility fallback:
// If protobuf schemas between edge-admin and edge-api are inconsistent,
// these newly-added fields may be lost on the wire. Read gRPC metadata as fallback.
if md, ok := metadata.FromIncomingContext(ctx); ok {
if values := md.Get("x-httpdns-auto-remote-start"); len(values) > 0 {
raw := strings.ToLower(strings.TrimSpace(values[0]))
req.AutoRemoteStart = raw == "1" || raw == "true" || raw == "on" || raw == "yes" || raw == "enabled"
}
if values := md.Get("x-httpdns-access-log-is-on"); len(values) > 0 {
raw := strings.ToLower(strings.TrimSpace(values[0]))
req.AccessLogIsOn = raw == "1" || raw == "true" || raw == "on" || raw == "yes" || raw == "enabled"
}
if values := md.Get("x-httpdns-time-zone"); len(values) > 0 {
raw := strings.TrimSpace(values[0])
if len(raw) > 0 {
req.TimeZone = raw
}
}
}
err = this.RunTx(func(tx *dbs.Tx) error {
err = models.SharedHTTPDNSClusterDAO.UpdateCluster(tx, req.ClusterId, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn)
// 先读取旧的 TLS 配置,用于判断是否真正发生了变化
var oldTLSJSON string
oldCluster, findErr := models.SharedHTTPDNSClusterDAO.FindEnabledCluster(tx, req.ClusterId)
if findErr == nil && oldCluster != nil {
oldTLSJSON = string(oldCluster.TLSPolicy)
}
err = models.SharedHTTPDNSClusterDAO.UpdateCluster(tx, req.ClusterId, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn, req.TimeZone)
if err != nil {
return err
}
taskType := models.HTTPDNSNodeTaskTypeConfigChanged
if len(req.TlsPolicyJSON) > 0 {
if len(req.TlsPolicyJSON) > 0 && string(req.TlsPolicyJSON) != oldTLSJSON {
taskType = models.HTTPDNSNodeTaskTypeTLSChanged
}
return notifyHTTPDNSClusterTask(tx, req.ClusterId, taskType)
@@ -86,6 +118,13 @@ func (this *HTTPDNSClusterService) FindHTTPDNSCluster(ctx context.Context, req *
if err != nil {
return nil, err
}
if cluster != nil {
_ = grpc.SetHeader(ctx, metadata.Pairs(
"x-httpdns-auto-remote-start", fmt.Sprintf("%t", cluster.AutoRemoteStart),
"x-httpdns-access-log-is-on", fmt.Sprintf("%t", cluster.AccessLogIsOn),
"x-httpdns-time-zone", cluster.TimeZone,
))
}
return &pb.FindHTTPDNSClusterResponse{Cluster: toPBCluster(cluster)}, nil
}
@@ -107,10 +146,12 @@ func (this *HTTPDNSClusterService) ListHTTPDNSClusters(ctx context.Context, req
func (this *HTTPDNSClusterService) FindAllHTTPDNSClusters(ctx context.Context, req *pb.FindAllHTTPDNSClustersRequest) (*pb.FindAllHTTPDNSClustersResponse, error) {
_, _, validateErr := this.ValidateAdminAndUser(ctx, true)
isNode := false
if validateErr != nil {
if _, nodeErr := this.ValidateHTTPDNSNode(ctx); nodeErr != nil {
return nil, validateErr
}
isNode = true
}
clusters, err := models.SharedHTTPDNSClusterDAO.FindAllEnabledClusters(this.NullTx())
if err != nil {
@@ -118,7 +159,12 @@ func (this *HTTPDNSClusterService) FindAllHTTPDNSClusters(ctx context.Context, r
}
var pbClusters []*pb.HTTPDNSCluster
for _, cluster := range clusters {
pbClusters = append(pbClusters, toPBCluster(cluster))
if isNode {
// 节点调用时解析证书引用,嵌入实际 PEM 数据
pbClusters = append(pbClusters, toPBClusterWithResolvedCerts(this.NullTx(), cluster))
} else {
pbClusters = append(pbClusters, toPBCluster(cluster))
}
}
return &pb.FindAllHTTPDNSClustersResponse{Clusters: pbClusters}, nil
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/installers"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
"github.com/TeaOSLab/EdgeAPI/internal/setup"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
@@ -247,6 +248,12 @@ func (this *HTTPDNSNodeService) DownloadHTTPDNSNodeInstallationFile(ctx context.
return nil, err
}
// 检查自动升级开关
upgradeConfig, _ := setup.LoadUpgradeConfig()
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
return &pb.DownloadHTTPDNSNodeInstallationFileResponse{}, nil
}
var file = installers.SharedDeployManager.FindHTTPDNSNodeFile(req.Os, req.Arch)
if file == nil {
return &pb.DownloadHTTPDNSNodeInstallationFileResponse{}, nil
@@ -274,6 +281,120 @@ func (this *HTTPDNSNodeService) DownloadHTTPDNSNodeInstallationFile(ctx context.
}, nil
}
// CountAllUpgradeHTTPDNSNodesWithClusterId 计算需要升级的HTTPDNS节点数量
func (this *HTTPDNSNodeService) CountAllUpgradeHTTPDNSNodesWithClusterId(ctx context.Context, req *pb.CountAllUpgradeHTTPDNSNodesWithClusterIdRequest) (*pb.RPCCountResponse, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
deployFiles := installers.SharedDeployManager.LoadHTTPDNSNodeFiles()
total := int64(0)
for _, deployFile := range deployFiles {
count, err := models.SharedHTTPDNSNodeDAO.CountAllLowerVersionNodesWithClusterId(tx, req.ClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
if err != nil {
return nil, err
}
total += count
}
return this.SuccessCount(total)
}
// FindAllUpgradeHTTPDNSNodesWithClusterId 列出所有需要升级的HTTPDNS节点
func (this *HTTPDNSNodeService) FindAllUpgradeHTTPDNSNodesWithClusterId(ctx context.Context, req *pb.FindAllUpgradeHTTPDNSNodesWithClusterIdRequest) (*pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
deployFiles := installers.SharedDeployManager.LoadHTTPDNSNodeFiles()
var result []*pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse_HTTPDNSNodeUpgrade
for _, deployFile := range deployFiles {
nodes, err := models.SharedHTTPDNSNodeDAO.FindAllLowerVersionNodesWithClusterId(tx, req.ClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
if err != nil {
return nil, err
}
for _, node := range nodes {
// 解析状态获取当前版本
var oldVersion string
if len(node.Status) > 0 {
var statusMap map[string]interface{}
if json.Unmarshal(node.Status, &statusMap) == nil {
if v, ok := statusMap["buildVersion"]; ok {
oldVersion, _ = v.(string)
}
}
}
pbNode := toPBNode(node)
// 认证信息
login, loginErr := models.SharedNodeLoginDAO.FindEnabledNodeLoginWithNodeId(tx, nodeconfigs.NodeRoleHTTPDNS, int64(node.Id))
if loginErr != nil {
return nil, loginErr
}
if login != nil && pbNode != nil {
pbNode.NodeLogin = &pb.NodeLogin{
Id: int64(login.Id),
Name: login.Name,
Type: login.Type,
Params: login.Params,
}
}
result = append(result, &pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse_HTTPDNSNodeUpgrade{
Node: pbNode,
Os: deployFile.OS,
Arch: deployFile.Arch,
OldVersion: oldVersion,
NewVersion: deployFile.Version,
})
}
}
return &pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse{Nodes: result}, nil
}
// UpgradeHTTPDNSNode 升级单个HTTPDNS节点
func (this *HTTPDNSNodeService) UpgradeHTTPDNSNode(ctx context.Context, req *pb.UpgradeHTTPDNSNodeRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedHTTPDNSNodeDAO.UpdateNodeIsInstalled(tx, req.NodeId, false)
if err != nil {
return nil, err
}
// 重置安装状态
installStatus, err := models.SharedHTTPDNSNodeDAO.FindNodeInstallStatus(tx, req.NodeId)
if err != nil {
return nil, err
}
if installStatus == nil {
installStatus = &models.NodeInstallStatus{}
}
installStatus.IsOk = false
installStatus.IsFinished = false
err = models.SharedHTTPDNSNodeDAO.UpdateNodeInstallStatus(tx, req.NodeId, installStatus)
if err != nil {
return nil, err
}
goman.New(func() {
installErr := installers.SharedHTTPDNSNodeQueue().InstallNodeProcess(req.NodeId, true)
if installErr != nil {
logs.Println("[RPC][HTTPDNS]upgrade node failed:", installErr.Error())
}
})
return this.Success()
}
func shouldTriggerHTTPDNSInstall(installStatusJSON []byte) bool {
if len(installStatusJSON) == 0 {
return false

View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/installers"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
"github.com/TeaOSLab/EdgeAPI/internal/setup"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
@@ -484,6 +485,12 @@ func (this *NSNodeService) DownloadNSNodeInstallationFile(ctx context.Context, r
return nil, err
}
// 检查自动升级开关
upgradeConfig, _ := setup.LoadUpgradeConfig()
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
return &pb.DownloadNSNodeInstallationFileResponse{}, nil
}
var file = installers.SharedDeployManager.FindNSNodeFile(req.Os, req.Arch)
if file == nil {
return &pb.DownloadNSNodeInstallationFileResponse{}, nil
@@ -738,3 +745,109 @@ func (this *NSNodeService) UpdateNSNodeAPIConfig(ctx context.Context, req *pb.Up
return this.Success()
}
// FindAllUpgradeNSNodesWithNSClusterId 列出所有需要升级的NS节点
func (this *NSNodeService) FindAllUpgradeNSNodesWithNSClusterId(ctx context.Context, req *pb.FindAllUpgradeNSNodesWithNSClusterIdRequest) (*pb.FindAllUpgradeNSNodesWithNSClusterIdResponse, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
deployFiles := installers.SharedDeployManager.LoadNSNodeFiles()
var result []*pb.FindAllUpgradeNSNodesWithNSClusterIdResponse_NSNodeUpgrade
for _, deployFile := range deployFiles {
nodes, err := models.SharedNSNodeDAO.FindAllLowerVersionNodesWithClusterId(tx, req.NsClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
if err != nil {
return nil, err
}
for _, node := range nodes {
// 解析状态获取当前版本
var oldVersion string
if len(node.Status) > 0 {
var statusMap map[string]interface{}
if json.Unmarshal(node.Status, &statusMap) == nil {
if v, ok := statusMap["buildVersion"]; ok {
oldVersion, _ = v.(string)
}
}
}
// 安装信息
installStatus, installErr := node.DecodeInstallStatus()
if installErr != nil {
return nil, installErr
}
pbInstallStatus := &pb.NodeInstallStatus{}
if installStatus != nil {
pbInstallStatus = &pb.NodeInstallStatus{
IsRunning: installStatus.IsRunning,
IsFinished: installStatus.IsFinished,
IsOk: installStatus.IsOk,
Error: installStatus.Error,
ErrorCode: installStatus.ErrorCode,
UpdatedAt: installStatus.UpdatedAt,
}
}
// 认证信息
login, loginErr := models.SharedNodeLoginDAO.FindEnabledNodeLoginWithNodeId(tx, nodeconfigs.NodeRoleDNS, int64(node.Id))
if loginErr != nil {
return nil, loginErr
}
var pbLogin *pb.NodeLogin
if login != nil {
pbLogin = &pb.NodeLogin{
Id: int64(login.Id),
Name: login.Name,
Type: login.Type,
Params: login.Params,
}
}
result = append(result, &pb.FindAllUpgradeNSNodesWithNSClusterIdResponse_NSNodeUpgrade{
NsNode: &pb.NSNode{
Id: int64(node.Id),
Name: node.Name,
IsOn: node.IsOn,
UniqueId: node.UniqueId,
IsInstalled: node.IsInstalled,
IsUp: node.IsUp,
IsActive: node.IsActive,
StatusJSON: node.Status,
InstallStatus: pbInstallStatus,
NodeLogin: pbLogin,
},
Os: deployFile.OS,
Arch: deployFile.Arch,
OldVersion: oldVersion,
NewVersion: deployFile.Version,
})
}
}
return &pb.FindAllUpgradeNSNodesWithNSClusterIdResponse{Nodes: result}, nil
}
// UpgradeNSNode 升级单个NS节点
func (this *NSNodeService) UpgradeNSNode(ctx context.Context, req *pb.UpgradeNSNodeRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedNSNodeDAO.UpdateNodeIsInstalled(tx, req.NsNodeId, false)
if err != nil {
return nil, err
}
goman.New(func() {
installErr := installers.SharedNSNodeQueue().InstallNodeProcess(req.NsNodeId, true)
if installErr != nil {
logs.Println("[RPC]upgrade dns node:" + installErr.Error())
}
})
return this.Success()
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/installers"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeAPI/internal/setup"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
@@ -1716,6 +1717,12 @@ func (this *NodeService) DownloadNodeInstallationFile(ctx context.Context, req *
return nil, err
}
// 检查自动升级开关
upgradeConfig, _ := setup.LoadUpgradeConfig()
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
return &pb.DownloadNodeInstallationFileResponse{}, nil
}
var file = installers.SharedDeployManager.FindNodeFile(req.Os, req.Arch)
if file == nil {
return &pb.DownloadNodeInstallationFileResponse{}, nil