feat: sync httpdns sdk/platform updates without large binaries
This commit is contained in:
@@ -3,5 +3,10 @@
|
||||
# generate 'internal/setup/sql.json' file
|
||||
|
||||
CWD="$(dirname "$0")"
|
||||
SQL_JSON="${CWD}/../internal/setup/sql.json"
|
||||
|
||||
go run "${CWD}"/../cmd/sql-dump/main.go -dir="${CWD}"
|
||||
if [ -f "$SQL_JSON" ]; then
|
||||
echo "sql.json already exists, skipping sql-dump (delete it manually to regenerate)"
|
||||
else
|
||||
go run "${CWD}"/../cmd/sql-dump/main.go -dir="${CWD}"
|
||||
fi
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
Tea.Env = "prod"
|
||||
|
||||
db, err := dbs.Default()
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.8" //1.3.9
|
||||
Version = "1.4.9" //1.3.9
|
||||
|
||||
ProductName = "Edge API"
|
||||
ProcessName = "edge-api"
|
||||
@@ -17,6 +17,6 @@ const (
|
||||
|
||||
// 其他节点版本号,用来检测是否有需要升级的节点
|
||||
|
||||
NodeVersion = "1.4.8" //1.3.8.2
|
||||
NodeVersion = "1.4.9" //1.3.8.2
|
||||
)
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
DNSNodeVersion = "1.4.8" //1.3.8.2
|
||||
UserNodeVersion = "1.4.8" //1.3.8.2
|
||||
DNSNodeVersion = "1.4.9" //1.3.8.2
|
||||
UserNodeVersion = "1.4.9" //1.3.8.2
|
||||
ReportNodeVersion = "0.1.5"
|
||||
|
||||
DefaultMaxNodes int32 = 50
|
||||
|
||||
@@ -34,7 +34,7 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
func (this *HTTPDNSClusterDAO) CreateCluster(tx *dbs.Tx, name string, serviceDomain string, defaultTTL int32, fallbackTimeoutMs int32, installDir string, tlsPolicyJSON []byte, isOn bool, isDefault bool, autoRemoteStart bool, accessLogIsOn bool) (int64, error) {
|
||||
func (this *HTTPDNSClusterDAO) CreateCluster(tx *dbs.Tx, name string, serviceDomain string, defaultTTL int32, fallbackTimeoutMs int32, installDir string, tlsPolicyJSON []byte, isOn bool, isDefault bool, autoRemoteStart bool, accessLogIsOn bool, timeZone string) (int64, error) {
|
||||
if isDefault {
|
||||
err := this.Query(tx).
|
||||
State(HTTPDNSClusterStateEnabled).
|
||||
@@ -55,6 +55,7 @@ func (this *HTTPDNSClusterDAO) CreateCluster(tx *dbs.Tx, name string, serviceDom
|
||||
op.IsDefault = isDefault
|
||||
op.AutoRemoteStart = autoRemoteStart
|
||||
op.AccessLogIsOn = accessLogIsOn
|
||||
op.TimeZone = timeZone
|
||||
op.CreatedAt = time.Now().Unix()
|
||||
op.UpdatedAt = time.Now().Unix()
|
||||
op.State = HTTPDNSClusterStateEnabled
|
||||
@@ -68,7 +69,7 @@ func (this *HTTPDNSClusterDAO) CreateCluster(tx *dbs.Tx, name string, serviceDom
|
||||
return types.Int64(op.Id), nil
|
||||
}
|
||||
|
||||
func (this *HTTPDNSClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name string, serviceDomain string, defaultTTL int32, fallbackTimeoutMs int32, installDir string, tlsPolicyJSON []byte, isOn bool, isDefault bool, autoRemoteStart bool, accessLogIsOn bool) error {
|
||||
func (this *HTTPDNSClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name string, serviceDomain string, defaultTTL int32, fallbackTimeoutMs int32, installDir string, tlsPolicyJSON []byte, isOn bool, isDefault bool, autoRemoteStart bool, accessLogIsOn bool, timeZone string) error {
|
||||
if isDefault {
|
||||
err := this.Query(tx).
|
||||
State(HTTPDNSClusterStateEnabled).
|
||||
@@ -91,6 +92,7 @@ func (this *HTTPDNSClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name s
|
||||
op.IsDefault = isDefault
|
||||
op.AutoRemoteStart = autoRemoteStart
|
||||
op.AccessLogIsOn = accessLogIsOn
|
||||
op.TimeZone = timeZone
|
||||
op.UpdatedAt = time.Now().Unix()
|
||||
if len(tlsPolicyJSON) > 0 {
|
||||
op.TLSPolicy = tlsPolicyJSON
|
||||
|
||||
@@ -15,6 +15,7 @@ type HTTPDNSCluster struct {
|
||||
TLSPolicy dbs.JSON `field:"tlsPolicy"` // TLS策略
|
||||
AutoRemoteStart bool `field:"autoRemoteStart"` // 自动远程启动
|
||||
AccessLogIsOn bool `field:"accessLogIsOn"` // 访问日志是否开启
|
||||
TimeZone string `field:"timeZone"` // 时区
|
||||
CreatedAt uint64 `field:"createdAt"` // 创建时间
|
||||
UpdatedAt uint64 `field:"updatedAt"` // 修改时间
|
||||
State uint8 `field:"state"` // 记录状态
|
||||
@@ -32,6 +33,7 @@ type HTTPDNSClusterOperator struct {
|
||||
TLSPolicy any // TLS策略
|
||||
AutoRemoteStart any // 自动远程启动
|
||||
AccessLogIsOn any // 访问日志是否开启
|
||||
TimeZone any // 时区
|
||||
CreatedAt any // 创建时间
|
||||
UpdatedAt any // 修改时间
|
||||
State any // 记录状态
|
||||
|
||||
@@ -2,6 +2,7 @@ package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -12,8 +13,8 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
HTTPDNSNodeStateEnabled = 1 // 已启用
|
||||
HTTPDNSNodeStateDisabled = 0 // 已禁用
|
||||
HTTPDNSNodeStateEnabled = 1
|
||||
HTTPDNSNodeStateDisabled = 0
|
||||
)
|
||||
|
||||
type HTTPDNSNodeDAO dbs.DAO
|
||||
@@ -37,7 +38,7 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// FindEnabledNodeIdWithUniqueId 根据唯一ID获取启用中的HTTPDNS节点ID
|
||||
// FindEnabledNodeIdWithUniqueId 鏍规嵁鍞竴ID鑾峰彇鍚敤涓殑HTTPDNS鑺傜偣ID
|
||||
func (this *HTTPDNSNodeDAO) FindEnabledNodeIdWithUniqueId(tx *dbs.Tx, uniqueId string) (int64, error) {
|
||||
return this.Query(tx).
|
||||
Attr("uniqueId", uniqueId).
|
||||
@@ -46,7 +47,7 @@ func (this *HTTPDNSNodeDAO) FindEnabledNodeIdWithUniqueId(tx *dbs.Tx, uniqueId s
|
||||
FindInt64Col(0)
|
||||
}
|
||||
|
||||
// CreateNode 创建节点
|
||||
// CreateNode 鍒涘缓鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) CreateNode(tx *dbs.Tx, clusterId int64, name string, installDir string, isOn bool) (int64, error) {
|
||||
uniqueId := rands.HexString(32)
|
||||
secret := rands.String(32)
|
||||
@@ -75,7 +76,7 @@ func (this *HTTPDNSNodeDAO) CreateNode(tx *dbs.Tx, clusterId int64, name string,
|
||||
return types.Int64(op.Id), nil
|
||||
}
|
||||
|
||||
// UpdateNode 更新节点
|
||||
// UpdateNode 鏇存柊鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, installDir string, isOn bool) error {
|
||||
var op = NewHTTPDNSNodeOperator()
|
||||
op.Id = nodeId
|
||||
@@ -86,7 +87,7 @@ func (this *HTTPDNSNodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, in
|
||||
return this.Save(tx, op)
|
||||
}
|
||||
|
||||
// DisableNode 禁用节点
|
||||
// DisableNode 绂佺敤鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) DisableNode(tx *dbs.Tx, nodeId int64) error {
|
||||
node, err := this.FindEnabledNode(tx, nodeId)
|
||||
if err != nil {
|
||||
@@ -112,7 +113,7 @@ func (this *HTTPDNSNodeDAO) DisableNode(tx *dbs.Tx, nodeId int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// FindEnabledNode 查找启用节点
|
||||
// FindEnabledNode 鏌ユ壘鍚敤鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) FindEnabledNode(tx *dbs.Tx, nodeId int64) (*HTTPDNSNode, error) {
|
||||
one, err := this.Query(tx).
|
||||
Pk(nodeId).
|
||||
@@ -124,7 +125,7 @@ func (this *HTTPDNSNodeDAO) FindEnabledNode(tx *dbs.Tx, nodeId int64) (*HTTPDNSN
|
||||
return one.(*HTTPDNSNode), nil
|
||||
}
|
||||
|
||||
// FindNodeClusterId 查询节点所属集群ID
|
||||
// FindNodeClusterId 鏌ヨ鑺傜偣鎵€灞為泦缇D
|
||||
func (this *HTTPDNSNodeDAO) FindNodeClusterId(tx *dbs.Tx, nodeId int64) (int64, error) {
|
||||
return this.Query(tx).
|
||||
Pk(nodeId).
|
||||
@@ -133,7 +134,7 @@ func (this *HTTPDNSNodeDAO) FindNodeClusterId(tx *dbs.Tx, nodeId int64) (int64,
|
||||
FindInt64Col(0)
|
||||
}
|
||||
|
||||
// ListEnabledNodes 列出节点
|
||||
// ListEnabledNodes 鍒楀嚭鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) ListEnabledNodes(tx *dbs.Tx, clusterId int64) (result []*HTTPDNSNode, err error) {
|
||||
query := this.Query(tx).
|
||||
State(HTTPDNSNodeStateEnabled).
|
||||
@@ -145,6 +146,20 @@ func (this *HTTPDNSNodeDAO) ListEnabledNodes(tx *dbs.Tx, clusterId int64) (resul
|
||||
return
|
||||
}
|
||||
|
||||
// FindAllInactiveNodesWithClusterId 取得一个集群离线的HTTPDNS节点
|
||||
func (this *HTTPDNSNodeDAO) FindAllInactiveNodesWithClusterId(tx *dbs.Tx, clusterId int64) (result []*HTTPDNSNode, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(HTTPDNSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true). // 只监控启用的节点
|
||||
Attr("isInstalled", true). // 只监控已经安装的节点
|
||||
Attr("isActive", false). // 当前处于离线状态
|
||||
Result("id", "name").
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateNodeStatus 更新节点状态
|
||||
func (this *HTTPDNSNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, isUp bool, isInstalled bool, isActive bool, statusJSON []byte, installStatusJSON []byte) error {
|
||||
var op = NewHTTPDNSNodeOperator()
|
||||
@@ -261,7 +276,46 @@ func (this *HTTPDNSNodeDAO) FindNodeInstallStatus(tx *dbs.Tx, nodeId int64) (*No
|
||||
return installStatus, nil
|
||||
}
|
||||
|
||||
// UpdateNodeIsInstalled 更新节点安装状态位
|
||||
// CountAllLowerVersionNodesWithClusterId 璁$畻鍗曚釜闆嗙兢涓墍鏈変綆浜庢煇涓増鏈殑鑺傜偣鏁伴噺
|
||||
func (this *HTTPDNSNodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterId int64, os string, arch string, version string) (int64, error) {
|
||||
return this.Query(tx).
|
||||
State(HTTPDNSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
Where("JSON_EXTRACT(status, '$.arch')=:arch").
|
||||
Where("(JSON_EXTRACT(status, '$.buildVersionCode') IS NULL OR JSON_EXTRACT(status, '$.buildVersionCode')<:version)").
|
||||
Param("os", os).
|
||||
Param("arch", arch).
|
||||
Param("version", utils.VersionToLong(version)).
|
||||
Count()
|
||||
}
|
||||
|
||||
// FindAllLowerVersionNodesWithClusterId 鏌ユ壘鍗曚釜闆嗙兢涓墍鏈変綆浜庢煇涓増鏈殑鑺傜偣
|
||||
func (this *HTTPDNSNodeDAO) FindAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterId int64, os string, arch string, version string) (result []*HTTPDNSNode, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(HTTPDNSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
Where("JSON_EXTRACT(status, '$.arch')=:arch").
|
||||
Where("(JSON_EXTRACT(status, '$.buildVersionCode') IS NULL OR JSON_EXTRACT(status, '$.buildVersionCode')<:version)").
|
||||
Param("os", os).
|
||||
Param("arch", arch).
|
||||
Param("version", utils.VersionToLong(version)).
|
||||
DescPk().
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateNodeIsInstalled 鏇存柊鑺傜偣瀹夎鐘舵€佷綅
|
||||
func (this *HTTPDNSNodeDAO) UpdateNodeIsInstalled(tx *dbs.Tx, nodeId int64, isInstalled bool) error {
|
||||
_, err := this.Query(tx).
|
||||
Pk(nodeId).
|
||||
|
||||
@@ -1521,6 +1521,8 @@ func (this *NodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterI
|
||||
return this.Query(tx).
|
||||
State(NodeStateEnabled).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Attr("clusterId", clusterId).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
@@ -1536,6 +1538,9 @@ func (this *NodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterI
|
||||
func (this *NodeDAO) FindAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterId int64, os string, arch string, version string) (result []*Node, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(NodeStateEnabled).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Attr("clusterId", clusterId).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
|
||||
@@ -94,6 +94,8 @@ func (this *NSNodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, cluste
|
||||
State(NSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
Where("JSON_EXTRACT(status, '$.arch')=:arch").
|
||||
@@ -104,6 +106,27 @@ func (this *NSNodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, cluste
|
||||
Count()
|
||||
}
|
||||
|
||||
// FindAllLowerVersionNodesWithClusterId 查找单个集群中所有低于某个版本的节点
|
||||
func (this *NSNodeDAO) FindAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterId int64, os string, arch string, version string) (result []*NSNode, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(NSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
Where("JSON_EXTRACT(status, '$.arch')=:arch").
|
||||
Where("(JSON_EXTRACT(status, '$.buildVersionCode') IS NULL OR JSON_EXTRACT(status, '$.buildVersionCode')<:version)").
|
||||
Param("os", os).
|
||||
Param("arch", arch).
|
||||
Param("version", utils.VersionToLong(version)).
|
||||
DescPk().
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// FindEnabledNodeIdWithUniqueId 根据唯一ID获取节点ID
|
||||
func (this *NSNodeDAO) FindEnabledNodeIdWithUniqueId(tx *dbs.Tx, uniqueId string) (int64, error) {
|
||||
return this.Query(tx).
|
||||
|
||||
@@ -209,6 +209,8 @@ func (this *NSNodeDAO) CountAllLowerVersionNodesWithClusterId(tx *dbs.Tx, cluste
|
||||
return this.Query(tx).
|
||||
State(NSNodeStateEnabled).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Attr("clusterId", clusterId).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
@@ -412,6 +414,27 @@ func (this *NSNodeDAO) CountAllLowerVersionNodes(tx *dbs.Tx, version string) (in
|
||||
Count()
|
||||
}
|
||||
|
||||
// FindAllLowerVersionNodesWithClusterId 查找单个集群中所有低于某个版本的节点
|
||||
func (this *NSNodeDAO) FindAllLowerVersionNodesWithClusterId(tx *dbs.Tx, clusterId int64, os string, arch string, version string) (result []*NSNode, err error) {
|
||||
_, err = this.Query(tx).
|
||||
State(NSNodeStateEnabled).
|
||||
Attr("clusterId", clusterId).
|
||||
Attr("isOn", true).
|
||||
Attr("isUp", true).
|
||||
Attr("isActive", true).
|
||||
Where("status IS NOT NULL").
|
||||
Where("JSON_EXTRACT(status, '$.os')=:os").
|
||||
Where("JSON_EXTRACT(status, '$.arch')=:arch").
|
||||
Where("(JSON_EXTRACT(status, '$.buildVersionCode') IS NULL OR JSON_EXTRACT(status, '$.buildVersionCode')<:version)").
|
||||
Param("os", os).
|
||||
Param("arch", arch).
|
||||
Param("version", utils.VersionToLong(version)).
|
||||
DescPk().
|
||||
Slice(&result).
|
||||
FindAll()
|
||||
return
|
||||
}
|
||||
|
||||
// ComposeNodeConfig 组合节点配置
|
||||
func (this *NSNodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64) (*dnsconfigs.NSNodeConfig, error) {
|
||||
if nodeId <= 0 {
|
||||
|
||||
@@ -187,6 +187,100 @@ func (q *HTTPDNSNodeQueue) InstallNode(nodeId int64, installStatus *models.NodeI
|
||||
return installer.Install(installDir, params, installStatus)
|
||||
}
|
||||
|
||||
// StartNode 启动HTTPDNS节点
|
||||
func (q *HTTPDNSNodeQueue) StartNode(nodeId int64) error {
|
||||
node, err := models.SharedHTTPDNSNodeDAO.FindEnabledNode(nil, nodeId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if node == nil {
|
||||
return errors.New("can not find node, ID '" + numberutils.FormatInt64(nodeId) + "'")
|
||||
}
|
||||
|
||||
// 登录信息
|
||||
login, err := models.SharedNodeLoginDAO.FindEnabledNodeLoginWithNodeId(nil, nodeconfigs.NodeRoleHTTPDNS, nodeId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if login == nil {
|
||||
return newGrantError("can not find node login information")
|
||||
}
|
||||
loginParams, err := login.DecodeSSHParams()
|
||||
if err != nil {
|
||||
return newGrantError(err.Error())
|
||||
}
|
||||
if len(strings.TrimSpace(loginParams.Host)) == 0 {
|
||||
return newGrantError("ssh host should not be empty")
|
||||
}
|
||||
if loginParams.Port <= 0 {
|
||||
loginParams.Port = 22
|
||||
}
|
||||
if loginParams.GrantId <= 0 {
|
||||
return newGrantError("can not find node grant")
|
||||
}
|
||||
|
||||
grant, err := models.SharedNodeGrantDAO.FindEnabledNodeGrant(nil, loginParams.GrantId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if grant == nil {
|
||||
return newGrantError("can not find user grant with id '" + numberutils.FormatInt64(loginParams.GrantId) + "'")
|
||||
}
|
||||
|
||||
installer := &HTTPDNSNodeInstaller{}
|
||||
err = installer.Login(&Credentials{
|
||||
Host: strings.TrimSpace(loginParams.Host),
|
||||
Port: loginParams.Port,
|
||||
Username: grant.Username,
|
||||
Password: grant.Password,
|
||||
PrivateKey: grant.PrivateKey,
|
||||
Passphrase: grant.Passphrase,
|
||||
Method: grant.Method,
|
||||
Sudo: grant.Su == 1,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = installer.Close()
|
||||
}()
|
||||
|
||||
installDir := strings.TrimSpace(node.InstallDir)
|
||||
if len(installDir) == 0 {
|
||||
cluster, err := models.SharedHTTPDNSClusterDAO.FindEnabledCluster(nil, int64(node.ClusterId))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cluster == nil {
|
||||
return errors.New("can not find cluster, ID '" + numberutils.FormatInt64(int64(node.ClusterId)) + "'")
|
||||
}
|
||||
installDir = strings.TrimSpace(cluster.InstallDir)
|
||||
if len(installDir) == 0 {
|
||||
installDir = installer.client.UserHome() + "/edge-httpdns"
|
||||
}
|
||||
}
|
||||
_, appDir := resolveHTTPDNSInstallPaths(installDir)
|
||||
exeFile := appDir + "/bin/edge-httpdns"
|
||||
|
||||
_, err = installer.client.Stat(exeFile)
|
||||
if err != nil {
|
||||
return errors.New("httpdns node is not installed correctly, can not find executable file: " + exeFile)
|
||||
}
|
||||
|
||||
// 先尝试 systemd 拉起
|
||||
_, _, _ = installer.client.Exec("/usr/bin/systemctl start edge-httpdns")
|
||||
|
||||
_, stderr, err := installer.client.Exec(exeFile + " start")
|
||||
if err != nil {
|
||||
return fmt.Errorf("start failed: %w", err)
|
||||
}
|
||||
if len(strings.TrimSpace(stderr)) > 0 {
|
||||
return errors.New("start failed: " + strings.TrimSpace(stderr))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *HTTPDNSNodeQueue) resolveClusterTLSCertPair(cluster *models.HTTPDNSCluster) ([]byte, []byte, error) {
|
||||
if cluster == nil {
|
||||
return nil, nil, errors.New("cluster not found")
|
||||
|
||||
25
EdgeAPI/internal/installers/upgrade_queue.go
Normal file
25
EdgeAPI/internal/installers/upgrade_queue.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package installers
|
||||
|
||||
// UpgradeQueue 升级队列,控制并发数
|
||||
type UpgradeQueue struct {
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// SharedUpgradeQueue 全局升级队列,最多5个并发
|
||||
var SharedUpgradeQueue = NewUpgradeQueue(5)
|
||||
|
||||
// NewUpgradeQueue 创建升级队列
|
||||
func NewUpgradeQueue(maxConcurrent int) *UpgradeQueue {
|
||||
return &UpgradeQueue{
|
||||
sem: make(chan struct{}, maxConcurrent),
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitNodeUpgrade 提交节点升级任务(异步执行,超过并发限制自动排队)
|
||||
func (q *UpgradeQueue) SubmitNodeUpgrade(nodeId int64, upgradeFunc func(int64) error) {
|
||||
go func() {
|
||||
q.sem <- struct{}{}
|
||||
defer func() { <-q.sem }()
|
||||
_ = upgradeFunc(nodeId)
|
||||
}()
|
||||
}
|
||||
@@ -2,9 +2,12 @@ package httpdns
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
)
|
||||
|
||||
func toPBCluster(cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
|
||||
@@ -25,9 +28,94 @@ func toPBCluster(cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
|
||||
UpdatedAt: int64(cluster.UpdatedAt),
|
||||
AutoRemoteStart: cluster.AutoRemoteStart,
|
||||
AccessLogIsOn: cluster.AccessLogIsOn,
|
||||
TimeZone: cluster.TimeZone,
|
||||
}
|
||||
}
|
||||
|
||||
// toPBClusterWithResolvedCerts 转换集群并解析证书引用为实际 PEM 数据
|
||||
// 供节点调用的 RPC 使用,确保节点能拿到完整的证书内容
|
||||
func toPBClusterWithResolvedCerts(tx *dbs.Tx, cluster *models.HTTPDNSCluster) *pb.HTTPDNSCluster {
|
||||
pbCluster := toPBCluster(cluster)
|
||||
if pbCluster == nil {
|
||||
return nil
|
||||
}
|
||||
resolved := resolveTLSPolicyCerts(tx, cluster.TLSPolicy)
|
||||
if resolved != nil {
|
||||
pbCluster.TlsPolicyJSON = resolved
|
||||
}
|
||||
return pbCluster
|
||||
}
|
||||
|
||||
// resolveTLSPolicyCerts 将 tlsPolicyJSON 中的 certRefs 解析为带实际 PEM 数据的 certs
|
||||
func resolveTLSPolicyCerts(tx *dbs.Tx, tlsPolicyJSON []byte) []byte {
|
||||
if len(tlsPolicyJSON) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 解析外层结构: {"listen": [...], "sslPolicy": {...}}
|
||||
var tlsConfig map[string]json.RawMessage
|
||||
if err := json.Unmarshal(tlsPolicyJSON, &tlsConfig); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
sslPolicyData, ok := tlsConfig["sslPolicy"]
|
||||
if !ok || len(sslPolicyData) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var sslPolicy sslconfigs.SSLPolicy
|
||||
if err := json.Unmarshal(sslPolicyData, &sslPolicy); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查 certs 是否已经有实际数据
|
||||
for _, cert := range sslPolicy.Certs {
|
||||
if cert != nil && len(cert.CertData) > 128 && len(cert.KeyData) > 128 {
|
||||
return nil // 已有完整 PEM 数据,无需处理
|
||||
}
|
||||
}
|
||||
|
||||
// 从 certRefs 解析实际证书数据
|
||||
if len(sslPolicy.CertRefs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var resolvedCerts []*sslconfigs.SSLCertConfig
|
||||
for _, ref := range sslPolicy.CertRefs {
|
||||
if ref == nil || ref.CertId <= 0 {
|
||||
continue
|
||||
}
|
||||
certConfig, err := models.SharedSSLCertDAO.ComposeCertConfig(tx, ref.CertId, false, nil, nil)
|
||||
if err != nil {
|
||||
log.Println("[HTTPDNS]resolve cert", ref.CertId, "failed:", err.Error())
|
||||
continue
|
||||
}
|
||||
if certConfig == nil || len(certConfig.CertData) == 0 || len(certConfig.KeyData) == 0 {
|
||||
continue
|
||||
}
|
||||
resolvedCerts = append(resolvedCerts, certConfig)
|
||||
}
|
||||
|
||||
if len(resolvedCerts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 把解析后的证书写回 sslPolicy.Certs
|
||||
sslPolicy.Certs = resolvedCerts
|
||||
|
||||
newPolicyData, err := json.Marshal(&sslPolicy)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
tlsConfig["sslPolicy"] = newPolicyData
|
||||
|
||||
result, err := json.Marshal(tlsConfig)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toPBNode(node *models.HTTPDNSNode) *pb.HTTPDNSNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
|
||||
@@ -67,10 +67,10 @@ func (this *HTTPDNSAppService) CreateHTTPDNSApp(ctx context.Context, req *pb.Cre
|
||||
return errors.New("appId already exists")
|
||||
}
|
||||
|
||||
// 使用 clusterIdsJSON;若为空则优先从用户关联集群获取,再 fallback 到全局默认
|
||||
// 使用 clusterIdsJSON;若为空则从用户关联集群获取
|
||||
clusterIdsJSON := req.ClusterIdsJSON
|
||||
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
|
||||
// 优先读取用户关联的 HTTPDNS 集群
|
||||
// 读取用户关联的 HTTPDNS 集群
|
||||
if req.UserId > 0 {
|
||||
user, userErr := models.SharedUserDAO.FindEnabledUser(tx, req.UserId, nil)
|
||||
if userErr != nil {
|
||||
@@ -83,16 +83,11 @@ func (this *HTTPDNSAppService) CreateHTTPDNSApp(ctx context.Context, req *pb.Cre
|
||||
}
|
||||
}
|
||||
}
|
||||
// fallback 到全局默认
|
||||
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
|
||||
defaultClusterIds, defaultErr := readHTTPDNSDefaultClusterIdList(tx)
|
||||
if defaultErr != nil {
|
||||
return defaultErr
|
||||
}
|
||||
if len(defaultClusterIds) > 0 {
|
||||
clusterIdsJSON, _ = json.Marshal(defaultClusterIds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果仍然没有集群,则不允许创建
|
||||
if len(clusterIdsJSON) == 0 || string(clusterIdsJSON) == "[]" || string(clusterIdsJSON) == "null" {
|
||||
return errors.New("用户尚未分配 HTTPDNS 集群,无法创建应用")
|
||||
}
|
||||
|
||||
appDbId, err = models.SharedHTTPDNSAppDAO.CreateApp(tx, appName, appId, clusterIdsJSON, req.IsOn, req.UserId)
|
||||
@@ -143,14 +138,6 @@ func readHTTPDNSDefaultClusterIdList(tx *dbs.Tx) ([]int64, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// fallback:默认主集群
|
||||
primaryClusterId, err := models.SharedHTTPDNSClusterDAO.FindDefaultPrimaryClusterId(tx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if primaryClusterId > 0 {
|
||||
return []int64{primaryClusterId}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,14 @@ package httpdns
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
// HTTPDNSClusterService HTTPDNS集群服务
|
||||
@@ -25,7 +29,7 @@ func (this *HTTPDNSClusterService) CreateHTTPDNSCluster(ctx context.Context, req
|
||||
}
|
||||
var clusterId int64
|
||||
err = this.RunTx(func(tx *dbs.Tx) error {
|
||||
clusterId, err = models.SharedHTTPDNSClusterDAO.CreateCluster(tx, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn)
|
||||
clusterId, err = models.SharedHTTPDNSClusterDAO.CreateCluster(tx, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn, req.TimeZone)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -42,13 +46,41 @@ func (this *HTTPDNSClusterService) UpdateHTTPDNSCluster(ctx context.Context, req
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Compatibility fallback:
|
||||
// If protobuf schemas between edge-admin and edge-api are inconsistent,
|
||||
// these newly-added fields may be lost on the wire. Read gRPC metadata as fallback.
|
||||
if md, ok := metadata.FromIncomingContext(ctx); ok {
|
||||
if values := md.Get("x-httpdns-auto-remote-start"); len(values) > 0 {
|
||||
raw := strings.ToLower(strings.TrimSpace(values[0]))
|
||||
req.AutoRemoteStart = raw == "1" || raw == "true" || raw == "on" || raw == "yes" || raw == "enabled"
|
||||
}
|
||||
if values := md.Get("x-httpdns-access-log-is-on"); len(values) > 0 {
|
||||
raw := strings.ToLower(strings.TrimSpace(values[0]))
|
||||
req.AccessLogIsOn = raw == "1" || raw == "true" || raw == "on" || raw == "yes" || raw == "enabled"
|
||||
}
|
||||
if values := md.Get("x-httpdns-time-zone"); len(values) > 0 {
|
||||
raw := strings.TrimSpace(values[0])
|
||||
if len(raw) > 0 {
|
||||
req.TimeZone = raw
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = this.RunTx(func(tx *dbs.Tx) error {
|
||||
err = models.SharedHTTPDNSClusterDAO.UpdateCluster(tx, req.ClusterId, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn)
|
||||
// 先读取旧的 TLS 配置,用于判断是否真正发生了变化
|
||||
var oldTLSJSON string
|
||||
oldCluster, findErr := models.SharedHTTPDNSClusterDAO.FindEnabledCluster(tx, req.ClusterId)
|
||||
if findErr == nil && oldCluster != nil {
|
||||
oldTLSJSON = string(oldCluster.TLSPolicy)
|
||||
}
|
||||
|
||||
err = models.SharedHTTPDNSClusterDAO.UpdateCluster(tx, req.ClusterId, req.Name, req.ServiceDomain, req.DefaultTTL, req.FallbackTimeoutMs, req.InstallDir, req.TlsPolicyJSON, req.IsOn, req.IsDefault, req.AutoRemoteStart, req.AccessLogIsOn, req.TimeZone)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
taskType := models.HTTPDNSNodeTaskTypeConfigChanged
|
||||
if len(req.TlsPolicyJSON) > 0 {
|
||||
if len(req.TlsPolicyJSON) > 0 && string(req.TlsPolicyJSON) != oldTLSJSON {
|
||||
taskType = models.HTTPDNSNodeTaskTypeTLSChanged
|
||||
}
|
||||
return notifyHTTPDNSClusterTask(tx, req.ClusterId, taskType)
|
||||
@@ -86,6 +118,13 @@ func (this *HTTPDNSClusterService) FindHTTPDNSCluster(ctx context.Context, req *
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cluster != nil {
|
||||
_ = grpc.SetHeader(ctx, metadata.Pairs(
|
||||
"x-httpdns-auto-remote-start", fmt.Sprintf("%t", cluster.AutoRemoteStart),
|
||||
"x-httpdns-access-log-is-on", fmt.Sprintf("%t", cluster.AccessLogIsOn),
|
||||
"x-httpdns-time-zone", cluster.TimeZone,
|
||||
))
|
||||
}
|
||||
return &pb.FindHTTPDNSClusterResponse{Cluster: toPBCluster(cluster)}, nil
|
||||
}
|
||||
|
||||
@@ -107,10 +146,12 @@ func (this *HTTPDNSClusterService) ListHTTPDNSClusters(ctx context.Context, req
|
||||
|
||||
func (this *HTTPDNSClusterService) FindAllHTTPDNSClusters(ctx context.Context, req *pb.FindAllHTTPDNSClustersRequest) (*pb.FindAllHTTPDNSClustersResponse, error) {
|
||||
_, _, validateErr := this.ValidateAdminAndUser(ctx, true)
|
||||
isNode := false
|
||||
if validateErr != nil {
|
||||
if _, nodeErr := this.ValidateHTTPDNSNode(ctx); nodeErr != nil {
|
||||
return nil, validateErr
|
||||
}
|
||||
isNode = true
|
||||
}
|
||||
clusters, err := models.SharedHTTPDNSClusterDAO.FindAllEnabledClusters(this.NullTx())
|
||||
if err != nil {
|
||||
@@ -118,7 +159,12 @@ func (this *HTTPDNSClusterService) FindAllHTTPDNSClusters(ctx context.Context, r
|
||||
}
|
||||
var pbClusters []*pb.HTTPDNSCluster
|
||||
for _, cluster := range clusters {
|
||||
pbClusters = append(pbClusters, toPBCluster(cluster))
|
||||
if isNode {
|
||||
// 节点调用时解析证书引用,嵌入实际 PEM 数据
|
||||
pbClusters = append(pbClusters, toPBClusterWithResolvedCerts(this.NullTx(), cluster))
|
||||
} else {
|
||||
pbClusters = append(pbClusters, toPBCluster(cluster))
|
||||
}
|
||||
}
|
||||
return &pb.FindAllHTTPDNSClustersResponse{Clusters: pbClusters}, nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/installers"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/setup"
|
||||
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
@@ -247,6 +248,12 @@ func (this *HTTPDNSNodeService) DownloadHTTPDNSNodeInstallationFile(ctx context.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查自动升级开关
|
||||
upgradeConfig, _ := setup.LoadUpgradeConfig()
|
||||
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
|
||||
return &pb.DownloadHTTPDNSNodeInstallationFileResponse{}, nil
|
||||
}
|
||||
|
||||
var file = installers.SharedDeployManager.FindHTTPDNSNodeFile(req.Os, req.Arch)
|
||||
if file == nil {
|
||||
return &pb.DownloadHTTPDNSNodeInstallationFileResponse{}, nil
|
||||
@@ -274,6 +281,120 @@ func (this *HTTPDNSNodeService) DownloadHTTPDNSNodeInstallationFile(ctx context.
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CountAllUpgradeHTTPDNSNodesWithClusterId 计算需要升级的HTTPDNS节点数量
|
||||
func (this *HTTPDNSNodeService) CountAllUpgradeHTTPDNSNodesWithClusterId(ctx context.Context, req *pb.CountAllUpgradeHTTPDNSNodesWithClusterIdRequest) (*pb.RPCCountResponse, error) {
|
||||
_, err := this.ValidateAdmin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
deployFiles := installers.SharedDeployManager.LoadHTTPDNSNodeFiles()
|
||||
total := int64(0)
|
||||
for _, deployFile := range deployFiles {
|
||||
count, err := models.SharedHTTPDNSNodeDAO.CountAllLowerVersionNodesWithClusterId(tx, req.ClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
total += count
|
||||
}
|
||||
return this.SuccessCount(total)
|
||||
}
|
||||
|
||||
// FindAllUpgradeHTTPDNSNodesWithClusterId 列出所有需要升级的HTTPDNS节点
|
||||
func (this *HTTPDNSNodeService) FindAllUpgradeHTTPDNSNodesWithClusterId(ctx context.Context, req *pb.FindAllUpgradeHTTPDNSNodesWithClusterIdRequest) (*pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse, error) {
|
||||
_, err := this.ValidateAdmin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
deployFiles := installers.SharedDeployManager.LoadHTTPDNSNodeFiles()
|
||||
var result []*pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse_HTTPDNSNodeUpgrade
|
||||
for _, deployFile := range deployFiles {
|
||||
nodes, err := models.SharedHTTPDNSNodeDAO.FindAllLowerVersionNodesWithClusterId(tx, req.ClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
// 解析状态获取当前版本
|
||||
var oldVersion string
|
||||
if len(node.Status) > 0 {
|
||||
var statusMap map[string]interface{}
|
||||
if json.Unmarshal(node.Status, &statusMap) == nil {
|
||||
if v, ok := statusMap["buildVersion"]; ok {
|
||||
oldVersion, _ = v.(string)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pbNode := toPBNode(node)
|
||||
|
||||
// 认证信息
|
||||
login, loginErr := models.SharedNodeLoginDAO.FindEnabledNodeLoginWithNodeId(tx, nodeconfigs.NodeRoleHTTPDNS, int64(node.Id))
|
||||
if loginErr != nil {
|
||||
return nil, loginErr
|
||||
}
|
||||
if login != nil && pbNode != nil {
|
||||
pbNode.NodeLogin = &pb.NodeLogin{
|
||||
Id: int64(login.Id),
|
||||
Name: login.Name,
|
||||
Type: login.Type,
|
||||
Params: login.Params,
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, &pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse_HTTPDNSNodeUpgrade{
|
||||
Node: pbNode,
|
||||
Os: deployFile.OS,
|
||||
Arch: deployFile.Arch,
|
||||
OldVersion: oldVersion,
|
||||
NewVersion: deployFile.Version,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &pb.FindAllUpgradeHTTPDNSNodesWithClusterIdResponse{Nodes: result}, nil
|
||||
}
|
||||
|
||||
// UpgradeHTTPDNSNode 升级单个HTTPDNS节点
|
||||
func (this *HTTPDNSNodeService) UpgradeHTTPDNSNode(ctx context.Context, req *pb.UpgradeHTTPDNSNodeRequest) (*pb.RPCSuccess, error) {
|
||||
_, err := this.ValidateAdmin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
err = models.SharedHTTPDNSNodeDAO.UpdateNodeIsInstalled(tx, req.NodeId, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 重置安装状态
|
||||
installStatus, err := models.SharedHTTPDNSNodeDAO.FindNodeInstallStatus(tx, req.NodeId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if installStatus == nil {
|
||||
installStatus = &models.NodeInstallStatus{}
|
||||
}
|
||||
installStatus.IsOk = false
|
||||
installStatus.IsFinished = false
|
||||
err = models.SharedHTTPDNSNodeDAO.UpdateNodeInstallStatus(tx, req.NodeId, installStatus)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
goman.New(func() {
|
||||
installErr := installers.SharedHTTPDNSNodeQueue().InstallNodeProcess(req.NodeId, true)
|
||||
if installErr != nil {
|
||||
logs.Println("[RPC][HTTPDNS]upgrade node failed:", installErr.Error())
|
||||
}
|
||||
})
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
func shouldTriggerHTTPDNSInstall(installStatusJSON []byte) bool {
|
||||
if len(installStatusJSON) == 0 {
|
||||
return false
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/installers"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/rpc/services"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/setup"
|
||||
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
@@ -484,6 +485,12 @@ func (this *NSNodeService) DownloadNSNodeInstallationFile(ctx context.Context, r
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查自动升级开关
|
||||
upgradeConfig, _ := setup.LoadUpgradeConfig()
|
||||
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
|
||||
return &pb.DownloadNSNodeInstallationFileResponse{}, nil
|
||||
}
|
||||
|
||||
var file = installers.SharedDeployManager.FindNSNodeFile(req.Os, req.Arch)
|
||||
if file == nil {
|
||||
return &pb.DownloadNSNodeInstallationFileResponse{}, nil
|
||||
@@ -738,3 +745,109 @@ func (this *NSNodeService) UpdateNSNodeAPIConfig(ctx context.Context, req *pb.Up
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
// FindAllUpgradeNSNodesWithNSClusterId 列出所有需要升级的NS节点
|
||||
func (this *NSNodeService) FindAllUpgradeNSNodesWithNSClusterId(ctx context.Context, req *pb.FindAllUpgradeNSNodesWithNSClusterIdRequest) (*pb.FindAllUpgradeNSNodesWithNSClusterIdResponse, error) {
|
||||
_, err := this.ValidateAdmin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
deployFiles := installers.SharedDeployManager.LoadNSNodeFiles()
|
||||
var result []*pb.FindAllUpgradeNSNodesWithNSClusterIdResponse_NSNodeUpgrade
|
||||
for _, deployFile := range deployFiles {
|
||||
nodes, err := models.SharedNSNodeDAO.FindAllLowerVersionNodesWithClusterId(tx, req.NsClusterId, deployFile.OS, deployFile.Arch, deployFile.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
// 解析状态获取当前版本
|
||||
var oldVersion string
|
||||
if len(node.Status) > 0 {
|
||||
var statusMap map[string]interface{}
|
||||
if json.Unmarshal(node.Status, &statusMap) == nil {
|
||||
if v, ok := statusMap["buildVersion"]; ok {
|
||||
oldVersion, _ = v.(string)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 安装信息
|
||||
installStatus, installErr := node.DecodeInstallStatus()
|
||||
if installErr != nil {
|
||||
return nil, installErr
|
||||
}
|
||||
pbInstallStatus := &pb.NodeInstallStatus{}
|
||||
if installStatus != nil {
|
||||
pbInstallStatus = &pb.NodeInstallStatus{
|
||||
IsRunning: installStatus.IsRunning,
|
||||
IsFinished: installStatus.IsFinished,
|
||||
IsOk: installStatus.IsOk,
|
||||
Error: installStatus.Error,
|
||||
ErrorCode: installStatus.ErrorCode,
|
||||
UpdatedAt: installStatus.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// 认证信息
|
||||
login, loginErr := models.SharedNodeLoginDAO.FindEnabledNodeLoginWithNodeId(tx, nodeconfigs.NodeRoleDNS, int64(node.Id))
|
||||
if loginErr != nil {
|
||||
return nil, loginErr
|
||||
}
|
||||
var pbLogin *pb.NodeLogin
|
||||
if login != nil {
|
||||
pbLogin = &pb.NodeLogin{
|
||||
Id: int64(login.Id),
|
||||
Name: login.Name,
|
||||
Type: login.Type,
|
||||
Params: login.Params,
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, &pb.FindAllUpgradeNSNodesWithNSClusterIdResponse_NSNodeUpgrade{
|
||||
NsNode: &pb.NSNode{
|
||||
Id: int64(node.Id),
|
||||
Name: node.Name,
|
||||
IsOn: node.IsOn,
|
||||
UniqueId: node.UniqueId,
|
||||
IsInstalled: node.IsInstalled,
|
||||
IsUp: node.IsUp,
|
||||
IsActive: node.IsActive,
|
||||
StatusJSON: node.Status,
|
||||
InstallStatus: pbInstallStatus,
|
||||
NodeLogin: pbLogin,
|
||||
},
|
||||
Os: deployFile.OS,
|
||||
Arch: deployFile.Arch,
|
||||
OldVersion: oldVersion,
|
||||
NewVersion: deployFile.Version,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &pb.FindAllUpgradeNSNodesWithNSClusterIdResponse{Nodes: result}, nil
|
||||
}
|
||||
|
||||
// UpgradeNSNode 升级单个NS节点
|
||||
func (this *NSNodeService) UpgradeNSNode(ctx context.Context, req *pb.UpgradeNSNodeRequest) (*pb.RPCSuccess, error) {
|
||||
_, err := this.ValidateAdmin(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tx = this.NullTx()
|
||||
|
||||
err = models.SharedNSNodeDAO.UpdateNodeIsInstalled(tx, req.NsNodeId, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
goman.New(func() {
|
||||
installErr := installers.SharedNSNodeQueue().InstallNodeProcess(req.NsNodeId, true)
|
||||
if installErr != nil {
|
||||
logs.Println("[RPC]upgrade dns node:" + installErr.Error())
|
||||
}
|
||||
})
|
||||
|
||||
return this.Success()
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/installers"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
|
||||
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/setup"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
@@ -1716,6 +1717,12 @@ func (this *NodeService) DownloadNodeInstallationFile(ctx context.Context, req *
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查自动升级开关
|
||||
upgradeConfig, _ := setup.LoadUpgradeConfig()
|
||||
if upgradeConfig != nil && !upgradeConfig.AutoUpgrade {
|
||||
return &pb.DownloadNodeInstallationFileResponse{}, nil
|
||||
}
|
||||
|
||||
var file = installers.SharedDeployManager.FindNodeFile(req.Os, req.Arch)
|
||||
if file == nil {
|
||||
return &pb.DownloadNodeInstallationFileResponse{}, nil
|
||||
|
||||
@@ -113,6 +113,9 @@ var upgradeFuncs = []*upgradeVersion{
|
||||
{
|
||||
"1.4.8", upgradeV1_4_8,
|
||||
},
|
||||
{
|
||||
"1.4.9", upgradeV1_4_9,
|
||||
},
|
||||
}
|
||||
|
||||
// UpgradeSQLData 升级SQL数据
|
||||
@@ -1274,10 +1277,25 @@ func upgradeV1_4_8(db *dbs.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1.4.9
|
||||
func upgradeV1_4_9(db *dbs.DB) error {
|
||||
_, err := db.Exec("ALTER TABLE `edgeHTTPDNSClusters` ALTER COLUMN `installDir` SET DEFAULT '/root/edge-httpdns'")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec("ALTER TABLE `edgeHTTPDNSNodes` ALTER COLUMN `installDir` SET DEFAULT '/root/edge-httpdns'")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createHTTPDNSTables(db *dbs.DB) error {
|
||||
sqls := []string{
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSClusters` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`name` varchar(255) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`isDefault` tinyint unsigned DEFAULT '0',`serviceDomain` varchar(255) DEFAULT NULL,`defaultTTL` int unsigned DEFAULT '30',`fallbackTimeoutMs` int unsigned DEFAULT '300',`installDir` varchar(255) DEFAULT '/opt/edge-httpdns',`tlsPolicy` json DEFAULT NULL,`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),KEY `name` (`name`),KEY `isDefault` (`isDefault`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS集群配置表(默认TTL、回退超时、服务域名等)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSNodes` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`clusterId` bigint unsigned DEFAULT '0',`name` varchar(255) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`isUp` tinyint unsigned DEFAULT '0',`isInstalled` tinyint unsigned DEFAULT '0',`isActive` tinyint unsigned DEFAULT '0',`uniqueId` varchar(64) DEFAULT NULL,`secret` varchar(64) DEFAULT NULL,`installDir` varchar(255) DEFAULT '/opt/edge-httpdns',`status` json DEFAULT NULL,`installStatus` json DEFAULT NULL,`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),UNIQUE KEY `uniqueId` (`uniqueId`),KEY `clusterId` (`clusterId`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS节点表(节点基础信息与运行状态)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSClusters` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`name` varchar(255) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`isDefault` tinyint unsigned DEFAULT '0',`serviceDomain` varchar(255) DEFAULT NULL,`defaultTTL` int unsigned DEFAULT '30',`fallbackTimeoutMs` int unsigned DEFAULT '300',`installDir` varchar(255) DEFAULT '/root/edge-httpdns',`tlsPolicy` json DEFAULT NULL,`autoRemoteStart` tinyint unsigned DEFAULT '0',`accessLogIsOn` tinyint unsigned DEFAULT '0',`timeZone` varchar(128) NOT NULL DEFAULT '',`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),KEY `name` (`name`),KEY `isDefault` (`isDefault`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS集群配置表(默认TTL、回退超时、服务域名等)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSNodes` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`clusterId` bigint unsigned DEFAULT '0',`name` varchar(255) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`isUp` tinyint unsigned DEFAULT '0',`isInstalled` tinyint unsigned DEFAULT '0',`isActive` tinyint unsigned DEFAULT '0',`uniqueId` varchar(64) DEFAULT NULL,`secret` varchar(64) DEFAULT NULL,`installDir` varchar(255) DEFAULT '/root/edge-httpdns',`status` json DEFAULT NULL,`installStatus` json DEFAULT NULL,`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),UNIQUE KEY `uniqueId` (`uniqueId`),KEY `clusterId` (`clusterId`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS节点表(节点基础信息与运行状态)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSApps` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`name` varchar(255) DEFAULT NULL,`appId` varchar(64) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`clusterIdsJSON` text DEFAULT NULL,`sniMode` varchar(64) DEFAULT 'fixed_hide',`userId` bigint unsigned DEFAULT '0',`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),UNIQUE KEY `appId` (`appId`),KEY `name` (`name`),KEY `userId` (`userId`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS应用表(应用与集群绑定关系)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSAppSecrets` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`appId` bigint unsigned DEFAULT '0',`signEnabled` tinyint unsigned DEFAULT '0',`signSecret` varchar(255) DEFAULT NULL,`signUpdatedAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),UNIQUE KEY `appId` (`appId`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS应用密钥表(请求验签开关与加签Secret)'",
|
||||
"CREATE TABLE IF NOT EXISTS `edgeHTTPDNSDomains` (`id` bigint unsigned NOT NULL AUTO_INCREMENT,`appId` bigint unsigned DEFAULT '0',`domain` varchar(255) DEFAULT NULL,`isOn` tinyint unsigned DEFAULT '1',`createdAt` bigint unsigned DEFAULT '0',`updatedAt` bigint unsigned DEFAULT '0',`state` tinyint unsigned DEFAULT '1',PRIMARY KEY (`id`),UNIQUE KEY `appId_domain` (`appId`,`domain`),KEY `domain` (`domain`),KEY `state` (`state`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='HTTPDNS应用域名表(应用绑定的业务域名)'",
|
||||
|
||||
45
EdgeAPI/internal/setup/upgrade_config.go
Normal file
45
EdgeAPI/internal/setup/upgrade_config.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package setup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
|
||||
)
|
||||
|
||||
var (
|
||||
sharedUpgradeConfig *systemconfigs.UpgradeConfig
|
||||
sharedUpgradeConfigTime time.Time
|
||||
sharedUpgradeConfigMu sync.Mutex
|
||||
)
|
||||
|
||||
const upgradeConfigTTL = 5 * time.Minute
|
||||
|
||||
// LoadUpgradeConfig 读取升级配置(带5分钟内存缓存)
|
||||
func LoadUpgradeConfig() (*systemconfigs.UpgradeConfig, error) {
|
||||
sharedUpgradeConfigMu.Lock()
|
||||
defer sharedUpgradeConfigMu.Unlock()
|
||||
|
||||
if sharedUpgradeConfig != nil && time.Since(sharedUpgradeConfigTime) < upgradeConfigTTL {
|
||||
return sharedUpgradeConfig, nil
|
||||
}
|
||||
|
||||
valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, systemconfigs.SettingCodeUpgradeConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := systemconfigs.NewUpgradeConfig()
|
||||
if len(valueJSON) > 0 {
|
||||
err = json.Unmarshal(valueJSON, config)
|
||||
if err != nil {
|
||||
return config, nil
|
||||
}
|
||||
}
|
||||
|
||||
sharedUpgradeConfig = config
|
||||
sharedUpgradeConfigTime = time.Now()
|
||||
return config, nil
|
||||
}
|
||||
107
EdgeAPI/internal/tasks/httpdns_node_monitor_task.go
Normal file
107
EdgeAPI/internal/tasks/httpdns_node_monitor_task.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeAPI/internal/installers"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
dbs.OnReadyDone(func() {
|
||||
goman.New(func() {
|
||||
NewHTTPDNSNodeMonitorTask(1 * time.Minute).Start()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
type httpdnsNodeStartingTry struct {
|
||||
count int
|
||||
timestamp int64
|
||||
}
|
||||
|
||||
// HTTPDNSNodeMonitorTask monitors HTTPDNS node activity and optionally tries to start offline nodes.
|
||||
type HTTPDNSNodeMonitorTask struct {
|
||||
BaseTask
|
||||
|
||||
ticker *time.Ticker
|
||||
|
||||
recoverMap map[int64]*httpdnsNodeStartingTry // nodeId => retry info
|
||||
}
|
||||
|
||||
func NewHTTPDNSNodeMonitorTask(duration time.Duration) *HTTPDNSNodeMonitorTask {
|
||||
return &HTTPDNSNodeMonitorTask{
|
||||
ticker: time.NewTicker(duration),
|
||||
recoverMap: map[int64]*httpdnsNodeStartingTry{},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *HTTPDNSNodeMonitorTask) Start() {
|
||||
for range t.ticker.C {
|
||||
if err := t.Loop(); err != nil {
|
||||
t.logErr("HTTPDNS_NODE_MONITOR", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *HTTPDNSNodeMonitorTask) Loop() error {
|
||||
// only run on primary api node
|
||||
if !t.IsPrimaryNode() {
|
||||
return nil
|
||||
}
|
||||
|
||||
clusters, err := models.SharedHTTPDNSClusterDAO.FindAllEnabledClusters(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, cluster := range clusters {
|
||||
if cluster == nil || !cluster.IsOn || !cluster.AutoRemoteStart {
|
||||
continue
|
||||
}
|
||||
clusterID := int64(cluster.Id)
|
||||
inactiveNodes, err := models.SharedHTTPDNSNodeDAO.FindAllInactiveNodesWithClusterId(nil, clusterID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(inactiveNodes) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
nodeQueue := installers.NewHTTPDNSNodeQueue()
|
||||
for _, node := range inactiveNodes {
|
||||
nodeID := int64(node.Id)
|
||||
tryInfo, ok := t.recoverMap[nodeID]
|
||||
if !ok {
|
||||
tryInfo = &httpdnsNodeStartingTry{
|
||||
count: 1,
|
||||
timestamp: time.Now().Unix(),
|
||||
}
|
||||
t.recoverMap[nodeID] = tryInfo
|
||||
} else {
|
||||
if tryInfo.count >= 3 {
|
||||
if tryInfo.timestamp+10*60 > time.Now().Unix() {
|
||||
continue
|
||||
}
|
||||
tryInfo.timestamp = time.Now().Unix()
|
||||
tryInfo.count = 0
|
||||
}
|
||||
tryInfo.count++
|
||||
}
|
||||
|
||||
err = nodeQueue.StartNode(nodeID)
|
||||
if err != nil {
|
||||
if !installers.IsGrantError(err) {
|
||||
_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelError, "NODE", "start node from remote API failed: "+err.Error(), time.Now().Unix(), "", nil)
|
||||
}
|
||||
continue
|
||||
}
|
||||
_ = models.SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleHTTPDNS, nodeID, 0, 0, models.LevelSuccess, "NODE", "start node from remote API successfully", time.Now().Unix(), "", nil)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user