From 12adfa50cc418538ae28cdd43113dbba90efb452 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Mon, 23 Dec 2024 15:16:11 +0800 Subject: [PATCH] feat: config --- go.mod | 5 - internal/api/config_manager.go | 146 ++++++++++++++- internal/api/init.go | 1 + internal/api/router.go | 2 +- pkg/apistruct/config_manager.go | 2 +- pkg/common/cmd/api.go | 6 + pkg/common/cmd/constant.go | 34 ---- pkg/common/cmd/msg_utils.go | 5 +- pkg/common/cmd/root.go | 89 ++++++--- pkg/common/config/config.go | 192 +++++++++++++++----- pkg/common/config/env.go | 30 +++ pkg/common/discovery/etcd/config_manager.go | 91 ++++++++++ pkg/common/discovery/etcd/doc.go | 2 +- tools/check-component/main.go | 13 +- 14 files changed, 498 insertions(+), 120 deletions(-) delete mode 100644 pkg/common/cmd/constant.go create mode 100644 pkg/common/config/env.go create mode 100644 pkg/common/discovery/etcd/config_manager.go diff --git a/go.mod b/go.mod index 5aa5708c6..4eaa18ccc 100644 --- a/go.mod +++ b/go.mod @@ -221,8 +221,3 @@ require ( golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -replace ( - github.com/openimsdk/protocol v0.0.72-alpha.66 => C:\App\Project\protocol - github.com/openimsdk/tools v0.0.50-alpha.57 => C:\App\Project\tools -) \ No newline at end of file diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index 8d2ec0e7e..72f33472e 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -2,26 +2,34 @@ package api import ( "encoding/json" + "reflect" "github.com/gin-gonic/gin" "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/runtimeenv" + clientv3 "go.etcd.io/etcd/client/v3" ) type ConfigManager struct { imAdminUserID []string config *config.AllConfig + client *clientv3.Client + configPath string + runtimeEnv string } -func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig) *ConfigManager { +func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, configPath string, runtimeEnv string) *ConfigManager { return &ConfigManager{ imAdminUserID: IMAdminUserID, config: cfg, + configPath: configPath, + runtimeEnv: runtimeEnv, } } @@ -66,10 +74,140 @@ func (cm *ConfigManager) SetConfig(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - b, err := json.Marshal(cm.config) + var err error + switch req.ConfigName { + case cm.config.Discovery.GetConfigFileName(): + err = compareAndSave[config.Discovery](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Kafka.GetConfigFileName(): + err = compareAndSave[config.Kafka](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.LocalCache.GetConfigFileName(): + err = compareAndSave[config.LocalCache](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Log.GetConfigFileName(): + err = compareAndSave[config.Log](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Minio.GetConfigFileName(): + err = compareAndSave[config.Minio](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Mongo.GetConfigFileName(): + err = compareAndSave[config.Mongo](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Notification.GetConfigFileName(): + err = compareAndSave[config.NotificationConfig](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.API.GetConfigFileName(): + err = compareAndSave[config.API](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.CronTask.GetConfigFileName(): + err = compareAndSave[config.CronTask](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.MsgGateway.GetConfigFileName(): + err = compareAndSave[config.MsgGateway](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.MsgTransfer.GetConfigFileName(): + err = compareAndSave[config.MsgTransfer](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Push.GetConfigFileName(): + err = compareAndSave[config.Push](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Auth.GetConfigFileName(): + err = compareAndSave[config.Auth](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Conversation.GetConfigFileName(): + err = compareAndSave[config.Conversation](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Friend.GetConfigFileName(): + err = compareAndSave[config.Friend](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Group.GetConfigFileName(): + err = compareAndSave[config.Group](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Msg.GetConfigFileName(): + err = compareAndSave[config.Msg](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Third.GetConfigFileName(): + err = compareAndSave[config.Third](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.User.GetConfigFileName(): + err = compareAndSave[config.User](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Redis.GetConfigFileName(): + err = compareAndSave[config.Redis](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Share.GetConfigFileName(): + err = compareAndSave[config.Share](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Webhooks.GetConfigFileName(): + err = compareAndSave[config.Webhooks](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + default: + apiresp.GinError(c, errs.ErrArgs.Wrap()) + return + } if err != nil { - apiresp.GinError(c, err) // args option error + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - apiresp.GinSuccess(c, string(b)) + apiresp.GinSuccess(c, nil) +} + +func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, client *clientv3.Client) error { + conf := new(T) + err := json.Unmarshal(req.Data, &conf) + if err != nil { + return errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + eq := reflect.DeepEqual(old, conf) + if eq { + return nil + } + data, err := json.Marshal(conf) + if err != nil { + return errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + _, err = client.Put(c, etcd.BuildKey(req.ConfigName), string(data)) + if err != nil { + return errs.WrapMsg(err, "save to etcd failed") + } + return nil +} + +func (cm *ConfigManager) ResetConfig(c *gin.Context) { + type initConf struct { + old any + new any + isChanged bool + } + configMap := map[string]*initConf{ + cm.config.Discovery.GetConfigFileName(): {old: &cm.config.Discovery, new: new(config.Discovery)}, + cm.config.Kafka.GetConfigFileName(): {old: &cm.config.Kafka, new: new(config.Kafka)}, + cm.config.LocalCache.GetConfigFileName(): {old: &cm.config.LocalCache, new: new(config.LocalCache)}, + cm.config.Log.GetConfigFileName(): {old: &cm.config.Log, new: new(config.Log)}, + cm.config.Minio.GetConfigFileName(): {old: &cm.config.Minio, new: new(config.Minio)}, + cm.config.Mongo.GetConfigFileName(): {old: &cm.config.Mongo, new: new(config.Mongo)}, + cm.config.Notification.GetConfigFileName(): {old: &cm.config.Notification, new: new(config.Notification)}, + cm.config.API.GetConfigFileName(): {old: &cm.config.API, new: new(config.API)}, + cm.config.CronTask.GetConfigFileName(): {old: &cm.config.CronTask, new: new(config.CronTask)}, + cm.config.MsgGateway.GetConfigFileName(): {old: &cm.config.MsgGateway, new: new(config.MsgGateway)}, + cm.config.MsgTransfer.GetConfigFileName(): {old: &cm.config.MsgTransfer, new: new(config.MsgTransfer)}, + cm.config.Push.GetConfigFileName(): {old: &cm.config.Push, new: new(config.Push)}, + cm.config.Auth.GetConfigFileName(): {old: &cm.config.Auth, new: new(config.Auth)}, + cm.config.Conversation.GetConfigFileName(): {old: &cm.config.Conversation, new: new(config.Conversation)}, + cm.config.Friend.GetConfigFileName(): {old: &cm.config.Friend, new: new(config.Friend)}, + cm.config.Group.GetConfigFileName(): {old: &cm.config.Group, new: new(config.Group)}, + cm.config.Msg.GetConfigFileName(): {old: &cm.config.Msg, new: new(config.Msg)}, + cm.config.Third.GetConfigFileName(): {old: &cm.config.Third, new: new(config.Third)}, + cm.config.User.GetConfigFileName(): {old: &cm.config.User, new: new(config.User)}, + cm.config.Redis.GetConfigFileName(): {old: &cm.config.Redis, new: new(config.Redis)}, + cm.config.Share.GetConfigFileName(): {old: &cm.config.Share, new: new(config.Share)}, + cm.config.Webhooks.GetConfigFileName(): {old: &cm.config.Webhooks, new: new(config.Webhooks)}, + } + + changedKeys := make([]string, 0, len(configMap)) + for k, v := range configMap { + err := config.Load(cm.configPath, k, config.EnvPrefixMap[k], + cm.runtimeEnv, v.new) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "load config failed")) + return + } + v.isChanged = reflect.DeepEqual(v.old, v.new) + if v.isChanged { + changedKeys = append(changedKeys, k) + } + } + + for _, k := range changedKeys { + data, err := json.Marshal(configMap[k].new) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "marshal config failed")) + return + } + _, err = cm.client.Put(c, etcd.BuildKey(k), string(data)) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "save to etcd failed")) + return + } + } + apiresp.GinSuccess(c, nil) } diff --git a/internal/api/init.go b/internal/api/init.go index 2b504f7e2..0686b6628 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -47,6 +47,7 @@ type Config struct { *conf.AllConfig RuntimeEnv string + ConfigPath string } func Start(ctx context.Context, index int, config *Config) error { diff --git a/internal/api/router.go b/internal/api/router.go index 787ead7a1..6fd139d1d 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -259,7 +259,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En proDiscoveryGroup.GET("/msg_gateway", pd.MessageGateway) proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) - cm := NewConfigManager(config.Share.IMAdminUserID, config.AllConfig) + cm := NewConfigManager(config.Share.IMAdminUserID, config.AllConfig, config.ConfigPath, config.RuntimeEnv) configGroup := r.Group("/config", cm.CheckAdmin) configGroup.POST("/get_config_list", cm.GetConfigList) configGroup.POST("/get_config", cm.GetConfig) diff --git a/pkg/apistruct/config_manager.go b/pkg/apistruct/config_manager.go index 6e3a21aed..a538cc52c 100644 --- a/pkg/apistruct/config_manager.go +++ b/pkg/apistruct/config_manager.go @@ -12,5 +12,5 @@ type GetConfigListResp struct { type SetConfigReq struct { ConfigName string `json:"config_name"` - Data string `json:"data"` + Data []byte `json:"data"` } diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index 7ec6d75a3..c744cc2d0 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -36,8 +36,12 @@ func NewApiCmd() *ApiCmd { ret := &ApiCmd{apiConfig: &apiConfig} ret.configMap = map[string]any{ config.DiscoveryConfigFilename: &apiConfig.Discovery, + config.KafkaConfigFileName: &apiConfig.Kafka, config.LocalCacheConfigFileName: &apiConfig.LocalCache, config.LogConfigFileName: &apiConfig.Log, + config.MinioConfigFileName: &apiConfig.Minio, + config.MongodbConfigFileName: &apiConfig.Mongo, + config.NotificationFileName: &apiConfig.Notification, config.OpenIMAPICfgFileName: &apiConfig.API, config.OpenIMCronTaskCfgFileName: &apiConfig.CronTask, config.OpenIMMsgGatewayCfgFileName: &apiConfig.MsgGateway, @@ -50,12 +54,14 @@ func NewApiCmd() *ApiCmd { config.OpenIMRPCMsgCfgFileName: &apiConfig.Msg, config.OpenIMRPCThirdCfgFileName: &apiConfig.Third, config.OpenIMRPCUserCfgFileName: &apiConfig.User, + config.RedisConfigFileName: &apiConfig.Redis, config.ShareFileName: &apiConfig.Share, config.WebhooksConfigFileName: &apiConfig.Webhooks, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { + apiConfig.ConfigPath = ret.configPath return ret.runE() } return ret diff --git a/pkg/common/cmd/constant.go b/pkg/common/cmd/constant.go deleted file mode 100644 index 38751d086..000000000 --- a/pkg/common/cmd/constant.go +++ /dev/null @@ -1,34 +0,0 @@ -package cmd - -import ( - "strings" - - "github.com/openimsdk/open-im-server/v3/pkg/common/config" -) - -var ConfigEnvPrefixMap map[string]string - -func init() { - ConfigEnvPrefixMap = make(map[string]string) - fileNames := []string{ - config.FileName, config.NotificationFileName, config.ShareFileName, config.WebhooksConfigFileName, - config.KafkaConfigFileName, config.RedisConfigFileName, - config.MongodbConfigFileName, config.MinioConfigFileName, config.LogConfigFileName, - config.OpenIMAPICfgFileName, config.OpenIMCronTaskCfgFileName, config.OpenIMMsgGatewayCfgFileName, - config.OpenIMMsgTransferCfgFileName, config.OpenIMPushCfgFileName, config.OpenIMRPCAuthCfgFileName, - config.OpenIMRPCConversationCfgFileName, config.OpenIMRPCFriendCfgFileName, config.OpenIMRPCGroupCfgFileName, - config.OpenIMRPCMsgCfgFileName, config.OpenIMRPCThirdCfgFileName, config.OpenIMRPCUserCfgFileName, config.DiscoveryConfigFilename, - } - - for _, fileName := range fileNames { - envKey := strings.TrimSuffix(strings.TrimSuffix(fileName, ".yml"), ".yaml") - envKey = "IMENV_" + envKey - envKey = strings.ToUpper(strings.ReplaceAll(envKey, "-", "_")) - ConfigEnvPrefixMap[fileName] = envKey - } -} - -const ( - FlagConf = "config_folder_path" - FlagTransferIndex = "index" -) diff --git a/pkg/common/cmd/msg_utils.go b/pkg/common/cmd/msg_utils.go index a0a9b0410..f0e590e2c 100644 --- a/pkg/common/cmd/msg_utils.go +++ b/pkg/common/cmd/msg_utils.go @@ -15,6 +15,7 @@ package cmd import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/spf13/cobra" ) @@ -26,11 +27,11 @@ func (m *MsgUtilsCmd) AddUserIDFlag() { m.Command.PersistentFlags().StringP("userID", "u", "", "openIM userID") } func (m *MsgUtilsCmd) AddIndexFlag() { - m.Command.PersistentFlags().IntP(FlagTransferIndex, "i", 0, "process startup sequence number") + m.Command.PersistentFlags().IntP(config.FlagTransferIndex, "i", 0, "process startup sequence number") } func (m *MsgUtilsCmd) AddConfigDirFlag() { - m.Command.PersistentFlags().StringP(FlagConf, "c", "", "path of config directory") + m.Command.PersistentFlags().StringP(config.FlagConf, "c", "", "path of config directory") } diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 57074c23e..20db4126e 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -1,28 +1,20 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cmd import ( + "context" + "encoding/json" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/version" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" ) type RootCmd struct { @@ -33,6 +25,7 @@ type RootCmd struct { log config.Log index int configPath string + etcdClient *clientv3.Client } func (r *RootCmd) ConfigPath() string { @@ -80,19 +73,43 @@ func NewRootCmd(processName string, opts ...func(*CmdOpts)) *RootCmd { SilenceUsage: true, SilenceErrors: false, } - cmd.Flags().StringP(FlagConf, "c", "", "path of config directory") - cmd.Flags().IntP(FlagTransferIndex, "i", 0, "process startup sequence number") + cmd.Flags().StringP(config.FlagConf, "c", "", "path of config directory") + cmd.Flags().IntP(config.FlagTransferIndex, "i", 0, "process startup sequence number") rootCmd.Command = cmd return rootCmd } +func (r *RootCmd) initEtcd() error { + configDirectory, _, err := r.getFlag(&r.Command) + if err != nil { + return err + } + disConfig := config.Discovery{} + env := runtimeenv.PrintRuntimeEnvironment() + err = config.Load(configDirectory, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], + env, &disConfig) + if err != nil { + return err + } + if disConfig.Enable == config.ETCD { + discov, _ := kdisc.NewDiscoveryRegister(&disConfig, env) + r.etcdClient = discov.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + } + return nil +} + func (r *RootCmd) persistentPreRun(cmd *cobra.Command, opts ...func(*CmdOpts)) error { + if err := r.initEtcd(); err != nil { + return err + } cmdOpts := r.applyOptions(opts...) if err := r.initializeConfiguration(cmd, cmdOpts); err != nil { return err } - + if err := r.updateConfigFromEtcd(cmdOpts); err != nil { + return err + } if err := r.initializeLogger(cmdOpts); err != nil { return errs.WrapMsg(err, "failed to initialize logger") } @@ -111,13 +128,43 @@ func (r *RootCmd) initializeConfiguration(cmd *cobra.Command, opts *CmdOpts) err // Load common configuration file //opts.configMap[ShareFileName] = StructEnvPrefix{EnvPrefix: shareEnvPrefix, ConfigStruct: &r.share} for configFileName, configStruct := range opts.configMap { - err := config.Load(configDirectory, configFileName, ConfigEnvPrefixMap[configFileName], runtimeEnv, configStruct) + err := config.Load(configDirectory, configFileName, config.EnvPrefixMap[configFileName], runtimeEnv, configStruct) if err != nil { return err } } // Load common log configuration file - return config.Load(configDirectory, config.LogConfigFileName, ConfigEnvPrefixMap[config.LogConfigFileName], runtimeEnv, &r.log) + return config.Load(configDirectory, config.LogConfigFileName, config.EnvPrefixMap[config.LogConfigFileName], runtimeEnv, &r.log) +} + +func (r *RootCmd) updateConfigFromEtcd(opts *CmdOpts) error { + if r.etcdClient == nil { + return nil + } + + update := func(configFileName string, configStruct any) error { + key := disetcd.BuildKey(configFileName) + etcdRes, err := r.etcdClient.Get(context.TODO(), key) + if err != nil || etcdRes.Count == 0 { + return nil + } + err = json.Unmarshal(etcdRes.Kvs[0].Value, configStruct) + if err != nil { + return errs.WrapMsg(err, "failed to unmarshal config from etcd") + } + return nil + } + for configFileName, configStruct := range opts.configMap { + if err := update(configFileName, configStruct); err != nil { + return err + } + } + if err := update(config.LogConfigFileName, &r.log); err != nil { + return err + } + // Load common log configuration file + return nil + } func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { @@ -158,12 +205,12 @@ func defaultCmdOpts() *CmdOpts { } func (r *RootCmd) getFlag(cmd *cobra.Command) (string, int, error) { - configDirectory, err := cmd.Flags().GetString(FlagConf) + configDirectory, err := cmd.Flags().GetString(config.FlagConf) if err != nil { return "", 0, errs.Wrap(err) } r.configPath = configDirectory - index, err := cmd.Flags().GetInt(FlagTransferIndex) + index, err := cmd.Flags().GetInt(config.FlagTransferIndex) if err != nil { return "", 0, errs.Wrap(err) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index e10c4df2e..1b9121b7a 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -682,8 +682,11 @@ func InitNotification(notification *Notification) { type AllConfig struct { Discovery Discovery + Kafka Kafka LocalCache LocalCache Log Log + Minio Minio + Mongo Mongo Notification Notification API API CronTask CronTask @@ -697,47 +700,56 @@ type AllConfig struct { Msg Msg Third Third User User + Redis Redis Share Share Webhooks Webhooks } func (a *AllConfig) Name2Config(name string) any { switch name { - case DiscoveryConfigFilename: + case a.Discovery.GetConfigFileName(): return a.Discovery - case LocalCacheConfigFileName: + case a.Kafka.GetConfigFileName(): + return a.Kafka + case a.LocalCache.GetConfigFileName(): return a.LocalCache - case LogConfigFileName: + case a.Log.GetConfigFileName(): return a.Log - case NotificationFileName: + case a.Minio.GetConfigFileName(): + return a.Minio + case a.Mongo.GetConfigFileName(): + return a.Mongo + case a.Notification.GetConfigFileName(): return a.Notification - case OpenIMAPICfgFileName: + case a.API.GetConfigFileName(): return a.API - case OpenIMCronTaskCfgFileName: + case a.CronTask.GetConfigFileName(): return a.CronTask - case OpenIMMsgGatewayCfgFileName: + case a.MsgGateway.GetConfigFileName(): return a.MsgGateway - case OpenIMMsgTransferCfgFileName: + case a.MsgTransfer.GetConfigFileName(): return a.MsgTransfer - case OpenIMPushCfgFileName: + case a.Push.GetConfigFileName(): return a.Push - case OpenIMRPCAuthCfgFileName: + case a.Auth.GetConfigFileName(): return a.Auth - case OpenIMRPCConversationCfgFileName: + case a.Conversation.GetConfigFileName(): return a.Conversation - case OpenIMRPCFriendCfgFileName: + case a.Friend.GetConfigFileName(): return a.Friend - case OpenIMRPCGroupCfgFileName: + case a.Group.GetConfigFileName(): return a.Group - case OpenIMRPCMsgCfgFileName: + case a.Msg.GetConfigFileName(): return a.Msg - case OpenIMRPCThirdCfgFileName: + case a.Third.GetConfigFileName(): return a.Third - case OpenIMRPCUserCfgFileName: + case a.User.GetConfigFileName(): return a.User - case ShareFileName: + case a.Redis.GetConfigFileName(): + return a.Redis + case a.Share.GetConfigFileName(): return a.Share - case WebhooksConfigFileName: + case a.Webhooks.GetConfigFileName(): return a.Webhooks default: return nil @@ -746,38 +758,40 @@ func (a *AllConfig) Name2Config(name string) any { func (a *AllConfig) GetConfigNames() []string { return []string{ - DiscoveryConfigFilename, - LocalCacheConfigFileName, - LogConfigFileName, - NotificationFileName, - OpenIMAPICfgFileName, - OpenIMCronTaskCfgFileName, - OpenIMMsgGatewayCfgFileName, - OpenIMMsgTransferCfgFileName, - OpenIMPushCfgFileName, - OpenIMRPCAuthCfgFileName, - OpenIMRPCConversationCfgFileName, - OpenIMRPCFriendCfgFileName, - OpenIMRPCGroupCfgFileName, - OpenIMRPCMsgCfgFileName, - OpenIMRPCThirdCfgFileName, - OpenIMRPCUserCfgFileName, - ShareFileName, - WebhooksConfigFileName, + a.Discovery.GetConfigFileName(), + a.Kafka.GetConfigFileName(), + a.LocalCache.GetConfigFileName(), + a.Log.GetConfigFileName(), + a.Minio.GetConfigFileName(), + a.Mongo.GetConfigFileName(), + a.Notification.GetConfigFileName(), + a.API.GetConfigFileName(), + a.CronTask.GetConfigFileName(), + a.MsgGateway.GetConfigFileName(), + a.MsgTransfer.GetConfigFileName(), + a.Push.GetConfigFileName(), + a.Auth.GetConfigFileName(), + a.Conversation.GetConfigFileName(), + a.Friend.GetConfigFileName(), + a.Group.GetConfigFileName(), + a.Msg.GetConfigFileName(), + a.Third.GetConfigFileName(), + a.User.GetConfigFileName(), + a.Redis.GetConfigFileName(), + a.Share.GetConfigFileName(), + a.Webhooks.GetConfigFileName(), } } var ( FileName = "config.yaml" - NotificationFileName = "notification.yml" - ShareFileName = "share.yml" - WebhooksConfigFileName = "webhooks.yml" - LocalCacheConfigFileName = "local-cache.yml" + DiscoveryConfigFilename = "discovery.yml" KafkaConfigFileName = "kafka.yml" - RedisConfigFileName = "redis.yml" - MongodbConfigFileName = "mongodb.yml" - MinioConfigFileName = "minio.yml" + LocalCacheConfigFileName = "local-cache.yml" LogConfigFileName = "log.yml" + MinioConfigFileName = "minio.yml" + MongodbConfigFileName = "mongodb.yml" + NotificationFileName = "notification.yml" OpenIMAPICfgFileName = "openim-api.yml" OpenIMCronTaskCfgFileName = "openim-crontask.yml" OpenIMMsgGatewayCfgFileName = "openim-msggateway.yml" @@ -790,5 +804,95 @@ var ( OpenIMRPCMsgCfgFileName = "openim-rpc-msg.yml" OpenIMRPCThirdCfgFileName = "openim-rpc-third.yml" OpenIMRPCUserCfgFileName = "openim-rpc-user.yml" - DiscoveryConfigFilename = "discovery.yml" + RedisConfigFileName = "redis.yml" + ShareFileName = "share.yml" + WebhooksConfigFileName = "webhooks.yml" ) + +func (d *Discovery) GetConfigFileName() string { + return DiscoveryConfigFilename +} + +func (k *Kafka) GetConfigFileName() string { + return KafkaConfigFileName +} + +func (lc *LocalCache) GetConfigFileName() string { + return LocalCacheConfigFileName +} + +func (l *Log) GetConfigFileName() string { + return LogConfigFileName +} + +func (m *Minio) GetConfigFileName() string { + return MinioConfigFileName +} + +func (m *Mongo) GetConfigFileName() string { + return MongodbConfigFileName +} + +func (n *Notification) GetConfigFileName() string { + return NotificationFileName +} + +func (a *API) GetConfigFileName() string { + return OpenIMAPICfgFileName +} + +func (ct *CronTask) GetConfigFileName() string { + return OpenIMCronTaskCfgFileName +} + +func (mg *MsgGateway) GetConfigFileName() string { + return OpenIMMsgGatewayCfgFileName +} + +func (mt *MsgTransfer) GetConfigFileName() string { + return OpenIMMsgTransferCfgFileName +} + +func (p *Push) GetConfigFileName() string { + return OpenIMPushCfgFileName +} + +func (a *Auth) GetConfigFileName() string { + return OpenIMRPCAuthCfgFileName +} + +func (c *Conversation) GetConfigFileName() string { + return OpenIMRPCConversationCfgFileName +} + +func (f *Friend) GetConfigFileName() string { + return OpenIMRPCFriendCfgFileName +} + +func (g *Group) GetConfigFileName() string { + return OpenIMRPCGroupCfgFileName +} + +func (m *Msg) GetConfigFileName() string { + return OpenIMRPCMsgCfgFileName +} + +func (t *Third) GetConfigFileName() string { + return OpenIMRPCThirdCfgFileName +} + +func (u *User) GetConfigFileName() string { + return OpenIMRPCUserCfgFileName +} + +func (r *Redis) GetConfigFileName() string { + return RedisConfigFileName +} + +func (s *Share) GetConfigFileName() string { + return ShareFileName +} + +func (w *Webhooks) GetConfigFileName() string { + return WebhooksConfigFileName +} diff --git a/pkg/common/config/env.go b/pkg/common/config/env.go new file mode 100644 index 000000000..99ccb3ca0 --- /dev/null +++ b/pkg/common/config/env.go @@ -0,0 +1,30 @@ +package config + +import "strings" + +var EnvPrefixMap map[string]string + +func init() { + EnvPrefixMap = make(map[string]string) + fileNames := []string{ + FileName, NotificationFileName, ShareFileName, WebhooksConfigFileName, + KafkaConfigFileName, RedisConfigFileName, + MongodbConfigFileName, MinioConfigFileName, LogConfigFileName, + OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName, + OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName, + OpenIMRPCConversationCfgFileName, OpenIMRPCFriendCfgFileName, OpenIMRPCGroupCfgFileName, + OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, DiscoveryConfigFilename, + } + + for _, fileName := range fileNames { + envKey := strings.TrimSuffix(strings.TrimSuffix(fileName, ".yml"), ".yaml") + envKey = "IMENV_" + envKey + envKey = strings.ToUpper(strings.ReplaceAll(envKey, "-", "_")) + EnvPrefixMap[fileName] = envKey + } +} + +const ( + FlagConf = "config_folder_path" + FlagTransferIndex = "index" +) diff --git a/pkg/common/discovery/etcd/config_manager.go b/pkg/common/discovery/etcd/config_manager.go new file mode 100644 index 000000000..7d62d1919 --- /dev/null +++ b/pkg/common/discovery/etcd/config_manager.go @@ -0,0 +1,91 @@ +package etcd + +import ( + "context" + "os" + "os/exec" + "runtime" + "syscall" + + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + ConfigKeyPrefix = "/config/" +) + +type ConfigManager struct { + client *clientv3.Client + watchConfigNames []string +} + +func BuildKey(s string) string { + return ConfigKeyPrefix + s +} + +func NewConfigManager(client *clientv3.Client, configNames []string) *ConfigManager { + return &ConfigManager{ + client: client, + watchConfigNames: datautil.Batch(func(s string) string { return BuildKey(s) }, configNames)} +} + +func (c *ConfigManager) Watch(ctx context.Context) { + chans := make([]clientv3.WatchChan, 0, len(c.watchConfigNames)) + for _, name := range c.watchConfigNames { + chans = append(chans, c.client.Watch(ctx, name, clientv3.WithPrefix())) + } + + doWatch := func(watchChan clientv3.WatchChan) { + for watchResp := range watchChan { + if watchResp.Err() != nil { + log.ZError(ctx, "watch err", errs.Wrap(watchResp.Err())) + continue + } + for _, event := range watchResp.Events { + if event.IsModify() { + if datautil.Contain(string(event.Kv.Key), c.watchConfigNames...) { + err := restartServer(ctx) + if err != nil { + log.ZError(ctx, "restart server err", err) + } + } + } + } + } + } + for _, ch := range chans { + go doWatch(ch) + } +} + +func restartServer(ctx context.Context) error { + exePath, err := os.Executable() + if err != nil { + return errs.New("get executable path fail").Wrap() + } + + args := os.Args + env := os.Environ() + + cmd := exec.Command(exePath, args[1:]...) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + if runtime.GOOS != "windows" { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + + log.ZInfo(ctx, "restart server") + err = cmd.Start() + if err != nil { + return errs.New("restart server fail").Wrap() + } + + os.Exit(0) + return nil +} diff --git a/pkg/common/discovery/etcd/doc.go b/pkg/common/discovery/etcd/doc.go index 60cf702e8..fedf5ad51 100644 --- a/pkg/common/discovery/etcd/doc.go +++ b/pkg/common/discovery/etcd/doc.go @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" +package etcd // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 6c764d8ed..9df0da7de 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -24,7 +24,6 @@ import ( "path/filepath" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" @@ -87,35 +86,35 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, ) runtimeEnv := runtimeenv.PrintRuntimeEnvironment() - err := config.Load(configDir, config.MongodbConfigFileName, cmd.ConfigEnvPrefixMap[config.MongodbConfigFileName], runtimeEnv, mongoConfig) + err := config.Load(configDir, config.MongodbConfigFileName, config.EnvPrefixMap[config.MongodbConfigFileName], runtimeEnv, mongoConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.RedisConfigFileName, cmd.ConfigEnvPrefixMap[config.RedisConfigFileName], runtimeEnv, redisConfig) + err = config.Load(configDir, config.RedisConfigFileName, config.EnvPrefixMap[config.RedisConfigFileName], runtimeEnv, redisConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.KafkaConfigFileName, cmd.ConfigEnvPrefixMap[config.KafkaConfigFileName], runtimeEnv, kafkaConfig) + err = config.Load(configDir, config.KafkaConfigFileName, config.EnvPrefixMap[config.KafkaConfigFileName], runtimeEnv, kafkaConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, cmd.ConfigEnvPrefixMap[config.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) + err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, config.EnvPrefixMap[config.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } if thirdConfig.Object.Enable == "minio" { - err = config.Load(configDir, config.MinioConfigFileName, cmd.ConfigEnvPrefixMap[config.MinioConfigFileName], runtimeEnv, minioConfig) + err = config.Load(configDir, config.MinioConfigFileName, config.EnvPrefixMap[config.MinioConfigFileName], runtimeEnv, minioConfig) if err != nil { return nil, nil, nil, nil, nil, err } } else { minioConfig = nil } - err = config.Load(configDir, config.DiscoveryConfigFilename, cmd.ConfigEnvPrefixMap[config.DiscoveryConfigFilename], runtimeEnv, discovery) + err = config.Load(configDir, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], runtimeEnv, discovery) if err != nil { return nil, nil, nil, nil, nil, err }