From a69d174e86817c9f702e98e20d49c1b370208839 Mon Sep 17 00:00:00 2001 From: Morya Date: Wed, 4 Dec 2024 11:18:39 +0800 Subject: [PATCH 01/10] fix https://github.com/openimsdk/open-im-server/issues/2895 (#2896) * fix: minor log typo * fix: #2895 no need to specify listen port in config file, just use system random * drop useless code --- config/openim-msggateway.yml | 4 +- config/openim-push.yml | 2 - config/openim-rpc-auth.yml | 3 +- config/openim-rpc-conversation.yml | 2 - config/openim-rpc-friend.yml | 2 - config/openim-rpc-group.yml | 2 - config/openim-rpc-msg.yml | 2 - config/openim-rpc-third.yml | 2 - config/openim-rpc-user.yml | 2 - internal/msggateway/hub_server.go | 8 +- internal/msggateway/init.go | 9 +- internal/tools/addr/addr.go | 165 +++++++++++++++++++++++++++++ pkg/common/cmd/auth.go | 1 - pkg/common/cmd/conversation.go | 1 - pkg/common/cmd/friend.go | 1 - pkg/common/cmd/group.go | 1 - pkg/common/cmd/msg.go | 1 - pkg/common/cmd/push.go | 1 - pkg/common/cmd/third.go | 1 - pkg/common/cmd/user.go | 1 - pkg/common/config/config.go | 1 - pkg/common/startrpc/start.go | 53 +++++---- 22 files changed, 198 insertions(+), 67 deletions(-) create mode 100644 internal/tools/addr/addr.go diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 6c46b52a8..84e232ccb 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,8 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] + registerIP: prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-push.yml b/config/openim-push.yml index 92f716ba2..2db09b37e 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 496803e43..9ebd78070 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -3,8 +3,7 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10200 ] + prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 3581d7e19..1cb56ac60 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 3022c09f3..11b805299 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 9a634d12f..d16ae7b02 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 82d6e2f53..5917ff5ae 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index d8f2d427f..54123d8ea 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 798105472..4bd6444a7 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -3,8 +3,6 @@ rpc: registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 - # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports - ports: [ 10320 ] prometheus: # Whether to enable prometheus diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8eff32899..04376b27f 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -46,8 +46,7 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, - conf.MsgGateway.RPC.RegisterIP, - conf.MsgGateway.RPC.Ports, index, + index, conf.Share.RpcRegisterName.MessageGateway, &conf.Share, conf, @@ -57,7 +56,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { type Server struct { msggateway.UnimplementedMsgGatewayServer - rpcPort int + LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} @@ -70,9 +69,8 @@ func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { +func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ - rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 50da06097..77bccc88c 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -35,16 +35,13 @@ type Config struct { // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "rpcPorts", conf.MsgGateway.RPC.Ports, + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) if err != nil { return err } - rpcPort, err := datautil.GetElemByIndex(conf.MsgGateway.RPC.Ports, index) - if err != nil { - return err - } + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) if err != nil { return err @@ -57,7 +54,7 @@ func Start(ctx context.Context, index int, conf *Config) error { WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + hubServer := NewServer(longServer, conf, func(srv *Server) error { longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) return nil }) diff --git a/internal/tools/addr/addr.go b/internal/tools/addr/addr.go new file mode 100644 index 000000000..b7cf8a2db --- /dev/null +++ b/internal/tools/addr/addr.go @@ -0,0 +1,165 @@ +// addr provides functions to retrieve local IP addresses from device interfaces. +package addr + +import ( + "net" + + "github.com/pkg/errors" +) + +var ( + // ErrIPNotFound no IP address found, and explicit IP not provided. + ErrIPNotFound = errors.New("no IP address found, and explicit IP not provided") +) + +// IsLocal checks whether an IP belongs to one of the device's interfaces. +func IsLocal(addr string) bool { + // Extract the host + host, _, err := net.SplitHostPort(addr) + if err == nil { + addr = host + } + + if addr == "localhost" { + return true + } + + // Check against all local ips + for _, ip := range IPs() { + if addr == ip { + return true + } + } + + return false +} + +// Extract returns a valid IP address. If the address provided is a valid +// address, it will be returned directly. Otherwise, the available interfaces +// will be iterated over to find an IP address, preferably private. +func Extract(addr string) (string, error) { + // if addr is already specified then it's directly returned + if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") { + return addr, nil + } + + var ( + addrs []net.Addr + loAddrs []net.Addr + ) + + ifaces, err := net.Interfaces() + if err != nil { + return "", errors.Wrap(err, "failed to get interfaces") + } + + for _, iface := range ifaces { + ifaceAddrs, err := iface.Addrs() + if err != nil { + // ignore error, interface can disappear from system + continue + } + + if iface.Flags&net.FlagLoopback != 0 { + loAddrs = append(loAddrs, ifaceAddrs...) + continue + } + + addrs = append(addrs, ifaceAddrs...) + } + + // Add loopback addresses to the end of the list + addrs = append(addrs, loAddrs...) + + // Try to find private IP in list, public IP otherwise + ip, err := findIP(addrs) + if err != nil { + return "", err + } + + return ip.String(), nil +} + +// IPs returns all available interface IP addresses. +func IPs() []string { + ifaces, err := net.Interfaces() + if err != nil { + return nil + } + + var ipAddrs []string + + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil { + continue + } + + ipAddrs = append(ipAddrs, ip.String()) + } + } + + return ipAddrs +} + +// findIP will return the first private IP available in the list. +// If no private IP is available it will return the first public IP, if present. +// If no public IP is available, it will return the first loopback IP, if present. +func findIP(addresses []net.Addr) (net.IP, error) { + var publicIP net.IP + var localIP net.IP + + for _, rawAddr := range addresses { + var ip net.IP + switch addr := rawAddr.(type) { + case *net.IPAddr: + ip = addr.IP + case *net.IPNet: + ip = addr.IP + default: + continue + } + + if ip.IsLoopback() { + if localIP == nil { + localIP = ip + } + continue + } + + if !ip.IsPrivate() { + if publicIP == nil { + publicIP = ip + } + continue + } + + // Return private IP if available + return ip, nil + } + + // Return public or virtual IP + if len(publicIP) > 0 { + return publicIP, nil + } + + // Return local IP + if len(localIP) > 0 { + return localIP, nil + } + + return nil, ErrIPNotFound +} diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index b35a95f39..f8232bc26 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -55,6 +55,5 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, - a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.Ports, a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index bdb4447f4..6a0da11e5 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -57,6 +57,5 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, - a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.Ports, a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index a564facd0..2ff25090e 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -58,6 +58,5 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, - a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.Ports, a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 9b0fbf8de..f1757c7fa 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -59,6 +59,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, - a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index bfd29398e..ae718a11d 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -59,6 +59,5 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, - a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.Ports, a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index ca22a697d..2f9e248f1 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -59,6 +59,5 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, - a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.Ports, a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index a301b738f..fa1d5d42d 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -58,6 +58,5 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, - a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.Ports, a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 9a614afca..8a22bfce9 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -59,6 +59,5 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, - a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.Ports, a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index fa571e9f2..5fb12b399 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -197,7 +197,6 @@ type Push struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` MaxConcurrentWorkers int `mapstructure:"maxConcurrentWorkers"` diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index fb8782d30..5c42586bf 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -24,10 +24,10 @@ import ( "net/http" "os" "os/signal" - "strconv" "syscall" "time" + "github.com/openimsdk/open-im-server/v3/internal/tools/addr" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -37,22 +37,15 @@ import ( "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "strconv" ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, - registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { - - rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) - if err != nil { - return err - } - - log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort, - "prometheusPorts", prometheusConfig.Ports) - rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP string, + index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), "0") listener, err := net.Listen( "tcp", rpcTcpAddr, @@ -60,6 +53,14 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo if err != nil { return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) } + + h, portStr, _ := net.SplitHostPort(listener.Addr().String()) + host, _ := addr.Extract(h) + port, _ := strconv.Atoi(portStr) + + log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, + "prometheusPorts", prometheusConfig.Ports) + defer listener.Close() client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { @@ -68,17 +69,13 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - registerIP, err = network.GetRpcRegisterIP(registerIP) - if err != nil { - return err - } - //var reg *prometheus.Registry - //var metric *grpcprometheus.ServerMetrics + // var reg *prometheus.Registry + // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) options = append( options, mw.GrpcServer(), @@ -98,8 +95,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo err = client.Register( rpcRegisterName, - registerIP, - rpcPort, + host, + port, grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { @@ -123,13 +120,13 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } - //metric.InitializeMetrics(srv) + // metric.InitializeMetrics(srv) // Create a HTTP server for prometheus. - //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) // netDone <- struct{}{} - //} + // } }() } From 17d5df88d1fa7333254e235a9efc486c106c02ae Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:41:54 +0800 Subject: [PATCH 02/10] feat: seq user and conversation seq synchronization (#2924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: group member update face_url * feat: seq user hook set conversation * feat: seq user hook set conversation * feat: seq user hook set conversation * feat: seq user hook set conversation --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/conversation/conversation.go | 19 ++++++++++++++++++- internal/rpc/group/group.go | 1 - internal/rpc/msg/seq.go | 18 ++++++++++++++++++ internal/rpc/msg/server.go | 1 - internal/rpc/third/log.go | 2 +- pkg/common/storage/controller/msg.go | 11 +++++++++++ 8 files changed, 51 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 8eae1edb2..840016646 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.59 + github.com/openimsdk/protocol v0.0.72-alpha.61 github.com/openimsdk/tools v0.0.50-alpha.38 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index e572f0b47..1ca89efd4 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.59 h1:+ycb2+68mLKPIo7VrxF0id/GXP6OqZ2/nBM1YZQr7qY= -github.com/openimsdk/protocol v0.0.72-alpha.59/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= +github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 0b6b656a4..6a8a6d800 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,6 +16,8 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + pbmsg "github.com/openimsdk/protocol/msg" "sort" "time" @@ -433,23 +435,38 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r if err != nil { return nil, err } + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) + if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: req.UserIDs, MaxSeq: 0}); err != nil { + return nil, err + } return &pbconversation.CreateGroupChatConversationsResp{}, nil } func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbconversation.SetConversationMaxSeqReq) (*pbconversation.SetConversationMaxSeqResp, error) { + if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MaxSeq: req.MaxSeq}); err != nil { + return nil, err + } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, map[string]any{"max_seq": req.MaxSeq}); err != nil { return nil, err } - + for _, userID := range req.OwnerUserID { + c.conversationNotificationSender.ConversationChangeNotification(ctx, userID, []string{req.ConversationID}) + } return &pbconversation.SetConversationMaxSeqResp{}, nil } func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { + if _, err := c.msgRpcClient.Client.SetUserConversationMinSeq(ctx, &pbmsg.SetUserConversationMinSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MinSeq: req.MinSeq}); err != nil { + return nil, err + } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, map[string]any{"min_seq": req.MinSeq}); err != nil { return nil, err } + for _, userID := range req.OwnerUserID { + c.conversationNotificationSender.ConversationChangeNotification(ctx, userID, []string{req.ConversationID}) + } return &pbconversation.SetConversationMinSeqResp{}, nil } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ba0c8a42d..9e610df0f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -964,7 +964,6 @@ func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, gro if err != nil { return err } - return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) } diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 7f5fa1adb..bd68138fb 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -84,3 +84,21 @@ func (m *msgServer) GetActiveConversation(ctx context.Context, req *pbmsg.GetAct } return &pbmsg.GetActiveConversationResp{Conversations: conversations}, nil } + +func (m *msgServer) SetUserConversationMaxSeq(ctx context.Context, req *pbmsg.SetUserConversationMaxSeqReq) (*pbmsg.SetUserConversationMaxSeqResp, error) { + for _, userID := range req.OwnerUserID { + if err := m.MsgDatabase.SetUserConversationsMaxSeq(ctx, req.ConversationID, userID, req.MaxSeq); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationMaxSeqResp{}, nil +} + +func (m *msgServer) SetUserConversationMinSeq(ctx context.Context, req *pbmsg.SetUserConversationMinSeqReq) (*pbmsg.SetUserConversationMinSeqResp, error) { + for _, userID := range req.OwnerUserID { + if err := m.MsgDatabase.SetUserConversationsMinSeq(ctx, req.ConversationID, userID, req.MinSeq); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationMinSeqResp{}, nil +} diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 6d5922ce3..758a7cf19 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,7 +16,6 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 657ea1689..cb4678b34 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -55,7 +55,7 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) CreateTime: time.Now(), Url: fileURL.URL, FileName: fileURL.Filename, - SystemType: req.SystemType, + SystemType: req.AppFramework, Version: req.Version, Ex: req.Ex, } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 85b797c40..464ad7604 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -97,6 +97,9 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) GetDocIDs(ctx context.Context) ([]string, error) + + SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -702,6 +705,14 @@ func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, us return db.seqUser.SetUserMinSeqs(ctx, userID, seqs) } +func (db *commonMsgDatabase) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return db.seqUser.SetUserMaxSeq(ctx, conversationID, userID, seq) +} + +func (db *commonMsgDatabase) SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return db.seqUser.SetUserMinSeq(ctx, conversationID, userID, seq) +} + func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } From fceaaa199b461c77c0526ece9ab3087bf04a4ffd Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:25:42 +0800 Subject: [PATCH 03/10] feat: compatible autoSetPorts (#2929) --- config/openim-msggateway.yml | 5 + config/openim-push.yml | 6 ++ config/openim-rpc-auth.yml | 5 + config/openim-rpc-conversation.yml | 5 + config/openim-rpc-friend.yml | 5 + config/openim-rpc-group.yml | 5 + config/openim-rpc-msg.yml | 5 + config/openim-rpc-third.yml | 5 + config/openim-rpc-user.yml | 5 + internal/msggateway/hub_server.go | 3 +- internal/msggateway/init.go | 3 +- internal/tools/addr/addr.go | 165 ----------------------------- pkg/common/cmd/auth.go | 1 + pkg/common/cmd/conversation.go | 1 + pkg/common/cmd/friend.go | 1 + pkg/common/cmd/group.go | 1 + pkg/common/cmd/msg.go | 1 + pkg/common/cmd/push.go | 1 + pkg/common/cmd/third.go | 1 + pkg/common/cmd/user.go | 1 + pkg/common/config/config.go | 60 ++++++----- pkg/common/startrpc/start.go | 108 ++++++++++--------- 22 files changed, 150 insertions(+), 243 deletions(-) delete mode 100644 internal/tools/addr/addr.go diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 84e232ccb..74eab35d5 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,6 +1,11 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-push.yml b/config/openim-push.yml index 2db09b37e..53c88ed41 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -3,6 +3,12 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] + prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 9ebd78070..230aa8720 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10200 ] prometheus: diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 1cb56ac60..ed61f7010 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 11b805299..e10ef496a 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index d16ae7b02..9f5ceabab 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 5917ff5ae..72e055a5d 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 54123d8ea..9578e82c6 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 4bd6444a7..6f7b9648b 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -3,6 +3,11 @@ rpc: registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10320 ] prometheus: # Whether to enable prometheus diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 04376b27f..b6760ed0f 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -46,7 +46,8 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, - index, + conf.MsgGateway.RPC.RegisterIP, + conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, conf.Share.RpcRegisterName.MessageGateway, &conf.Share, conf, diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 77bccc88c..1796a5482 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -35,7 +35,8 @@ type Config struct { // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) if err != nil { diff --git a/internal/tools/addr/addr.go b/internal/tools/addr/addr.go deleted file mode 100644 index b7cf8a2db..000000000 --- a/internal/tools/addr/addr.go +++ /dev/null @@ -1,165 +0,0 @@ -// addr provides functions to retrieve local IP addresses from device interfaces. -package addr - -import ( - "net" - - "github.com/pkg/errors" -) - -var ( - // ErrIPNotFound no IP address found, and explicit IP not provided. - ErrIPNotFound = errors.New("no IP address found, and explicit IP not provided") -) - -// IsLocal checks whether an IP belongs to one of the device's interfaces. -func IsLocal(addr string) bool { - // Extract the host - host, _, err := net.SplitHostPort(addr) - if err == nil { - addr = host - } - - if addr == "localhost" { - return true - } - - // Check against all local ips - for _, ip := range IPs() { - if addr == ip { - return true - } - } - - return false -} - -// Extract returns a valid IP address. If the address provided is a valid -// address, it will be returned directly. Otherwise, the available interfaces -// will be iterated over to find an IP address, preferably private. -func Extract(addr string) (string, error) { - // if addr is already specified then it's directly returned - if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") { - return addr, nil - } - - var ( - addrs []net.Addr - loAddrs []net.Addr - ) - - ifaces, err := net.Interfaces() - if err != nil { - return "", errors.Wrap(err, "failed to get interfaces") - } - - for _, iface := range ifaces { - ifaceAddrs, err := iface.Addrs() - if err != nil { - // ignore error, interface can disappear from system - continue - } - - if iface.Flags&net.FlagLoopback != 0 { - loAddrs = append(loAddrs, ifaceAddrs...) - continue - } - - addrs = append(addrs, ifaceAddrs...) - } - - // Add loopback addresses to the end of the list - addrs = append(addrs, loAddrs...) - - // Try to find private IP in list, public IP otherwise - ip, err := findIP(addrs) - if err != nil { - return "", err - } - - return ip.String(), nil -} - -// IPs returns all available interface IP addresses. -func IPs() []string { - ifaces, err := net.Interfaces() - if err != nil { - return nil - } - - var ipAddrs []string - - for _, i := range ifaces { - addrs, err := i.Addrs() - if err != nil { - continue - } - - for _, addr := range addrs { - var ip net.IP - switch v := addr.(type) { - case *net.IPNet: - ip = v.IP - case *net.IPAddr: - ip = v.IP - } - - if ip == nil { - continue - } - - ipAddrs = append(ipAddrs, ip.String()) - } - } - - return ipAddrs -} - -// findIP will return the first private IP available in the list. -// If no private IP is available it will return the first public IP, if present. -// If no public IP is available, it will return the first loopback IP, if present. -func findIP(addresses []net.Addr) (net.IP, error) { - var publicIP net.IP - var localIP net.IP - - for _, rawAddr := range addresses { - var ip net.IP - switch addr := rawAddr.(type) { - case *net.IPAddr: - ip = addr.IP - case *net.IPNet: - ip = addr.IP - default: - continue - } - - if ip.IsLoopback() { - if localIP == nil { - localIP = ip - } - continue - } - - if !ip.IsPrivate() { - if publicIP == nil { - publicIP = ip - } - continue - } - - // Return private IP if available - return ip, nil - } - - // Return public or virtual IP - if len(publicIP) > 0 { - return publicIP, nil - } - - // Return local IP - if len(localIP) > 0 { - return localIP, nil - } - - return nil, ErrIPNotFound -} diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index f8232bc26..34f450fb1 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -55,5 +55,6 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, + a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 6a0da11e5..f8ec570a7 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -57,5 +57,6 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, + a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 2ff25090e..786be0d8e 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -58,5 +58,6 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, + a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f1757c7fa..e405c9135 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -59,5 +59,6 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, + a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index ae718a11d..3f1a2fa31 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -59,5 +59,6 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, + a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 2f9e248f1..e0f6b39f6 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -59,5 +59,6 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, + a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index fa1d5d42d..c5a21f91b 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -58,5 +58,6 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, + a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 8a22bfce9..1d916ea37 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -59,5 +59,6 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, + a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 5fb12b399..ed803359c 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -176,8 +176,9 @@ type Prometheus struct { type MsgGateway struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` ListenIP string `mapstructure:"listenIP"` @@ -195,8 +196,10 @@ type MsgTransfer struct { type Push struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` MaxConcurrentWorkers int `mapstructure:"maxConcurrentWorkers"` @@ -229,9 +232,10 @@ type Push struct { type Auth struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` TokenPolicy struct { @@ -241,27 +245,30 @@ type Auth struct { type Conversation struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } type Friend struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } type Group struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` EnableHistoryForNewMembers bool `mapstructure:"enableHistoryForNewMembers"` @@ -269,9 +276,10 @@ type Group struct { type Msg struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` FriendVerify bool `mapstructure:"friendVerify"` @@ -279,9 +287,10 @@ type Msg struct { type Third struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` Object struct { @@ -328,9 +337,10 @@ type Kodo struct { type User struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 5c42586bf..26fbb0ffa 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -16,6 +16,7 @@ package startrpc import ( "context" + "errors" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" @@ -27,7 +28,6 @@ import ( "syscall" "time" - "github.com/openimsdk/open-im-server/v3/internal/tools/addr" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -41,11 +41,60 @@ import ( ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP string, - index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, + registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + + var ( + rpcTcpAddr string + netDone = make(chan struct{}, 2) + netErr error + ) + + if !autoSetPorts { + rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) + if err != nil { + return err + } + rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) + } else { + rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0") + } + + // var reg *prometheus.Registry + // var metric *grpcprometheus.ServerMetrics + if prometheusConfig.Enable { + // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append( + options, mw.GrpcServer(), + prommetricsUnaryInterceptor(rpcRegisterName), + prommetricsStreamInterceptor(rpcRegisterName), + ) + prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) + if err != nil { + return err + } + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + go func() { + if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) + netDone <- struct{}{} + } + // metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + // } + }() + } else { + options = append(options, mw.GrpcServer()) + } - rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), "0") listener, err := net.Listen( "tcp", rpcTcpAddr, @@ -54,8 +103,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) } - h, portStr, _ := net.SplitHostPort(listener.Addr().String()) - host, _ := addr.Extract(h) + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + registerIP = network.GetListenIP(registerIP) port, _ := strconv.Atoi(portStr) log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, @@ -70,22 +119,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - // var reg *prometheus.Registry - // var metric *grpcprometheus.ServerMetrics - if prometheusConfig.Enable { - // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) - options = append( - options, mw.GrpcServer(), - prommetricsUnaryInterceptor(rpcRegisterName), - prommetricsStreamInterceptor(rpcRegisterName), - ) - } else { - options = append(options, mw.GrpcServer()) - } - srv := grpc.NewServer(options...) err = rpcFn(ctx, config, client, srv) @@ -95,7 +128,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo err = client.Register( rpcRegisterName, - host, + registerIP, port, grpc.WithTransportCredentials(insecure.NewCredentials()), ) @@ -103,33 +136,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo return err } - var ( - netDone = make(chan struct{}, 2) - netErr error - ) - if prometheusConfig.Enable { - go func() { - prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) - if err != nil { - netErr = err - netDone <- struct{}{} - return - } - cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) - netDone <- struct{}{} - } - // metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) - // netDone <- struct{}{} - // } - }() - } - go func() { err := srv.Serve(listener) if err != nil { From bbbf17cbe4330ba0fcac45d28f129f33a4fc4b7e Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Sat, 7 Dec 2024 12:11:12 +0800 Subject: [PATCH 04/10] feat: config (#2934) --- config/discovery.yml | 11 ++++++++++ internal/api/init.go | 2 +- internal/api/router.go | 14 ++++++------ internal/msggateway/hub_server.go | 4 ++-- internal/msggateway/message_handler.go | 2 +- internal/msggateway/ws_server.go | 10 ++++----- internal/msgtransfer/init.go | 6 ++--- internal/push/onlinepusher.go | 2 +- internal/push/push_handler.go | 10 ++++----- internal/rpc/auth/auth.go | 4 ++-- internal/rpc/conversation/conversation.go | 6 ++--- internal/rpc/group/group.go | 6 ++--- internal/rpc/msg/server.go | 8 +++---- internal/rpc/relation/friend.go | 6 ++--- internal/rpc/third/third.go | 2 +- internal/rpc/user/user.go | 6 ++--- internal/tools/cron_task.go | 22 ++++--------------- pkg/common/cmd/auth.go | 2 +- pkg/common/cmd/conversation.go | 2 +- pkg/common/cmd/friend.go | 2 +- pkg/common/cmd/group.go | 2 +- pkg/common/cmd/msg.go | 2 +- pkg/common/cmd/push.go | 2 +- pkg/common/cmd/third.go | 2 +- pkg/common/cmd/user.go | 2 +- pkg/common/config/config.go | 17 +++++++------- .../discoveryregister/discoveryregister.go | 14 ++---------- pkg/common/prommetrics/rpc.go | 12 +++++----- pkg/common/startrpc/start.go | 4 ++-- tools/check-component/main.go | 4 ---- 30 files changed, 85 insertions(+), 103 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 6e68cbff4..fad49e3ed 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,4 +5,15 @@ etcd: username: '' password: '' +rpcService: + user: user-rpc-service + friend: friend-rpc-service + msg: msg-rpc-service + push: push-rpc-service + messageGateway: messageGateway-rpc-service + group: group-rpc-service + auth: auth-rpc-service + conversation: conversation-rpc-service + third: third-rpc-service + diff --git a/internal/api/init.go b/internal/api/init.go index e83dfc2ea..6c25609f6 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -51,7 +51,7 @@ func Start(ctx context.Context, index int, config *Config) error { var client discovery.SvcDiscoveryRegistry // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err = kdisc.NewDiscoveryRegister(&config.Discovery) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/internal/api/router.go b/internal/api/router.go index 8e4d17ef1..52d26bdc5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -57,14 +57,14 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En _ = v.RegisterValidation("required_if", RequiredIf) } // init rpc client here - userRpc := rpcclient.NewUser(disCov, config.Share.RpcRegisterName.User, config.Share.RpcRegisterName.MessageGateway, + userRpc := rpcclient.NewUser(disCov, config.Discovery.RpcService.User, config.Discovery.RpcService.MessageGateway, config.Share.IMAdminUserID) - groupRpc := rpcclient.NewGroup(disCov, config.Share.RpcRegisterName.Group) - friendRpc := rpcclient.NewFriend(disCov, config.Share.RpcRegisterName.Friend) - messageRpc := rpcclient.NewMessage(disCov, config.Share.RpcRegisterName.Msg) - conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) - authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) - thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) + groupRpc := rpcclient.NewGroup(disCov, config.Discovery.RpcService.Group) + friendRpc := rpcclient.NewFriend(disCov, config.Discovery.RpcService.Friend) + messageRpc := rpcclient.NewMessage(disCov, config.Discovery.RpcService.Msg) + conversationRpc := rpcclient.NewConversation(disCov, config.Discovery.RpcService.Conversation) + authRpc := rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) + thirdRpc := rpcclient.NewThird(disCov, config.Discovery.RpcService.Third, config.API.Prometheus.GrafanaURL) switch config.API.Api.CompressionLevel { case NoCompression: case DefaultCompression: diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index b6760ed0f..8e1edbec7 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -37,7 +37,7 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) - s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) if s.ready != nil { return s.ready(s) } @@ -48,7 +48,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, conf.MsgGateway.RPC.RegisterIP, conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, - conf.Share.RpcRegisterName.MessageGateway, + conf.Discovery.RpcService.MessageGateway, &conf.Share, conf, s.InitServer, diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 5407ba90c..f2b0ce9da 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -120,7 +120,7 @@ type GrpcHandler struct { validate *validator.Validate } -func NewGrpcHandler(validate *validator.Validate, client discovery.SvcDiscoveryRegistry, rpcRegisterName *config.RpcRegisterName) *GrpcHandler { +func NewGrpcHandler(validate *validator.Validate, client discovery.SvcDiscoveryRegistry, rpcRegisterName *config.RpcService) *GrpcHandler { msgRpcClient := rpcclient.NewMessageRpcClient(client, rpcRegisterName.Msg) pushRpcClient := rpcclient.NewPushRpcClient(client, rpcRegisterName.Push) return &GrpcHandler{ diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index e6b4f3fa4..335556ae9 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -72,9 +72,9 @@ type kickHandler struct { } func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, config *Config) { - ws.MessageHandler = NewGrpcHandler(ws.validate, disCov, &config.Share.RpcRegisterName) - u := rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - ws.authClient = rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) + ws.MessageHandler = NewGrpcHandler(ws.validate, disCov, &config.Discovery.RpcService) + u := rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + ws.authClient = rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) ws.userClient = &u ws.disCov = disCov } @@ -113,7 +113,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { for _, o := range opts { o(&config) } - //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) v := validator.New() return &WsServer{ @@ -192,7 +192,7 @@ func (ws *WsServer) Run(done chan error) error { var concurrentRequest = 3 func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *Client) error { - conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Share.RpcRegisterName.MessageGateway) + conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Discovery.RpcService.MessageGateway) if err != nil { return err } diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 92053931c..b65532a8e 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -75,7 +75,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery) if err != nil { return err } @@ -101,8 +101,8 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) + conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) if err != nil { return err diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9521a84a0..f1ff7fed5 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -60,7 +60,7 @@ func NewDefaultAllNode(disCov discovery.SvcDiscoveryRegistry, config *Config) *D func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - conns, err := d.disCov.GetConns(ctx, d.config.Share.RpcRegisterName.MessageGateway) + conns, err := d.disCov.GetConns(ctx, d.config.Discovery.RpcService.MessageGateway) if len(conns) == 0 { log.ZWarn(ctx, "get gateway conn 0 ", nil) } else { diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index c1d1ac2f9..ee3dc5b84 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -58,14 +58,14 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin return nil, err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) - consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) + consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) - consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) + consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config @@ -414,7 +414,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, if err != nil { return err } - + return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index f0f4a022d..c9cf8d345 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -59,7 +59,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) pbauth.RegisterAuthServer(server, &authServer{ userRpcClient: &userRpcClient, RegisterCenter: client, @@ -182,7 +182,7 @@ func (s *authServer) ForceLogout(ctx context.Context, req *pbauth.ForceLogoutReq } func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID int32) error { - conns, err := s.RegisterCenter.GetConns(ctx, s.config.Share.RpcRegisterName.MessageGateway) + conns, err := s.RegisterCenter.GetConns(ctx, s.config.Discovery.RpcService.MessageGateway) if err != nil { return err } diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 6a8a6d800..7345c965d 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -78,9 +78,9 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) localcache.InitLocalCache(&config.LocalCacheConfig) pbconversation.RegisterConversationServer(server, &conversationServer{ msgRpcClient: &msgRpcClient, diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9e610df0f..62020f980 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -99,9 +99,9 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) var gs groupServer database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 758a7cf19..b0cd771a4 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -89,10 +89,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) + conversationClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 036c7aff5..617e31348 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -93,8 +93,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // Initialize RPC clients - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) // Initialize notification sender notificationSender := NewFriendNotificationSender( @@ -119,7 +119,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg userRpcClient: &userRpcClient, notificationSender: notificationSender, RegisterCenter: client, - conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), + conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), queue: memamq.NewMemoryQueue(16, 1024*1024), diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 4206a2d6f..75076ab74 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -101,7 +101,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), + userRpcClient: rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID), s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 2dfbb01df..ae02b997f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -96,9 +96,9 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ online: redis.NewUserOnline(rdb), diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 4f87036c7..3864c43dc 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package tools import ( @@ -47,24 +33,24 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) - msgConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) if err != nil { return err } - thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + thirdConn, err := client.GetConn(ctx, config.Discovery.RpcService.Third) if err != nil { return err } - conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) if err != nil { return err } diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 34f450fb1..80a675ace 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -56,5 +56,5 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, - a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) + a.Index(), a.authConfig.Discovery.RpcService.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index f8ec570a7..72fa694ed 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -58,5 +58,5 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, - a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) + a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 786be0d8e..b40449b2b 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -59,5 +59,5 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, - a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) + a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index e405c9135..faed56233 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -60,5 +60,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) + a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index 3f1a2fa31..962d1468a 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -60,5 +60,5 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, - a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) + a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index e0f6b39f6..09883fc34 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -60,5 +60,5 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, - a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) + a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index c5a21f91b..8e9ffb9bd 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -59,5 +59,5 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, - a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) + a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 1d916ea37..40f372770 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -60,5 +60,5 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, - a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) + a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index ed803359c..6082b5f23 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -372,10 +372,9 @@ type AfterConfig struct { } type Share struct { - Secret string `mapstructure:"secret"` - RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"` - IMAdminUserID []string `mapstructure:"imAdminUserID"` - MultiLogin MultiLogin `mapstructure:"multiLogin"` + Secret string `mapstructure:"secret"` + IMAdminUserID []string `mapstructure:"imAdminUserID"` + MultiLogin MultiLogin `mapstructure:"multiLogin"` } type MultiLogin struct { @@ -383,7 +382,7 @@ type MultiLogin struct { MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` } -type RpcRegisterName struct { +type RpcService struct { User string `mapstructure:"user"` Friend string `mapstructure:"friend"` Msg string `mapstructure:"msg"` @@ -395,7 +394,7 @@ type RpcRegisterName struct { Third string `mapstructure:"third"` } -func (r *RpcRegisterName) GetServiceNames() []string { +func (r *RpcService) GetServiceNames() []string { return []string{ r.User, r.Friend, @@ -470,9 +469,9 @@ type ZooKeeper struct { } type Discovery struct { - Enable string `mapstructure:"enable"` - Etcd Etcd `mapstructure:"etcd"` - ZooKeeper ZooKeeper `mapstructure:"zooKeeper"` + Enable string `mapstructure:"enable"` + Etcd Etcd `mapstructure:"etcd"` + RpcService RpcService `mapstructure:"rpcService"` } type Etcd struct { diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 559c937c1..be82aa526 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -19,25 +19,15 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(discovery *config.Discovery) (discovery.SvcDiscoveryRegistry, error) { switch discovery.Enable { - case "zookeeper": - return zookeeper.NewZkClient( - discovery.ZooKeeper.Address, - discovery.ZooKeeper.Schema, - zookeeper.WithFreq(time.Hour), - zookeeper.WithUserNameAndPassword(discovery.ZooKeeper.Username, discovery.ZooKeeper.Password), - zookeeper.WithRoundRobin(), - zookeeper.WithTimeout(10), - ) case "k8s": - return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) + return kubernetes.NewK8sDiscoveryRegister(discovery.RpcService.MessageGateway) case "etcd": return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 7162fa7e8..809d509b2 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -42,25 +42,25 @@ func GetGrpcServerMetrics() *gp.ServerMetrics { return grpcMetrics } -func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Collector { +func GetGrpcCusMetrics(registerName string, discovery *config.Discovery) []prometheus.Collector { switch registerName { - case share.RpcRegisterName.MessageGateway: + case discovery.RpcService.MessageGateway: return []prometheus.Collector{OnlineUserGauge} - case share.RpcRegisterName.Msg: + case discovery.RpcService.Msg: return []prometheus.Collector{ SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter, } - case share.RpcRegisterName.Push: + case discovery.RpcService.Push: return []prometheus.Collector{ MsgOfflinePushFailedCounter, MsgLoneTimePushCounter, } - case share.RpcRegisterName.Auth: + case discovery.RpcService.Auth: return []prometheus.Collector{UserLoginCounter} - case share.RpcRegisterName.User: + case discovery.RpcService.User: return []prometheus.Collector{UserRegisterCounter} default: return nil diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 26fbb0ffa..7671b1736 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -77,7 +77,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo if err != nil { return err } - cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, discovery) go func() { if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) @@ -111,7 +111,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo "prometheusPorts", prometheusConfig.Ports) defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery, share) + client, err := kdisc.NewDiscoveryRegister(discovery) if err != nil { return err } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 94dbd613c..a84b7bd87 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -160,10 +160,6 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * checks["Etcd"] = func(ctx context.Context) error { return CheckEtcd(ctx, &discovery.Etcd) } - } else if discovery.Enable == "zookeeper" { - checks["Zookeeper"] = func(ctx context.Context) error { - return CheckZookeeper(ctx, &discovery.ZooKeeper) - } } for i := 0; i < maxRetry; i++ { From 59fb9886e5c182f4cde5fd1acdc87d1b268739fa Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Sat, 7 Dec 2024 14:21:14 +0800 Subject: [PATCH 05/10] feat: support aws (#2938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: group member update face_url * feat: support aws * feat: support aws * feat: support aws --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- config/openim-rpc-third.yml | 17 +++++++++---- go.mod | 15 ++++++++--- go.sum | 50 ++++++++++++++++++++++++++----------- internal/rpc/third/third.go | 5 +++- pkg/common/config/config.go | 30 ++++++++++++++++------ 5 files changed, 86 insertions(+), 31 deletions(-) diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 9578e82c6..719c65e21 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -34,10 +34,17 @@ object: sessionToken: publicRead: false kodo: - endpoint: http://s3.cn-south-1.qiniucs.com - bucket: kodo-bucket-test - bucketURL: http://kodo-bucket-test-oetobfb.qiniudns.com - accessKeyID: - accessKeySecret: + endpoint: https://s3.cn-south-1.qiniucs.com + bucket: testdemo12313 + bucketURL: http://so2at6d05.hn-bkt.clouddn.com + accessKeyID: + accessKeySecret: sessionToken: publicRead: false + aws: + region: ap-southeast-2 + bucket: testdemo832234 + accessKeyID: + secretAccessKey: + sessionToken: + publicRead: false \ No newline at end of file diff --git a/go.mod b/go.mod index 840016646..e7f340894 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.38 + github.com/openimsdk/tools v0.0.50-alpha.43 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -55,9 +55,11 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect @@ -90,15 +92,19 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/elastic/go-sysinfo v1.0.2 // indirect + github.com/elastic/go-windows v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -119,6 +125,7 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -143,7 +150,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.18.2 // indirect + github.com/qiniu/go-sdk/v7 v7.25.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -186,6 +193,8 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gorm.io/gorm v1.25.8 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + modernc.org/fileutil v1.0.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 1ca89efd4..afbd01f09 100644 --- a/go.sum +++ b/go.sum @@ -14,15 +14,19 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= @@ -86,6 +90,7 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -103,6 +108,10 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= +github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -119,6 +128,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= +github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -143,7 +154,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -155,6 +166,8 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -204,6 +217,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -222,6 +237,7 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -236,12 +252,15 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -268,6 +287,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -321,13 +341,14 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= -github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= +github.com/openimsdk/tools v0.0.50-alpha.43 h1:IKa8++fT46YBAVlfy/DcNPOJvJRlWX38pWIfwFIQ2AE= +github.com/openimsdk/tools v0.0.50-alpha.43/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -342,11 +363,12 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= -github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= +github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -461,9 +483,7 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -491,7 +511,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -501,6 +520,7 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -510,22 +530,20 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -533,7 +551,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -596,6 +613,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -607,6 +625,10 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 75076ab74..52e48855c 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -21,6 +21,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/s3/kodo" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" @@ -31,7 +33,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" - "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" "google.golang.org/grpc" @@ -92,6 +93,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg o, err = oss.NewOSS(*config.RpcConfig.Object.Oss.Build()) case "kodo": o, err = kodo.NewKodo(*config.RpcConfig.Object.Kodo.Build()) + case "aws": + o, err = aws.NewAws(*config.RpcConfig.Object.Aws.Build()) default: err = fmt.Errorf("invalid object enable: %s", enable) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6082b5f23..b13da1197 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,6 +15,7 @@ package config import ( + "github.com/openimsdk/tools/s3/aws" "strings" "time" @@ -298,14 +299,7 @@ type Third struct { Cos Cos `mapstructure:"cos"` Oss Oss `mapstructure:"oss"` Kodo Kodo `mapstructure:"kodo"` - Aws struct { - Endpoint string `mapstructure:"endpoint"` - Region string `mapstructure:"region"` - Bucket string `mapstructure:"bucket"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - PublicRead bool `mapstructure:"publicRead"` - } `mapstructure:"aws"` + Aws Aws `mapstructure:"aws"` } `mapstructure:"object"` } type Cos struct { @@ -335,6 +329,15 @@ type Kodo struct { PublicRead bool `mapstructure:"publicRead"` } +type Aws struct { + Region string `mapstructure:"region"` + Bucket string `mapstructure:"bucket"` + AccessKeyID string `mapstructure:"accessKeyID"` + SecretAccessKey string `mapstructure:"secretAccessKey"` + SessionToken string `mapstructure:"sessionToken"` + PublicRead bool `mapstructure:"publicRead"` +} + type User struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` @@ -541,6 +544,7 @@ func (m *Minio) Build() *minio.Config { SignEndpoint: formatEndpoint(m.ExternalAddress), } } + func (c *Cos) Build() *cos.Config { return &cos.Config{ BucketURL: c.BucketURL, @@ -575,6 +579,16 @@ func (o *Kodo) Build() *kodo.Config { } } +func (o *Aws) Build() *aws.Config { + return &aws.Config{ + Region: o.Region, + Bucket: o.Bucket, + AccessKeyID: o.AccessKeyID, + SecretAccessKey: o.SecretAccessKey, + SessionToken: o.SessionToken, + } +} + func (l *CacheConfig) Failed() time.Duration { return time.Second * time.Duration(l.FailedExpire) } From 24ab94087560595f3b95f8b2da6b84049b0847d4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 7 Dec 2024 17:35:42 +0800 Subject: [PATCH 06/10] build: implement services image build and CI release. (#2920) * build: implement services image build. * remove unused tools * update test. * update images. * update dockerfile and go mod. * update go mod. * Add comments. * update go pkg. * update loadConfig and discovery logic in kubernetes. * update go pkg and discovery field. * update Load method args. --- .../build-and-release-services-images.yml | 91 +++++ Dockerfile | 2 +- build/images/Dockerfile | 38 +- build/images/openim-api/Dockerfile | 56 ++- build/images/openim-cmdutils/Dockerfile | 44 --- build/images/openim-crontask/Dockerfile | 55 ++- build/images/openim-msggateway/Dockerfile | 55 ++- build/images/openim-msgtransfer/Dockerfile | 55 ++- build/images/openim-push/Dockerfile | 55 ++- build/images/openim-rpc-auth/Dockerfile | 55 ++- .../images/openim-rpc-conversation/Dockerfile | 55 ++- build/images/openim-rpc-friend/Dockerfile | 55 ++- build/images/openim-rpc-group/Dockerfile | 55 ++- build/images/openim-rpc-msg/Dockerfile | 55 ++- build/images/openim-rpc-third/Dockerfile | 55 ++- build/images/openim-rpc-user/Dockerfile | 55 ++- .../images/openim-tools/component/Dockerfile | 124 ++++-- config/discovery.yml | 7 +- config/openim-msggateway.yml | 1 + config/openim-push.yml | 1 + config/openim-rpc-auth.yml | 1 + config/openim-rpc-conversation.yml | 1 + config/openim-rpc-friend.yml | 1 + config/openim-rpc-group.yml | 1 + config/openim-rpc-msg.yml | 1 + config/openim-rpc-third.yml | 1 + config/openim-rpc-user.yml | 1 + go.mod | 30 +- go.sum | 75 +++- internal/api/init.go | 16 +- internal/msggateway/init.go | 10 +- internal/msgtransfer/init.go | 10 +- internal/tools/cron_task.go | 9 +- magefile.go | 13 +- pkg/common/cmd/root.go | 11 +- pkg/common/config/config.go | 8 +- pkg/common/config/constant.go | 6 + pkg/common/config/load_config.go | 20 +- pkg/common/config/load_config_test.go | 40 +- .../discoveryregister/discoveryregister.go | 17 +- .../kubernetes/kubernetes.go | 365 +++++++++++------- pkg/common/startrpc/start.go | 17 +- tools/check-component/main.go | 15 +- 43 files changed, 980 insertions(+), 658 deletions(-) create mode 100644 .github/workflows/build-and-release-services-images.yml delete mode 100644 build/images/openim-cmdutils/Dockerfile diff --git a/.github/workflows/build-and-release-services-images.yml b/.github/workflows/build-and-release-services-images.yml new file mode 100644 index 000000000..d53fd2142 --- /dev/null +++ b/.github/workflows/build-and-release-services-images.yml @@ -0,0 +1,91 @@ +name: Build and release services Images + +on: + push: + branches: + - release-* + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Tag version to be used for Docker image" + required: true + default: "v3.8.3" + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Aliyun Container Registry + uses: docker/login-action@v2 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.ALIREGISTRY_USERNAME }} + password: ${{ secrets.ALIREGISTRY_TOKEN }} + + - name: Extract metadata for Docker (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=ref,event=tag + type=schedule + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern=v{{version}} + # type=semver,pattern={{major}}.{{minor}} + type=semver,pattern=release-{{raw}} + type=sha + type=raw,value=${{ github.event.inputs.tag }} + + - name: Build and push Docker images + run: | + ROOT_DIR="build/images" + for dir in "$ROOT_DIR"/*/; do + # Find Dockerfile or *.dockerfile in a case-insensitive manner + dockerfile=$(find "$dir" -maxdepth 1 -type f \( -iname 'dockerfile' -o -iname '*.dockerfile' \) | head -n 1) + + if [ -n "$dockerfile" ] && [ -f "$dockerfile" ]; then + IMAGE_NAME=$(basename "$dir") + echo "Building Docker image for $IMAGE_NAME with tags:" + + # Initialize tag arguments + tag_args=() + + # Read each tag and append --tag arguments + while IFS= read -r tag; do + tag_args+=(--tag "${{ secrets.DOCKER_USERNAME }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "registry.cn-hangzhou.aliyuncs.com/openimsdk/$IMAGE_NAME:$tag") + done <<< "${{ steps.meta.outputs.tags }}" + + # Build and push the Docker image with all tags + docker buildx build --platform linux/amd64,linux/arm64 \ + --file "$dockerfile" \ + "${tag_args[@]}" \ + --push "$dir" + else + echo "No valid Dockerfile found in $dir" + fi + done \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f6a2ee9fe..8a95b6851 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.14-alpha.5 +RUN go get github.com/openimsdk/gomake@v0.0.15-alpha.1 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/build/images/Dockerfile b/build/images/Dockerfile index 51fe94e1c..020d50786 100644 --- a/build/images/Dockerfile +++ b/build/images/Dockerfile @@ -1,24 +1,24 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright © 2023 OpenIM. All rights reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -FROM BASE_IMAGE +# FROM BASE_IMAGE -WORKDIR ${SERVER_WORKDIR} +# WORKDIR ${SERVER_WORKDIR} -# Set HTTP proxy -ARG BINARY_NAME +# # Set HTTP proxy +# ARG BINARY_NAME -COPY BINARY_NAME ./bin/BINARY_NAME +# COPY BINARY_NAME ./bin/BINARY_NAME -ENTRYPOINT ["./bin/BINARY_NAME"] \ No newline at end of file +# ENTRYPOINT ["./bin/BINARY_NAME"] \ No newline at end of file diff --git a/build/images/openim-api/Dockerfile b/build/images/openim-api/Dockerfile index 662223956..eb9971c9b 100644 --- a/build/images/openim-api/Dockerfile +++ b/build/images/openim-api/Dockerfile @@ -1,44 +1,36 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# Set go mod installation source and proxy +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -FROM golang:1.20 AS builder +# Set the Go proxy to improve dependency resolution speed +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on - -WORKDIR /openim/openim-server - -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +# Copy all files from the current directory into the container +COPY . . -COPY go.mod go.sum ./ -RUN go mod download +RUN go mod tidy -COPY . . +RUN go build -o _output/openim-api ./cmd/openim-api -RUN make build BINS=openim-api +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-api /usr/bin/openim-api +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-api ./bin/openim-api +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-api"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-api"] diff --git a/build/images/openim-cmdutils/Dockerfile b/build/images/openim-cmdutils/Dockerfile deleted file mode 100644 index 34bcd41f5..000000000 --- a/build/images/openim-cmdutils/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# OpenIM base image: https://github.com/openim-sigs/openim-base-image - -# Set go mod installation source and proxy - -FROM golang:1.20 AS builder - -ARG GO111MODULE=on - -WORKDIR /openim/openim-server - -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY - -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . - -RUN make build BINS=openim-cmdutils -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-cmdutils /usr/bin/openim-cmdutils - -FROM ghcr.io/openim-sigs/openim-bash-image:latest - -WORKDIR /openim/openim-server - -COPY --from=builder /usr/bin/openim-cmdutils ./bin/openim-cmdutils - -ENTRYPOINT ["./bin/openim-cmdutils"] - -CMD ["--help"] diff --git a/build/images/openim-crontask/Dockerfile b/build/images/openim-crontask/Dockerfile index 90a562926..9ac0d8eca 100644 --- a/build/images/openim-crontask/Dockerfile +++ b/build/images/openim-crontask/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-crontask ./cmd/openim-crontask + -RUN make build BINS=openim-crontask +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-crontask /usr/bin/openim-crontask +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-crontask ./bin/openim-crontask +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-crontask"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-crontask"] \ No newline at end of file diff --git a/build/images/openim-msggateway/Dockerfile b/build/images/openim-msggateway/Dockerfile index d3a8694ed..90b9c1fa6 100644 --- a/build/images/openim-msggateway/Dockerfile +++ b/build/images/openim-msggateway/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-msggateway ./cmd/openim-msggateway + -RUN make build BINS=openim-msggateway +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-msggateway /usr/bin/openim-msggateway +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-msggateway ./bin/openim-msggateway +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-msggateway"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-msggateway"] \ No newline at end of file diff --git a/build/images/openim-msgtransfer/Dockerfile b/build/images/openim-msgtransfer/Dockerfile index f94978648..4c679863b 100644 --- a/build/images/openim-msgtransfer/Dockerfile +++ b/build/images/openim-msgtransfer/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-msgtransfer ./cmd/openim-msgtransfer + -RUN make build BINS=openim-msgtransfer +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-msgtransfer /usr/bin/openim-msgtransfer +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-msgtransfer ./bin/openim-msgtransfer +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-msgtransfer"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-msgtransfer"] \ No newline at end of file diff --git a/build/images/openim-push/Dockerfile b/build/images/openim-push/Dockerfile index faebbe9c0..68b711025 100644 --- a/build/images/openim-push/Dockerfile +++ b/build/images/openim-push/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-push ./cmd/openim-push + -RUN make build BINS=openim-push +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-push /usr/bin/openim-push +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-push ./bin/openim-push +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-push"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-push"] \ No newline at end of file diff --git a/build/images/openim-rpc-auth/Dockerfile b/build/images/openim-rpc-auth/Dockerfile index 1e905d4b2..0f9383805 100644 --- a/build/images/openim-rpc-auth/Dockerfile +++ b/build/images/openim-rpc-auth/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-auth ./cmd/openim-rpc/openim-rpc-auth + -RUN make build BINS=openim-rpc-auth +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-auth /usr/bin/openim-rpc-auth +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-auth ./bin/openim-rpc-auth +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-auth"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-auth"] \ No newline at end of file diff --git a/build/images/openim-rpc-conversation/Dockerfile b/build/images/openim-rpc-conversation/Dockerfile index 5a69aa89f..2c3578c0c 100644 --- a/build/images/openim-rpc-conversation/Dockerfile +++ b/build/images/openim-rpc-conversation/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-conversation ./cmd/openim-rpc/openim-rpc-conversation + -RUN make build BINS=openim-rpc-conversation +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-conversation /usr/bin/openim-rpc-conversation +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-conversation ./bin/openim-rpc-conversation +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-conversation"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-conversation"] \ No newline at end of file diff --git a/build/images/openim-rpc-friend/Dockerfile b/build/images/openim-rpc-friend/Dockerfile index fad21a880..7aa4b6a54 100644 --- a/build/images/openim-rpc-friend/Dockerfile +++ b/build/images/openim-rpc-friend/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-friend ./cmd/openim-rpc/openim-rpc-friend + -RUN make build BINS=openim-rpc-friend +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-friend /usr/bin/openim-rpc-friend +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-friend ./bin/openim-rpc-friend +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-friend"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-friend"] \ No newline at end of file diff --git a/build/images/openim-rpc-group/Dockerfile b/build/images/openim-rpc-group/Dockerfile index d7dede6b9..d6a4e64e0 100644 --- a/build/images/openim-rpc-group/Dockerfile +++ b/build/images/openim-rpc-group/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-group ./cmd/openim-rpc/openim-rpc-group + -RUN make build BINS=openim-rpc-group +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-group /usr/bin/openim-rpc-group +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-group ./bin/openim-rpc-group +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-group"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-group"] \ No newline at end of file diff --git a/build/images/openim-rpc-msg/Dockerfile b/build/images/openim-rpc-msg/Dockerfile index 71ad85f37..9dac2620a 100644 --- a/build/images/openim-rpc-msg/Dockerfile +++ b/build/images/openim-rpc-msg/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-msg ./cmd/openim-rpc/openim-rpc-msg + -RUN make build BINS=openim-rpc-msg +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-msg /usr/bin/openim-rpc-msg +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-msg ./bin/openim-rpc-msg +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-msg"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-msg"] \ No newline at end of file diff --git a/build/images/openim-rpc-third/Dockerfile b/build/images/openim-rpc-third/Dockerfile index 3560ad0d7..e68f9fe98 100644 --- a/build/images/openim-rpc-third/Dockerfile +++ b/build/images/openim-rpc-third/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-third ./cmd/openim-rpc/openim-rpc-third + -RUN make build BINS=openim-rpc-third +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-third /usr/bin/openim-rpc-third +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-third ./bin/openim-rpc-third +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-third"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-third"] \ No newline at end of file diff --git a/build/images/openim-rpc-user/Dockerfile b/build/images/openim-rpc-user/Dockerfile index a9891ece8..42f9ceec6 100644 --- a/build/images/openim-rpc-user/Dockerfile +++ b/build/images/openim-rpc-user/Dockerfile @@ -1,44 +1,37 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on - -WORKDIR /openim/openim-server +# Copy all files from the current directory into the container +COPY . . -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +RUN go mod tidy -COPY go.mod go.sum ./ -RUN go mod download +RUN go build -o _output/openim-rpc-user ./cmd/openim-rpc/openim-rpc-user -COPY . . -RUN make build BINS=openim-rpc-user +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-user /usr/bin/openim-rpc-user +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-user ./bin/openim-rpc-user +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-user"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-user"] \ No newline at end of file diff --git a/build/images/openim-tools/component/Dockerfile b/build/images/openim-tools/component/Dockerfile index 6bdfa6942..ae8de800f 100644 --- a/build/images/openim-tools/component/Dockerfile +++ b/build/images/openim-tools/component/Dockerfile @@ -1,48 +1,108 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright © 2023 OpenIM. All rights reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# # OpenIM base image: https://github.com/openim-sigs/openim-base-image -# Set go mod installation source and proxy +# # Set go mod installation source and proxy -FROM golang:1.20 AS builder +# FROM golang:1.20 AS builder -ARG GO111MODULE=on +# -WORKDIR /openim/openim-server +# WORKDIR /openim/openim-server -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +# +# ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download +# COPY go.mod go.sum ./ +# RUN go mod download -COPY . . +# COPY . . -RUN make clean -RUN make build BINS=component +# RUN make clean +# RUN make build BINS=component +# # FROM ghcr.io/openim-sigs/openim-bash-image:latest # FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest -WORKDIR /openim/openim-server +# WORKDIR /openim/openim-server + +# COPY --from=builder /openim/openim-server/_output/bin/tools /openim/openim-server/_output/bin/tools/ +# COPY --from=builder /openim/openim-server/config /openim/openim-server/config + +# ENV OPENIM_SERVER_CONFIG_NAME=/openim/openim-server/config + +# RUN mv ${OPENIM_SERVER_BINDIR}/platforms/$(get_os)/$(get_arch)/component /usr/bin/component + +# ENTRYPOINT ["bash", "-c", "component -c $OPENIM_SERVER_CONFIG_NAME"] + + +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server + +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR + +# Set the Go proxy to improve dependency resolution speed + +#ENV GOPROXY=https://goproxy.io,direct + +# Copy all files from the current directory into the container +COPY . . + +RUN go mod download + +# Install Mage to use for building the application +RUN go install github.com/magefile/mage@v1.15.0 + +# ENV BINS=openim-rpc-user + +# Optionally build your application if needed +# RUN mage build ${BINS} check-free-memory seq || true +RUN mage build check-free-memory seq || true + +# Using Alpine Linux with Go environment for the final image +FROM golang:1.22-alpine + +# Install necessary packages, such as bash +RUN apk add bash + +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR + + +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config +COPY --from=builder /go/bin/mage /usr/local/bin/mage +COPY --from=builder $SERVER_DIR/magefile_windows.go $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/magefile_unix.go $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/magefile.go $SERVER_DIR/ +# COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -COPY --from=builder /openim/openim-server/_output/bin/tools /openim/openim-server/_output/bin/tools/ -COPY --from=builder /openim/openim-server/config /openim/openim-server/config -ENV OPENIM_SERVER_CONFIG_NAME=/openim/openim-server/config +RUN echo -e "serviceBinaries:\n \n" \ + > $SERVER_DIR/start-config.yml && \ + echo -e "toolBinaries:\n - check-free-memory\n - seq\n" >> $SERVER_DIR/start-config.yml && \ + echo "maxFileDescriptors: 10000" >> $SERVER_DIR/start-config.yml -RUN mv ${OPENIM_SERVER_BINDIR}/platforms/$(get_os)/$(get_arch)/component /usr/bin/component +RUN go get github.com/openimsdk/gomake@v0.0.15-alpha.1 -ENTRYPOINT ["bash", "-c", "component -c $OPENIM_SERVER_CONFIG_NAME"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/config/discovery.yml b/config/discovery.yml index fad49e3ed..a04d7e40f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,6 +5,9 @@ etcd: username: '' password: '' +kubernetes: + namespace: default + rpcService: user: user-rpc-service friend: friend-rpc-service @@ -14,6 +17,4 @@ rpcService: group: group-rpc-service auth: auth-rpc-service conversation: conversation-rpc-service - third: third-rpc-service - - + third: third-rpc-service \ No newline at end of file diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 74eab35d5..696bc3d06 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -2,6 +2,7 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-push.yml b/config/openim-push.yml index 53c88ed41..a3fe0c765 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 230aa8720..93f365386 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index ed61f7010..c17b5709f 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index e10ef496a..5b51981d2 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 9f5ceabab..06fb489d2 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 72e055a5d..5a4beae5c 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 719c65e21..e24f93883 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 6f7b9648b..02b6d45b1 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -4,6 +4,7 @@ rpc: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/go.mod b/go.mod index e7f340894..b278f2a9b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.43 + github.com/openimsdk/tools v0.0.50-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -37,15 +37,17 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.14-alpha.5 + github.com/openimsdk/gomake v0.0.15-alpha.1 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 - github.com/stathat/consistent v1.0.0 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 + k8s.io/api v0.31.2 + k8s.io/apimachinery v0.31.2 + k8s.io/client-go v0.31.2 ) require ( @@ -94,21 +96,29 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/elastic/go-sysinfo v1.0.2 // indirect github.com/elastic/go-windows v1.0.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect @@ -126,6 +136,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -135,6 +146,7 @@ require ( github.com/lithammer/shortuuid v3.0.0+incompatible // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -144,6 +156,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -165,6 +178,7 @@ require ( github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -186,16 +200,24 @@ require ( golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gorm.io/gorm v1.25.8 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect modernc.org/fileutil v1.0.0 // indirect - stathat.com/c/consistent v1.0.0 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) require ( diff --git a/go.sum b/go.sum index afbd01f09..4657f0eca 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -126,6 +128,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= @@ -143,6 +147,13 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -161,6 +172,8 @@ github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGK github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -192,6 +205,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -199,14 +214,19 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -263,6 +283,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -305,6 +327,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -331,18 +355,22 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= -github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.1 h1:a3U0gayzEJJucOoIOt9Jju+HQTG2BtKcuLG4AVjeVF4= +github.com/openimsdk/gomake v0.0.15-alpha.1/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.43 h1:IKa8++fT46YBAVlfy/DcNPOJvJRlWX38pWIfwFIQ2AE= -github.com/openimsdk/tools v0.0.50-alpha.43/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= +github.com/openimsdk/tools v0.0.50-alpha.45 h1:bu3HCkdzLNQAPUR1VhF40XA9OhaNGHGxZqcC4rIYJyY= +github.com/openimsdk/tools v0.0.50-alpha.45/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -378,8 +406,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -401,8 +429,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= -github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= -github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -432,6 +458,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -471,8 +499,8 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -545,6 +573,8 @@ golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -565,6 +595,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -609,12 +641,15 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -627,9 +662,25 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= +k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= +k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= +k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= +k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= -stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/api/init.go b/internal/api/init.go index 6c25609f6..ea7b6133f 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -17,9 +17,6 @@ package api import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/network" "net" "net/http" "os" @@ -28,6 +25,11 @@ import ( "syscall" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/tools/utils/runtimeenv" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -40,6 +42,8 @@ type Config struct { API config.API Share config.Share Discovery config.Discovery + + RuntimeEnv string } func Start(ctx context.Context, index int, config *Config) error { @@ -48,10 +52,12 @@ func Start(ctx context.Context, index int, config *Config) error { return err } + config.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() + var client discovery.SvcDiscoveryRegistry // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.Discovery) + client, err = kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } @@ -81,7 +87,7 @@ func Start(ctx context.Context, index int, config *Config) error { address := net.JoinHostPort(network.GetListenIP(config.API.Api.ListenIP), strconv.Itoa(apiPort)) server := http.Server{Addr: address, Handler: router} - log.CInfo(ctx, "API server is initializing", "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) + log.CInfo(ctx, "API server is initializing", "runtimeEnv", config.RuntimeEnv, "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) go func() { err = server.ListenAndServe() if err != nil && err != http.ErrServerClosed { diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 1796a5482..7603147de 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -16,11 +16,13 @@ package msggateway import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" - "time" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/openimsdk/tools/log" ) @@ -31,11 +33,15 @@ type Config struct { RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery + + RuntimeEnv string } // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index b65532a8e..6131d8c77 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -29,6 +29,7 @@ import ( "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" @@ -51,6 +52,8 @@ type MsgTransfer struct { historyMongoCH *OnlineHistoryMongoConsumerHandler ctx context.Context cancel context.CancelFunc + + runTimeEnv string } type Config struct { @@ -64,7 +67,9 @@ type Config struct { } func Start(ctx context.Context, index int, config *Config) error { - log.CInfo(ctx, "MSG-TRANSFER server is initializing", "prometheusPorts", + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runTimeEnv, "prometheusPorts", config.MsgTransfer.Prometheus.Ports, "index", index) mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) @@ -75,7 +80,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, runTimeEnv) if err != nil { return err } @@ -115,6 +120,7 @@ func Start(ctx context.Context, index int, config *Config) error { msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, + runTimeEnv: runTimeEnv, } return msgTransfer.Start(index, config) } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 3864c43dc..2fe7d0e39 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -14,6 +14,7 @@ import ( "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" + "github.com/openimsdk/tools/utils/runtimeenv" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -26,14 +27,18 @@ type CronTaskConfig struct { CronTask config.CronTask Share config.Share Discovery config.Discovery + + runTimeEnv string } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) + config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", config.runTimeEnv, "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.runTimeEnv) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/magefile.go b/magefile.go index a8a1c4040..d0e77f6fc 100644 --- a/magefile.go +++ b/magefile.go @@ -4,14 +4,23 @@ package main import ( - "github.com/openimsdk/gomake/mageutil" + "flag" "os" + + "github.com/openimsdk/gomake/mageutil" ) var Default = Build func Build() { - mageutil.Build() + flag.Parse() + + bin := flag.Args() + if len(bin) != 0 { + bin = bin[1:] + } + + mageutil.Build(bin) } func Start() { diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 5edea4377..87252c133 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -16,12 +16,12 @@ package cmd import ( "fmt" - "path/filepath" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/cobra" ) @@ -105,18 +105,19 @@ func (r *RootCmd) initializeConfiguration(cmd *cobra.Command, opts *CmdOpts) err if err != nil { return err } + + runtimeEnv := runtimeenv.PrintRuntimeEnvironment() + // Load common configuration file //opts.configMap[ShareFileName] = StructEnvPrefix{EnvPrefix: shareEnvPrefix, ConfigStruct: &r.share} for configFileName, configStruct := range opts.configMap { - err := config.LoadConfig(filepath.Join(configDirectory, configFileName), - ConfigEnvPrefixMap[configFileName], configStruct) + err := config.Load(configDirectory, configFileName, ConfigEnvPrefixMap[configFileName], runtimeEnv, configStruct) if err != nil { return err } } // Load common log configuration file - return config.LoadConfig(filepath.Join(configDirectory, LogConfigFileName), - ConfigEnvPrefixMap[LogConfigFileName], &r.log) + return config.Load(configDirectory, LogConfigFileName, ConfigEnvPrefixMap[LogConfigFileName], runtimeEnv, &r.log) } func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index b13da1197..b8a60c3df 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,10 +15,11 @@ package config import ( - "github.com/openimsdk/tools/s3/aws" "strings" "time" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" @@ -474,9 +475,14 @@ type ZooKeeper struct { type Discovery struct { Enable string `mapstructure:"enable"` Etcd Etcd `mapstructure:"etcd"` + Kubernetes Kubernetes `mapstructure:"kubernetes"` RpcService RpcService `mapstructure:"rpcService"` } +type Kubernetes struct { + Namespace string `mapstructure:"namespace"` +} + type Etcd struct { RootDirectory string `mapstructure:"rootDirectory"` Address []string `mapstructure:"address"` diff --git a/pkg/common/config/constant.go b/pkg/common/config/constant.go index f425a624c..9aeaedca9 100644 --- a/pkg/common/config/constant.go +++ b/pkg/common/config/constant.go @@ -16,6 +16,12 @@ package config const ConfKey = "conf" +const ( + MountConfigFilePath = "CONFIG_PATH" + DeploymentType = "DEPLOYMENT_TYPE" + KUBERNETES = "kubernetes" +) + const ( // DefaultDirPerm is used for creating general directories, allowing the owner to read, write, and execute, // while the group and others can only read and execute. diff --git a/pkg/common/config/load_config.go b/pkg/common/config/load_config.go index 9272896b4..aa87211f9 100644 --- a/pkg/common/config/load_config.go +++ b/pkg/common/config/load_config.go @@ -1,13 +1,29 @@ package config import ( + "os" + "path/filepath" + "strings" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/tools/errs" "github.com/spf13/viper" - "strings" ) -func LoadConfig(path string, envPrefix string, config any) error { +func Load(configDirectory string, configFileName string, envPrefix string, runtimeEnv string, config any) error { + if runtimeEnv == KUBERNETES { + mountPath := os.Getenv(MountConfigFilePath) + if mountPath == "" { + return errs.ErrArgs.WrapMsg(MountConfigFilePath + " env is empty") + } + + return loadConfig(filepath.Join(mountPath, configFileName), envPrefix, config) + } + + return loadConfig(filepath.Join(configDirectory, configFileName), envPrefix, config) +} + +func loadConfig(path string, envPrefix string, config any) error { v := viper.New() v.SetConfigFile(path) v.SetEnvPrefix(envPrefix) diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index a0345fc7a..ea11b0574 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -1,27 +1,51 @@ package config import ( - "github.com/stretchr/testify/assert" + "os" "testing" + + "github.com/stretchr/testify/assert" ) func TestLoadLogConfig(t *testing.T) { var log Log - err := LoadConfig("../../../config/log.yml", "IMENV_LOG", &log) + os.Setenv("IMENV_LOG_REMAINLOGLEVEL", "5") + err := Load("../../../config/", "log.yml", "IMENV_LOG", "source", &log) + assert.Nil(t, err) + t.Log(log.RemainLogLevel) + // assert.Equal(t, "../../../../logs/", log.StorageLocation) +} + +func TestLoadMongoConfig(t *testing.T) { + var mongo Mongo + // os.Setenv("DEPLOYMENT_TYPE", "kubernetes") + os.Setenv("IMENV_MONGODB_PASSWORD", "openIM1231231") + // os.Setenv("IMENV_MONGODB_URI", "openIM123") + // os.Setenv("IMENV_MONGODB_USERNAME", "openIM123") + err := Load("../../../config/", "mongodb.yml", "IMENV_MONGODB", "source", &mongo) + // err := LoadConfig("../../../config/mongodb.yml", "IMENV_MONGODB", &mongo) + assert.Nil(t, err) - assert.Equal(t, "../../../../logs/", log.StorageLocation) + t.Log(mongo.Password) + // assert.Equal(t, "openIM123", mongo.Password) + t.Log(os.Getenv("IMENV_MONGODB_PASSWORD")) + t.Log(mongo) + // //export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0" + // assert.Equal(t, "0.0.0.0", user.RPC.ListenIP) + // //export IMENV_OPENIM_RPC_USER_RPC_PORTS="10110,10111,10112" + // assert.Equal(t, []int{10110, 10111, 10112}, user.RPC.Ports) } func TestLoadMinioConfig(t *testing.T) { var storageConfig Minio - err := LoadConfig("../../../config/minio.yml", "IMENV_MINIO", &storageConfig) + err := Load("../../../config/minio.yml", "IMENV_MINIO", "", "source", &storageConfig) assert.Nil(t, err) assert.Equal(t, "openim", storageConfig.Bucket) } func TestLoadWebhooksConfig(t *testing.T) { var webhooks Webhooks - err := LoadConfig("../../../config/webhooks.yml", "IMENV_WEBHOOKS", &webhooks) + err := Load("../../../config/webhooks.yml", "IMENV_WEBHOOKS", "", "source", &webhooks) assert.Nil(t, err) assert.Equal(t, 5, webhooks.BeforeAddBlack.Timeout) @@ -29,7 +53,7 @@ func TestLoadWebhooksConfig(t *testing.T) { func TestLoadOpenIMRpcUserConfig(t *testing.T) { var user User - err := LoadConfig("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", &user) + err := Load("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", "", "source", &user) assert.Nil(t, err) //export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0" assert.Equal(t, "0.0.0.0", user.RPC.ListenIP) @@ -39,14 +63,14 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) { func TestLoadNotificationConfig(t *testing.T) { var noti Notification - err := LoadConfig("../../../config/notification.yml", "IMENV_NOTIFICATION", ¬i) + err := Load("../../../config/notification.yml", "IMENV_NOTIFICATION", "", "source", ¬i) assert.Nil(t, err) assert.Equal(t, "Your friend's profile has been changed", noti.FriendRemarkSet.OfflinePush.Title) } func TestLoadOpenIMThirdConfig(t *testing.T) { var third Third - err := LoadConfig("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", &third) + err := Load("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", "", "source", &third) assert.Nil(t, err) assert.Equal(t, "enabled", third.Object.Enable) assert.Equal(t, "https://oss-cn-chengdu.aliyuncs.com", third.Object.Oss.Endpoint) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index be82aa526..419293a91 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -15,19 +15,26 @@ package discoveryregister import ( + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" + + "github.com/openimsdk/tools/discovery/kubernetes" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" - "time" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { + if runtimeEnv == "kubernetes" { + discovery.Enable = "kubernetes" + } + switch discovery.Enable { - case "k8s": - return kubernetes.NewK8sDiscoveryRegister(discovery.RpcService.MessageGateway) + case "kubernetes": + return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace) case "etcd": return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discoveryregister/kubernetes/kubernetes.go index f1ce0bbdc..cf6a904be 100644 --- a/pkg/common/discoveryregister/kubernetes/kubernetes.go +++ b/pkg/common/discoveryregister/kubernetes/kubernetes.go @@ -16,184 +16,281 @@ package kubernetes import ( "context" - "errors" "fmt" - "os" - "strconv" - "strings" + "sync" + "time" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/log" - "github.com/stathat/consistent" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" ) -// K8sDR represents the Kubernetes service discovery and registration client. -type K8sDR struct { - options []grpc.DialOption - rpcRegisterAddr string - gatewayHostConsistent *consistent.Consistent - gatewayName string +type KubernetesConnManager struct { + clientset *kubernetes.Clientset + namespace string + dialOptions []grpc.DialOption + + selfTarget string + + mu sync.RWMutex + connMap map[string][]*grpc.ClientConn } -func NewK8sDiscoveryRegister(gatewayName string) (discovery.SvcDiscoveryRegistry, error) { - gatewayConsistent := consistent.New() - gatewayHosts := getMsgGatewayHost(context.Background(), gatewayName) - for _, v := range gatewayHosts { - gatewayConsistent.Add(v) +// NewKubernetesConnManager creates a new connection manager that uses Kubernetes services for service discovery. +func NewKubernetesConnManager(namespace string, options ...grpc.DialOption) (*KubernetesConnManager, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster config: %v", err) } - return &K8sDR{gatewayHostConsistent: gatewayConsistent}, nil -} -func (cli *K8sDR) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - if serviceName != cli.gatewayName { - cli.rpcRegisterAddr = serviceName - } else { - cli.rpcRegisterAddr = getSelfHost(context.Background(), cli.gatewayName) + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create clientset: %v", err) } - return nil -} + k := &KubernetesConnManager{ + clientset: clientset, + namespace: namespace, + dialOptions: options, + connMap: make(map[string][]*grpc.ClientConn), + } -func (cli *K8sDR) UnRegister() error { + go k.watchEndpoints() - return nil + return k, nil } -func (cli *K8sDR) CreateRpcRootNodes(serviceNames []string) error { +func (k *KubernetesConnManager) initializeConns(serviceName string) error { + port, err := k.getServicePort(serviceName) + if err != nil { + return err + } - return nil -} + endpoints, err := k.clientset.CoreV1().Endpoints(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get endpoints for service %s: %v", serviceName, err) + } + + var conns []*grpc.ClientConn + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + target := fmt.Sprintf("%s:%d", address.IP, port) + conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("failed to dial endpoint %s: %v", target, err) + } + conns = append(conns, conn) + } + } + + k.mu.Lock() + defer k.mu.Unlock() + k.connMap[serviceName] = conns -func (cli *K8sDR) RegisterConf2Registry(key string, conf []byte) error { + // go k.watchEndpoints(serviceName) return nil } -func (cli *K8sDR) GetConfFromRegistry(key string) ([]byte, error) { +// GetConns returns gRPC client connections for a given Kubernetes service name. +func (k *KubernetesConnManager) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { + k.mu.RLock() + conns, exists := k.connMap[serviceName] + defer k.mu.RUnlock() + + if exists { + return conns, nil + } + + k.mu.Lock() + defer k.mu.Unlock() + + // Check if another goroutine has already initialized the connections when we released the read lock + conns, exists = k.connMap[serviceName] + if exists { + return conns, nil + } + + if err := k.initializeConns(serviceName); err != nil { + return nil, fmt.Errorf("failed to initialize connections for service %s: %v", serviceName, err) + } - return nil, nil + return k.connMap[serviceName], nil } -func (cli *K8sDR) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { - host, err := cli.gatewayHostConsistent.Get(userId) +// GetConn returns a single gRPC client connection for a given Kubernetes service name. +func (k *KubernetesConnManager) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + port, err := k.getServicePort(serviceName) if err != nil { - log.ZError(ctx, "GetUserIdHashGatewayHost error", err) + return nil, err } - return host, err + + fmt.Println("SVC port:", port) + + target := fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceName, k.namespace, port) + + fmt.Println("SVC target:", target) + + return grpc.DialContext( + ctx, + target, + append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, k.dialOptions...)..., + ) } -func getSelfHost(ctx context.Context, gatewayName string) string { - port := 88 - instance := "openimserver" - selfPodName := os.Getenv("MY_POD_NAME") - ns := os.Getenv("MY_POD_NAMESPACE") - statefuleIndex := 0 - gatewayEnds := strings.Split(gatewayName, ":") - if len(gatewayEnds) != 2 { - log.ZError(ctx, "msggateway RpcRegisterName is error:config.RpcRegisterName.OpenImMessageGatewayName", errors.New("config error")) - } else { - port, _ = strconv.Atoi(gatewayEnds[1]) - } - podInfo := strings.Split(selfPodName, "-") - instance = podInfo[0] - count := len(podInfo) - statefuleIndex, _ = strconv.Atoi(podInfo[count-1]) - host := fmt.Sprintf("%s-openim-msggateway-%d.%s-openim-msggateway-headless.%s.svc.cluster.local:%d", instance, statefuleIndex, instance, ns, port) - return host +// GetSelfConnTarget returns the connection target for the current service. +func (k *KubernetesConnManager) GetSelfConnTarget() string { + return k.selfTarget } -// like openimserver-openim-msggateway-0.openimserver-openim-msggateway-headless.openim-lin.svc.cluster.local:88. -// Replica set in kubernetes environment -func getMsgGatewayHost(ctx context.Context, gatewayName string) []string { - port := 88 - instance := "openimserver" - selfPodName := os.Getenv("MY_POD_NAME") - replicas := os.Getenv("MY_MSGGATEWAY_REPLICACOUNT") - ns := os.Getenv("MY_POD_NAMESPACE") - gatewayEnds := strings.Split(gatewayName, ":") - if len(gatewayEnds) != 2 { - log.ZError(ctx, "msggateway RpcRegisterName is error:config.RpcRegisterName.OpenImMessageGatewayName", errors.New("config error")) - } else { - port, _ = strconv.Atoi(gatewayEnds[1]) - } - nReplicas, _ := strconv.Atoi(replicas) - podInfo := strings.Split(selfPodName, "-") - instance = podInfo[0] - var ret []string - for i := 0; i < nReplicas; i++ { - host := fmt.Sprintf("%s-openim-msggateway-%d.%s-openim-msggateway-headless.%s.svc.cluster.local:%d", instance, i, instance, ns, port) - ret = append(ret, host) - } - log.ZDebug(ctx, "getMsgGatewayHost", "instance", instance, "selfPodName", selfPodName, "replicas", replicas, "ns", ns, "ret", ret) - return ret +// AddOption appends gRPC dial options to the existing options. +func (k *KubernetesConnManager) AddOption(opts ...grpc.DialOption) { + k.mu.Lock() + defer k.mu.Unlock() + k.dialOptions = append(k.dialOptions, opts...) } -// GetConns returns the gRPC client connections to the specified service. -func (cli *K8sDR) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - - // This conditional checks if the serviceName is not the OpenImMessageGatewayName. - // It seems to handle a special case for the OpenImMessageGateway. - if serviceName != cli.gatewayName { - // DialContext creates a client connection to the given target (serviceName) using the specified context. - // 'cli.options' are likely default or common options for all connections in this struct. - // 'opts...' allows for additional gRPC dial options to be passed and used. - conn, err := grpc.DialContext(ctx, serviceName, append(cli.options, opts...)...) - - // The function returns a slice of client connections with the new connection, or an error if occurred. - return []*grpc.ClientConn{conn}, err - } else { - // This block is executed if the serviceName is OpenImMessageGatewayName. - // 'ret' will accumulate the connections to return. - var ret []*grpc.ClientConn - - // getMsgGatewayHost presumably retrieves hosts for the message gateway service. - // The context is passed, likely for cancellation and timeout control. - gatewayHosts := getMsgGatewayHost(ctx, cli.gatewayName) - - // Iterating over the retrieved gateway hosts. - for _, host := range gatewayHosts { - // Establishes a connection to each host. - // Again, appending cli.options with any additional opts provided. - conn, err := grpc.DialContext(ctx, host, append(cli.options, opts...)...) - - // If there's an error while dialing any host, the function returns immediately with the error. - if err != nil { - return nil, err - } else { - // If the connection is successful, it is added to the 'ret' slice. - ret = append(ret, conn) - } +// CloseConn closes a given gRPC client connection. +func (k *KubernetesConnManager) CloseConn(conn *grpc.ClientConn) { + conn.Close() +} + +// Close closes all gRPC connections managed by KubernetesConnManager. +func (k *KubernetesConnManager) Close() { + k.mu.Lock() + defer k.mu.Unlock() + for _, conns := range k.connMap { + for _, conn := range conns { + _ = conn.Close() } - // After all hosts are processed, the slice of connections is returned. - return ret, nil } + k.connMap = make(map[string][]*grpc.ClientConn) } -func (cli *K8sDR) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { +func (k *KubernetesConnManager) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { + return nil +} +func (k *KubernetesConnManager) UnRegister() error { + return nil +} - return grpc.DialContext(ctx, serviceName, append(cli.options, opts...)...) +func (k *KubernetesConnManager) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { + return "", nil } -func (cli *K8sDR) GetSelfConnTarget() string { +func (k *KubernetesConnManager) getServicePort(serviceName string) (int32, error) { + svc, err := k.clientset.CoreV1().Services(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{}) + if err != nil { + fmt.Print("namespace:", k.namespace) + return 0, fmt.Errorf("failed to get service %s: %v", serviceName, err) + } - return cli.rpcRegisterAddr -} + if len(svc.Spec.Ports) == 0 { + return 0, fmt.Errorf("service %s has no ports defined", serviceName) + } -func (cli *K8sDR) AddOption(opts ...grpc.DialOption) { - cli.options = append(cli.options, opts...) + return svc.Spec.Ports[0].Port, nil } -func (cli *K8sDR) CloseConn(conn *grpc.ClientConn) { - conn.Close() +// watchEndpoints listens for changes in Pod resources. +func (k *KubernetesConnManager) watchEndpoints() { + informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute*10) + informer := informerFactory.Core().V1().Pods().Informer() + + // Watch for Pod changes (add, update, delete) + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + k.handleEndpointChange(obj) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + k.handleEndpointChange(newObj) + }, + DeleteFunc: func(obj interface{}) { + k.handleEndpointChange(obj) + }, + }) + + informerFactory.Start(context.Background().Done()) + <-context.Background().Done() // Block forever } -// do not use this method for call rpc. -func (cli *K8sDR) GetClientLocalConns() map[string][]*grpc.ClientConn { - log.ZError(context.Background(), "should not call this function!", nil) - return nil +func (k *KubernetesConnManager) handleEndpointChange(obj interface{}) { + endpoint, ok := obj.(*v1.Endpoints) + if !ok { + return + } + serviceName := endpoint.Name + if err := k.initializeConns(serviceName); err != nil { + fmt.Printf("Error initializing connections for %s: %v\n", serviceName, err) + } } -func (cli *K8sDR) Close() { - -} +// ================= + +// initEndpoints initializes connections by fetching all available endpoints in the specified namespace. + +// func (k *KubernetesConnManager) initEndpoints() error { +// k.mu.Lock() +// defer k.mu.Unlock() + +// pods, err := k.clientset.CoreV1().Pods(k.namespace).List(context.TODO(), metav1.ListOptions{}) +// if err != nil { +// return fmt.Errorf("failed to list pods: %v", err) +// } + +// for _, pod := range pods.Items { +// if pod.Status.Phase == v1.PodRunning { +// target := fmt.Sprintf("%s:%d", address.IP, port) +// conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) +// conn, err := k.createGRPCConnection(pod) +// if err != nil { +// return fmt.Errorf("failed to create GRPC connection for pod %s: %v", pod.Name, err) +// } +// k.connMap[pod.Name] = append(k.connMap[pod.Name], conn) +// } +// } + +// return nil +// } + +// ----- + +// func (k *KubernetesConnManager) watchEndpoints1(serviceName string) { +// // watch for changes to the service's endpoints +// informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute) +// endpointsInformer := informerFactory.Core().V1().Endpoints().Informer() + +// endpointsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ +// AddFunc: func(obj interface{}) { +// eps := obj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.initializeConns(serviceName) +// } +// }, +// UpdateFunc: func(oldObj, newObj interface{}) { +// eps := newObj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.initializeConns(serviceName) +// } +// }, +// DeleteFunc: func(obj interface{}) { +// eps := obj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.mu.Lock() +// defer k.mu.Unlock() +// for _, conn := range k.connMap[serviceName] { +// _ = conn.Close() +// } +// delete(k.connMap, serviceName) +// } +// }, +// }) + +// informerFactory.Start(wait.NeverStop) +// informerFactory.WaitForCacheSync(wait.NeverStop) +// } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 7671b1736..ae8344923 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -18,9 +18,6 @@ import ( "context" "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "google.golang.org/grpc/status" "net" "net/http" "os" @@ -28,6 +25,13 @@ import ( "syscall" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/runtimeenv" + "google.golang.org/grpc/status" + + "strconv" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -37,7 +41,6 @@ import ( "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "strconv" ) // Start rpc server. @@ -51,6 +54,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo netErr error ) + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() + if !autoSetPorts { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { @@ -107,11 +112,11 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo registerIP = network.GetListenIP(registerIP) port, _ := strconv.Atoi(portStr) - log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, + log.CInfo(ctx, "RPC server is initializing", "runTimeEnv", runTimeEnv, "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, "prometheusPorts", prometheusConfig.Ports) defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery) + client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) if err != nil { return err } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index a84b7bd87..130035456 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -33,6 +33,7 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" + "github.com/openimsdk/tools/utils/runtimeenv" ) const maxRetry = 180 @@ -78,35 +79,37 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, discovery = &config.Discovery{} thirdConfig = &config.Third{} ) - err := config.LoadConfig(filepath.Join(configDir, cmd.MongodbConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], mongoConfig) + runtimeEnv := runtimeenv.PrintRuntimeEnvironment() + + err := config.Load(configDir, cmd.MongodbConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], runtimeEnv, mongoConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.RedisConfigFileName), cmd.ConfigEnvPrefixMap[cmd.RedisConfigFileName], redisConfig) + err = config.Load(configDir, cmd.RedisConfigFileName, cmd.ConfigEnvPrefixMap[cmd.RedisConfigFileName], runtimeEnv, redisConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.KafkaConfigFileName), cmd.ConfigEnvPrefixMap[cmd.KafkaConfigFileName], kafkaConfig) + err = config.Load(configDir, cmd.KafkaConfigFileName, cmd.ConfigEnvPrefixMap[cmd.KafkaConfigFileName], runtimeEnv, kafkaConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.OpenIMRPCThirdCfgFileName), cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], thirdConfig) + err = config.Load(configDir, cmd.OpenIMRPCThirdCfgFileName, cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } if thirdConfig.Object.Enable == "minio" { - err = config.LoadConfig(filepath.Join(configDir, cmd.MinioConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], minioConfig) + err = config.Load(configDir, cmd.MinioConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], runtimeEnv, minioConfig) if err != nil { return nil, nil, nil, nil, nil, err } } else { minioConfig = nil } - err = config.LoadConfig(filepath.Join(configDir, cmd.DiscoveryConfigFilename), cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], discovery) + err = config.Load(configDir, cmd.DiscoveryConfigFilename, cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], runtimeEnv, discovery) if err != nil { return nil, nil, nil, nil, nil, err } From 54be83776555c57c600a8adeee6b81e54dea6975 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:57:20 +0800 Subject: [PATCH 07/10] fix: compilation failed under Windows (#2940) * fix: kodo sdk windows build error * fix: kodo sdk windows build error --- go.mod | 17 ++++------------- go.sum | 54 ++++++++++++++++-------------------------------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index b278f2a9b..53392f739 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.45 + github.com/openimsdk/tools v0.0.50-alpha.46 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -37,7 +37,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.15-alpha.1 + github.com/openimsdk/gomake v0.0.15-alpha.2 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible @@ -57,11 +57,9 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect - github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect @@ -94,14 +92,11 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/elastic/go-sysinfo v1.0.2 // indirect - github.com/elastic/go-windows v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -111,7 +106,6 @@ require ( github.com/go-openapi/swag v0.22.4 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect - github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -135,7 +129,6 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect @@ -163,7 +156,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.25.0 // indirect + github.com/qiniu/go-sdk/v7 v7.18.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -210,11 +203,9 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gorm.io/gorm v1.25.8 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - modernc.org/fileutil v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 4657f0eca..9bfbb38fb 100644 --- a/go.sum +++ b/go.sum @@ -14,19 +14,15 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= -github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= @@ -90,7 +86,6 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -108,10 +103,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= -github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= -github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= -github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -132,8 +123,6 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= -github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -165,7 +154,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -179,8 +168,6 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -237,8 +224,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -257,7 +242,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -272,15 +256,12 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -309,7 +290,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -365,18 +345,17 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.15-alpha.1 h1:a3U0gayzEJJucOoIOt9Jju+HQTG2BtKcuLG4AVjeVF4= -github.com/openimsdk/gomake v0.0.15-alpha.1/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= +github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.45 h1:bu3HCkdzLNQAPUR1VhF40XA9OhaNGHGxZqcC4rIYJyY= -github.com/openimsdk/tools v0.0.50-alpha.45/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= +github.com/openimsdk/tools v0.0.50-alpha.46 h1:j3HxPxhDptVHwr7eChL2rCH8mKfpUEcr4nHi5k4yDME= +github.com/openimsdk/tools v0.0.50-alpha.46/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -391,12 +370,11 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= -github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -511,7 +489,9 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -539,6 +519,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -548,7 +529,6 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -558,20 +538,22 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= @@ -581,6 +563,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -647,7 +630,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -660,8 +642,6 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= @@ -674,8 +654,6 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 6a7ae690ee00673cf0e5b9cd0254d2714a0969d8 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:43:19 +0800 Subject: [PATCH 08/10] feat: Prometheus can auto set port (#2943) * feat: config * feat: prometheus auto set port --- config/openim-api.yml | 3 + config/openim-msggateway.yml | 1 + config/openim-msgtransfer.yml | 4 +- config/openim-push.yml | 1 + config/openim-rpc-auth.yml | 1 + config/openim-rpc-conversation.yml | 1 + config/openim-rpc-friend.yml | 1 + config/openim-rpc-group.yml | 1 + config/openim-rpc-msg.yml | 1 + config/openim-rpc-third.yml | 1 + config/openim-rpc-user.yml | 1 + config/prometheus.yml | 127 +++++++++++------- docker-compose.yml | 92 ++++++------- go.mod | 4 +- go.sum | 4 +- internal/api/init.go | 65 +++++++-- internal/api/prometheus_discovery.go | 113 ++++++++++++++++ internal/api/router.go | 16 ++- internal/msggateway/init.go | 2 +- internal/msgtransfer/init.go | 69 ++++++++-- pkg/common/config/config.go | 16 ++- pkg/common/config/load_config_test.go | 8 ++ .../discoveryregister/discoveryregister.go | 6 +- pkg/common/prommetrics/api.go | 5 +- pkg/common/prommetrics/discovery.go | 31 +++++ pkg/common/prommetrics/prommetrics.go | 6 +- pkg/common/prommetrics/rpc.go | 5 +- pkg/common/prommetrics/transfer.go | 5 +- pkg/common/startrpc/start.go | 95 +++++++++---- 29 files changed, 521 insertions(+), 164 deletions(-) create mode 100644 internal/api/prometheus_discovery.go create mode 100644 pkg/common/prommetrics/discovery.go diff --git a/config/openim-api.yml b/config/openim-api.yml index 4c38e1005..a23b5fb31 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -10,7 +10,10 @@ api: prometheus: # Whether to enable prometheus enable: true + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true # Prometheus listening ports, must match the number of api.ports + # It will only take effect when autoSetPorts is set to false. ports: [ 12002 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 696bc3d06..b7d6d9847 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -12,6 +12,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 94ed073d8..39b23b222 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -1,6 +1,8 @@ prometheus: # Enable or disable Prometheus monitoring enable: true + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly - # Because four instances have been launched, four ports need to be specified + # It will only take effect when autoSetPorts is set to false. ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027, 12028, 12029, 12030, 12031, 12032, 12033, 12034, 12035 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index a3fe0c765..dcc8f1aec 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -15,6 +15,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 93f365386..961244861 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -15,6 +15,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12200 ] tokenPolicy: diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index c17b5709f..eaedfe21f 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -14,4 +14,5 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12220 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 5b51981d2..920c4860b 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -14,4 +14,5 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12240 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 06fb489d2..c48065cca 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12260 ] diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 5a4beae5c..d07854622 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12280 ] diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index e24f93883..95a50fac4 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12300 ] diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 02b6d45b1..3a1335895 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -14,4 +14,5 @@ prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports + # It will only take effect when autoSetPorts is set to false. ports: [ 12320 ] diff --git a/config/prometheus.yml b/config/prometheus.yml index ab427ee82..6fb112824 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -26,61 +26,94 @@ scrape_configs: - job_name: node_exporter static_configs: - targets: [ internal_ip:20500 ] + - job_name: openimserver-openim-api - static_configs: - - targets: [ internal_ip:12002 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/api" +# static_configs: +# - targets: [ internal_ip:12002 ] +# labels: +# namespace: default + - job_name: openimserver-openim-msggateway - static_configs: - - targets: [ internal_ip:12140 ] -# - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg_gateway" +# static_configs: +# - targets: [ internal_ip:12140 ] +# # - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] +# labels: +# namespace: default + - job_name: openimserver-openim-msgtransfer - static_configs: - - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] -# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg_transfer" +# static_configs: +# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] +# # - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] +# labels: +# namespace: default + - job_name: openimserver-openim-push - static_configs: - - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] -# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/push" +# static_configs: +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] +## - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-auth - static_configs: - - targets: [ internal_ip:12200 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/auth" +# static_configs: +# - targets: [ internal_ip:12200 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-conversation - static_configs: - - targets: [ internal_ip:12220 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/conversation" +# static_configs: +# - targets: [ internal_ip:12220 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-friend - static_configs: - - targets: [ internal_ip:12240 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/friend" +# static_configs: +# - targets: [ internal_ip:12240 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-group - static_configs: - - targets: [ internal_ip:12260 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/group" +# static_configs: +# - targets: [ internal_ip:12260 ] +# labels: +# namespace: default. + - job_name: openimserver-openim-rpc-msg - static_configs: - - targets: [ internal_ip:12280 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg" +# static_configs: +# - targets: [ internal_ip:12280 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-third - static_configs: - - targets: [ internal_ip:12300 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/third" +# static_configs: +# - targets: [ internal_ip:12300 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-user - static_configs: - - targets: [ internal_ip:12320 ] - labels: - namespace: default \ No newline at end of file + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/user" +# static_configs: +# - targets: [ internal_ip:12320 ] +# labels: +# namespace: default \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8d25383bc..57e654208 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,49 +146,49 @@ services: networks: - openim -# prometheus: -# image: ${PROMETHEUS_IMAGE} -# container_name: prometheus -# restart: always -# user: root -# volumes: -# - ./config/prometheus.yml:/etc/prometheus/prometheus.yml -# - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml -# - ${DATA_DIR}/components/prometheus/data:/prometheus -# command: -# - '--config.file=/etc/prometheus/prometheus.yml' -# - '--storage.tsdb.path=/prometheus' -# ports: -# - "19091:9090" -# networks: -# - openim -# -# alertmanager: -# image: ${ALERTMANAGER_IMAGE} -# container_name: alertmanager -# restart: always -# volumes: -# - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml -# - ./config/email.tmpl:/etc/alertmanager/email.tmpl -# ports: -# - "19093:9093" -# networks: -# - openim -# -# grafana: -# image: ${GRAFANA_IMAGE} -# container_name: grafana -# user: root -# restart: always -# environment: -# - GF_SECURITY_ALLOW_EMBEDDING=true -# - GF_SESSION_COOKIE_SAMESITE=none -# - GF_SESSION_COOKIE_SECURE=true -# - GF_AUTH_ANONYMOUS_ENABLED=true -# - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin -# ports: -# - "13000:3000" -# volumes: -# - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana -# networks: -# - openim + prometheus: + image: ${PROMETHEUS_IMAGE} + container_name: prometheus + restart: always + user: root + volumes: + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml + - ${DATA_DIR}/components/prometheus/data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + ports: + - "19091:9090" + networks: + - openim + + alertmanager: + image: ${ALERTMANAGER_IMAGE} + container_name: alertmanager + restart: always + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + - ./config/email.tmpl:/etc/alertmanager/email.tmpl + ports: + - "19093:9093" + networks: + - openim + + grafana: + image: ${GRAFANA_IMAGE} + container_name: grafana + user: root + restart: always + environment: + - GF_SECURITY_ALLOW_EMBEDDING=true + - GF_SESSION_COOKIE_SAMESITE=none + - GF_SESSION_COOKIE_SECURE=true + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "13000:3000" + volumes: + - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana + networks: + - openim diff --git a/go.mod b/go.mod index 53392f739..5d81e67d3 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.46 + github.com/openimsdk/tools v0.0.50-alpha.47 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -42,6 +42,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 + go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 @@ -179,7 +180,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect - go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/go.sum b/go.sum index 9bfbb38fb..99f569a38 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.46 h1:j3HxPxhDptVHwr7eChL2rCH8mKfpUEcr4nHi5k4yDME= -github.com/openimsdk/tools v0.0.50-alpha.46/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.47 h1:Cfe2va/g6WhLjOoQqZkjrdlEDq1dUsfcQsdUB5oADVA= +github.com/openimsdk/tools v0.0.50-alpha.47/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index ea7b6133f..8fb6baff5 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -16,6 +16,7 @@ package api import ( "context" + "errors" "fmt" "net" "net/http" @@ -26,16 +27,17 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/network" - "github.com/openimsdk/tools/utils/runtimeenv" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/system/program" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/tools/utils/runtimeenv" ) type Config struct { @@ -68,16 +70,57 @@ func Start(ctx context.Context, index int, config *Config) error { prometheusPort int ) - router := newGinRouter(client, config) + registerIP, err := network.GetRpcRegisterIP("") + if err != nil { + return err + } + + getAutoPort := func() (net.Listener, int, error) { + registerAddr := net.JoinHostPort(registerIP, "0") + listener, err := net.Listen("tcp", registerAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if config.API.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() + } + + router := newGinRouter(client, config, client) if config.API.Prometheus.Enable { - go func() { + var ( + listener net.Listener + ) + + if config.API.Prometheus.AutoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { prometheusPort, err = datautil.GetElemByIndex(config.API.Prometheus.Ports, index) if err != nil { - netErr = err - netDone <- struct{}{} - return + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) } - if err := prommetrics.ApiInit(prometheusPort); err != nil && err != http.ErrServerClosed { + } + + go func() { + if err := prommetrics.ApiInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } @@ -90,7 +133,7 @@ func Start(ctx context.Context, index int, config *Config) error { log.CInfo(ctx, "API server is initializing", "runtimeEnv", config.RuntimeEnv, "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) go func() { err = server.ListenAndServe() - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("api start err: %s", server.Addr)) netDone <- struct{}{} diff --git a/internal/api/prometheus_discovery.go b/internal/api/prometheus_discovery.go new file mode 100644 index 000000000..6e33274be --- /dev/null +++ b/internal/api/prometheus_discovery.go @@ -0,0 +1,113 @@ +package api + +import ( + "encoding/json" + "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + clientv3 "go.etcd.io/etcd/client/v3" + "net/http" +) + +type PrometheusDiscoveryApi struct { + config *Config + client *clientv3.Client +} + +func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegistry) *PrometheusDiscoveryApi { + api := &PrometheusDiscoveryApi{ + config: config, + } + if config.Discovery.Enable == discoveryregister.Etcd { + api.client = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + } + return api +} + +func (p *PrometheusDiscoveryApi) Enable(c *gin.Context) { + if p.config.Discovery.Enable != discoveryregister.Etcd { + c.JSON(http.StatusOK, []struct{}{}) + c.Abort() + } +} + +func (p *PrometheusDiscoveryApi) discovery(c *gin.Context, key string) { + eResp, err := p.client.Get(c, prommetrics.BuildDiscoveryKey(key)) + if err != nil { + // Log and respond with an error if preparation fails. + apiresp.GinError(c, errs.WrapMsg(err, "etcd get err")) + return + } + if len(eResp.Kvs) == 0 { + c.JSON(http.StatusOK, []*prommetrics.Target{}) + } + + var ( + resp = &prommetrics.RespTarget{ + Targets: make([]string, 0, len(eResp.Kvs)), + } + ) + + for i := range eResp.Kvs { + var target prommetrics.Target + err = json.Unmarshal(eResp.Kvs[i].Value, &target) + if err != nil { + log.ZError(c, "prometheus unmarshal err", errs.Wrap(err)) + } + resp.Targets = append(resp.Targets, target.Target) + if resp.Labels == nil { + resp.Labels = target.Labels + } + } + + c.JSON(200, []*prommetrics.RespTarget{resp}) +} + +func (p *PrometheusDiscoveryApi) Api(c *gin.Context) { + p.discovery(c, prommetrics.APIKeyName) +} + +func (p *PrometheusDiscoveryApi) User(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.User) +} + +func (p *PrometheusDiscoveryApi) Group(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Group) +} + +func (p *PrometheusDiscoveryApi) Msg(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Msg) +} + +func (p *PrometheusDiscoveryApi) Friend(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Friend) +} + +func (p *PrometheusDiscoveryApi) Conversation(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Conversation) +} + +func (p *PrometheusDiscoveryApi) Third(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Third) +} + +func (p *PrometheusDiscoveryApi) Auth(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Auth) +} + +func (p *PrometheusDiscoveryApi) Push(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Push) +} + +func (p *PrometheusDiscoveryApi) MessageGateway(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.MessageGateway) +} + +func (p *PrometheusDiscoveryApi) MessageTransfer(c *gin.Context) { + p.discovery(c, prommetrics.MessageTransferKeyName) +} diff --git a/internal/api/router.go b/internal/api/router.go index 52d26bdc5..6714d645c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -48,7 +48,7 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { +func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client discovery.SvcDiscoveryRegistry) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) gin.SetMode(gin.ReleaseMode) @@ -78,6 +78,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) + pd := NewPrometheusDiscoveryApi(config, client) userRouterGroup := r.Group("/user") { userRouterGroup.POST("/user_register", u.UserRegister) @@ -254,6 +255,19 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En jssdk.POST("/get_conversations", j.GetConversations) jssdk.POST("/get_active_conversations", j.GetActiveConversations) + proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) + proDiscoveryGroup.GET("/api", pd.Api) + proDiscoveryGroup.GET("/user", pd.User) + proDiscoveryGroup.GET("/group", pd.Group) + proDiscoveryGroup.GET("/msg", pd.Msg) + proDiscoveryGroup.GET("/friend", pd.Friend) + proDiscoveryGroup.GET("/conversation", pd.Conversation) + proDiscoveryGroup.GET("/third", pd.Third) + proDiscoveryGroup.GET("/auth", pd.Auth) + proDiscoveryGroup.GET("/push", pd.Push) + proDiscoveryGroup.GET("/msg_gateway", pd.MessageGateway) + proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) + return r } diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 7603147de..6654e6598 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -41,7 +41,7 @@ type Config struct { func Start(ctx context.Context, index int, conf *Config) error { conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 6131d8c77..9da2e7974 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -18,11 +18,17 @@ import ( "context" "errors" "fmt" + "net" "net/http" "os" "os/signal" + "strconv" "syscall" + "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -33,6 +39,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" @@ -140,21 +147,67 @@ func (m *MsgTransfer) Start(index int, config *Config) error { return err } + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, m.runTimeEnv) + if err != nil { + return errs.WrapMsg(err, "failed to register discovery service") + } + + registerIP, err := network.GetRpcRegisterIP("") + if err != nil { + return err + } + + getAutoPort := func() (net.Listener, int, error) { + registerAddr := net.JoinHostPort(registerIP, "0") + listener, err := net.Listen("tcp", registerAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if config.MsgTransfer.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() + } + if config.MsgTransfer.Prometheus.Enable { + var ( + listener net.Listener + prometheusPort int + ) + + if config.MsgTransfer.Prometheus.AutoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(context.TODO(), prommetrics.BuildDiscoveryKey(prommetrics.MessageTransferKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { + prometheusPort, err = datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) + if err != nil { + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) + } + } + go func() { defer func() { if r := recover(); r != nil { mw.PanicStackToLog(m.ctx, r) } }() - prometheusPort, err := datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) - if err != nil { - netErr = err - netDone <- struct{}{} - return - } - - if err := prommetrics.TransferInit(prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index b8a60c3df..25748a618 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -18,11 +18,10 @@ import ( "strings" "time" - "github.com/openimsdk/tools/s3/aws" - "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/s3/aws" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" @@ -108,9 +107,10 @@ type API struct { CompressionLevel int `mapstructure:"compressionLevel"` } `mapstructure:"api"` Prometheus struct { - Enable bool `mapstructure:"enable"` - Ports []int `mapstructure:"ports"` - GrafanaURL string `mapstructure:"grafanaURL"` + Enable bool `mapstructure:"enable"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` + GrafanaURL string `mapstructure:"grafanaURL"` } `mapstructure:"prometheus"` } @@ -193,7 +193,11 @@ type MsgGateway struct { } type MsgTransfer struct { - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus struct { + Enable bool `mapstructure:"enable"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` + } `mapstructure:"prometheus"` } type Push struct { diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index ea11b0574..1552c77d2 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -83,3 +83,11 @@ func TestLoadOpenIMThirdConfig(t *testing.T) { // Environment: IMENV_OPENIM_RPC_THIRD_OBJECT_ENABLE=enabled;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ENDPOINT=https://oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKET=my_bucket_name;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKETURL=https://my_bucket_name.oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYID=AKID1234567890;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYSECRET=abc123xyz789;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_SESSIONTOKEN=session_token_value;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_PUBLICREAD=true } + +func TestTransferConfig(t *testing.T) { + var tran MsgTransfer + err := LoadConfig("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", &tran) + assert.Nil(t, err) + assert.Equal(t, true, tran.Prometheus.Enable) + assert.Equal(t, true, tran.Prometheus.AutoSetPorts) +} diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 419293a91..dde629308 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -26,6 +26,10 @@ import ( "github.com/openimsdk/tools/errs" ) +const ( + Etcd = "etcd" +) + // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { if runtimeEnv == "kubernetes" { @@ -35,7 +39,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (disco switch discovery.Enable { case "kubernetes": return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace) - case "etcd": + case Etcd: return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 95b5c06b6..2dc5cb65d 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -3,6 +3,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" "strconv" ) @@ -23,14 +24,14 @@ var ( ) ) -func ApiInit(prometheusPort int) error { +func ApiInit(listener net.Listener) error { apiRegistry := prometheus.NewRegistry() cs := append( baseCollector, apiCounter, httpCounter, ) - return Init(apiRegistry, prometheusPort, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) + return Init(apiRegistry, listener, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) } func APICall(path string, method string, apiCode int) { diff --git a/pkg/common/prommetrics/discovery.go b/pkg/common/prommetrics/discovery.go new file mode 100644 index 000000000..8f03bc2ae --- /dev/null +++ b/pkg/common/prommetrics/discovery.go @@ -0,0 +1,31 @@ +package prommetrics + +import "fmt" + +const ( + APIKeyName = "api" + MessageTransferKeyName = "message-transfer" +) + +type Target struct { + Target string `json:"target"` + Labels map[string]string `json:"labels"` +} + +type RespTarget struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + +func BuildDiscoveryKey(name string) string { + return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name) +} + +func BuildDefaultTarget(host string, ip int) Target { + return Target{ + Target: fmt.Sprintf("%s:%d", host, ip), + Labels: map[string]string{ + "namespace": "default", + }, + } +} diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 02e408d63..2fc5d76b4 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,9 +15,9 @@ package prommetrics import ( - "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "net" "net/http" ) @@ -30,9 +30,9 @@ var ( } ) -func Init(registry *prometheus.Registry, prometheusPort int, path string, handler http.Handler, cs ...prometheus.Collector) error { +func Init(registry *prometheus.Registry, listener net.Listener, path string, handler http.Handler, cs ...prometheus.Collector) error { registry.MustRegister(cs...) srv := http.NewServeMux() srv.Handle(path, handler) - return http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv) + return http.Serve(listener, srv) } diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 809d509b2..3f115d30b 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -5,6 +5,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" "strconv" ) @@ -21,13 +22,13 @@ var ( ) ) -func RpcInit(cs []prometheus.Collector, prometheusPort int) error { +func RpcInit(cs []prometheus.Collector, listener net.Listener) error { reg := prometheus.NewRegistry() cs = append(append( baseCollector, rpcCounter, ), cs...) - return Init(reg, prometheusPort, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) + return Init(reg, listener, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) } func RPCCall(name string, path string, code int) { diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index f0abb8285..36fe1d568 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -17,6 +17,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" ) var ( @@ -42,7 +43,7 @@ var ( }) ) -func TransferInit(prometheusPort int) error { +func TransferInit(listener net.Listener) error { reg := prometheus.NewRegistry() cs := append( baseCollector, @@ -52,5 +53,5 @@ func TransferInit(prometheusPort int) error { MsgInsertMongoFailedCounter, SeqSetFailedCounter, ) - return Init(reg, prometheusPort, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) + return Init(reg, listener, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index ae8344923..f75a50541 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -22,15 +22,17 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/runtimeenv" + "github.com/openimsdk/tools/utils/jsonutil" "google.golang.org/grpc/status" - "strconv" + "github.com/openimsdk/tools/utils/runtimeenv" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -49,11 +51,17 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { var ( - rpcTcpAddr string - netDone = make(chan struct{}, 2) - netErr error + rpcTcpAddr string + netDone = make(chan struct{}, 2) + netErr error + prometheusPort int ) + registerIP, err := network.GetRpcRegisterIP(registerIP) + if err != nil { + return err + } + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() if !autoSetPorts { @@ -66,6 +74,27 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0") } + getAutoPort := func() (net.Listener, int, error) { + listener, err := net.Listen("tcp", rpcTcpAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if autoSetPorts && discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap() + } + client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) + if err != nil { + return err + } + + defer client.Close() + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + // var reg *prometheus.Registry // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { @@ -78,17 +107,40 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo prommetricsUnaryInterceptor(rpcRegisterName), prommetricsStreamInterceptor(rpcRegisterName), ) - prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) - if err != nil { - return err + + var ( + listener net.Listener + ) + + if autoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { + prometheusPort, err = datautil.GetElemByIndex(prometheusConfig.Ports, index) + if err != nil { + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) + } } cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, discovery) go func() { - if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := prommetrics.RpcInit(cs, listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } - // metric.InitializeMetrics(srv) + //metric.InitializeMetrics(srv) // Create a HTTP server for prometheus. // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -100,30 +152,15 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo options = append(options, mw.GrpcServer()) } - listener, err := net.Listen( - "tcp", - rpcTcpAddr, - ) - if err != nil { - return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } - - _, portStr, _ := net.SplitHostPort(listener.Addr().String()) - registerIP = network.GetListenIP(registerIP) - port, _ := strconv.Atoi(portStr) - - log.CInfo(ctx, "RPC server is initializing", "runTimeEnv", runTimeEnv, "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, - "prometheusPorts", prometheusConfig.Ports) - - defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) + listener, port, err := getAutoPort() if err != nil { return err } - defer client.Close() - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", port, + "prometheusPort", prometheusPort) + defer listener.Close() srv := grpc.NewServer(options...) err = rpcFn(ctx, config, client, srv) From 42136c6b93170f90e63fc7fa4034a3c98b123a21 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Thu, 12 Dec 2024 10:50:40 +0800 Subject: [PATCH 09/10] fix: server can return isEnd to control fetch messages when sdk pull messages end normally. (#2949) --- go.mod | 2 +- go.sum | 4 +- internal/rpc/msg/sync_msg.go | 4 +- pkg/common/storage/controller/msg.go | 81 +++++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5d81e67d3..6c5bf79e3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.61 + github.com/openimsdk/protocol v0.0.72-alpha.63 github.com/openimsdk/tools v0.0.50-alpha.47 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 99f569a38..a7cf42eca 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= -github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= +github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.47 h1:Cfe2va/g6WhLjOoQqZkjrdlEDq1dUsfcQsdUB5oADVA= github.com/openimsdk/tools v0.0.50-alpha.47/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 2f7788167..13e3cfd33 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -92,11 +92,13 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq NotificationMsgs: make(map[string]*sdkws.PullMsgs), } for _, conv := range req.Conversations { - _, _, msgs, err := m.MsgDatabase.GetMsgBySeqs(ctx, req.UserID, conv.ConversationID, conv.Seqs) + isEnd, endSeq, msgs, err := m.MsgDatabase.GetMessagesBySeqWithBounds(ctx, req.UserID, conv.ConversationID, conv.Seqs, req.GetOrder()) if err != nil { return nil, err } var pullMsgs *sdkws.PullMsgs + pullMsgs.IsEnd = isEnd + pullMsgs.EndSeq = endSeq if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] if !ok { diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 464ad7604..8d82d8543 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -24,6 +24,9 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -35,8 +38,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/timeutil" - "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/mongo" ) const ( @@ -56,6 +57,7 @@ type CommonMsgDatabase interface { GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // DeleteConversationMsgsAndSetMinSeq deletes conversation messages and resets the minimum sequence number. If `remainTime` is 0, all messages are deleted (this method does not delete Redis // cache). + GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) @@ -517,6 +519,81 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co return minSeq, maxSeq, successMsgs, nil } +func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) { + var endSeq int64 + var isEnd bool + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) + if err != nil { + return false, 0, nil, err + } + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { + return false, 0, nil, err + } + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { + return false, 0, nil, err + } + userMaxSeq, err := db.seqUser.GetUserMaxSeq(ctx, conversationID, userID) + if err != nil { + return false, 0, nil, err + } + if userMinSeq > minSeq { + minSeq = userMinSeq + } + if userMaxSeq > 0 && userMaxSeq < maxSeq { + maxSeq = userMaxSeq + } + newSeqs := make([]int64, 0, len(seqs)) + for _, seq := range seqs { + if seq <= 0 { + continue + } + // The normal range and can fetch messages + if seq >= minSeq && seq <= maxSeq { + newSeqs = append(newSeqs, seq) + continue + } + // If the requested seq is smaller than the minimum seq and the pull order is descending (pulling older messages) + if seq < minSeq && pullOrder == sdkws.PullOrder_PullOrderDesc { + isEnd = true + endSeq = minSeq + } + // If the requested seq is larger than the maximum seq and the pull order is ascending (pulling newer messages) + if seq > maxSeq && pullOrder == sdkws.PullOrder_PullOrderAsc { + isEnd = true + endSeq = maxSeq + } + } + if len(newSeqs) == 0 { + return isEnd, endSeq, nil, nil + } + successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + if err != nil { + if !errors.Is(err, redis.Nil) { + log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) + } + } + log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", + seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) + + if len(failedSeqs) > 0 { + mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) + if err != nil { + + return false, 0, nil, err + } + + successMsgs = append(successMsgs, mongoMsgs...) + + //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + //if err != nil { + // return 0, 0, nil, err + //} + } + return isEnd, endSeq, successMsgs, nil +} + func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error { var delStruct delMsgRecursionStruct var skip int64 From 120afdf6ae386542d74bddec20f2dafeda2cfc3d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:47:52 +0800 Subject: [PATCH 10/10] feat: Change upload logs systemType to AppFramework. (#2927) * feat: change upload logs systemType to AppFramework * feat: change upload logs systemType to AppFramework * Merge: main --- internal/rpc/third/log.go | 16 ++++++++-------- pkg/common/storage/model/log.go | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index cb4678b34..4eeb5d558 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -50,14 +50,14 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) platform := constant.PlatformID2Name[int(req.Platform)] for _, fileURL := range req.FileURLs { log := relationtb.Log{ - Platform: platform, - UserID: userID, - CreateTime: time.Now(), - Url: fileURL.URL, - FileName: fileURL.Filename, - SystemType: req.AppFramework, - Version: req.Version, - Ex: req.Ex, + Platform: platform, + UserID: userID, + CreateTime: time.Now(), + Url: fileURL.URL, + FileName: fileURL.Filename, + AppFramework: req.AppFramework, + Version: req.Version, + Ex: req.Ex, } for i := 0; i < 20; i++ { id := genLogID() diff --git a/pkg/common/storage/model/log.go b/pkg/common/storage/model/log.go index 9db72c695..9dc392179 100644 --- a/pkg/common/storage/model/log.go +++ b/pkg/common/storage/model/log.go @@ -19,13 +19,14 @@ import ( ) type Log struct { - LogID string `bson:"log_id"` - Platform string `bson:"platform"` - UserID string `bson:"user_id"` - CreateTime time.Time `bson:"create_time"` - Url string `bson:"url"` - FileName string `bson:"file_name"` - SystemType string `bson:"system_type"` - Version string `bson:"version"` - Ex string `bson:"ex"` + LogID string `bson:"log_id"` + Platform string `bson:"platform"` + UserID string `bson:"user_id"` + CreateTime time.Time `bson:"create_time"` + Url string `bson:"url"` + FileName string `bson:"file_name"` + SystemType string `bson:"system_type"` + AppFramework string `bson:"app_framework"` + Version string `bson:"version"` + Ex string `bson:"ex"` }