From 86a325f309f0598abf4e5d0a60ee5d893e62c0bd Mon Sep 17 00:00:00 2001
From: icey-yu <119291641+icey-yu@users.noreply.github.com>
Date: Mon, 26 Aug 2024 18:15:05 +0800
Subject: [PATCH] Fix push (#2559)

* fix:log

* fix: add log

* fix: del return

* fix: push config

* feat: add push err log and extend push wait time

* feat: group config

* feat: add log in write binary msg

* feat: Modify Prometheus data scraping ports and reserve port space.

* feat: change group rpc num

* feat: change log

* fix: remove quotation mark
---
 config/openim-api.yml              |  2 +-
 config/openim-msggateway.yml       |  2 +-
 config/openim-msgtransfer.yml      |  2 +-
 config/openim-push.yml             |  4 ++--
 config/openim-rpc-auth.yml         |  2 +-
 config/openim-rpc-conversation.yml |  2 +-
 config/openim-rpc-friend.yml       |  2 +-
 config/openim-rpc-group.yml        |  4 ++--
 config/openim-rpc-msg.yml          |  2 +-
 config/openim-rpc-third.yml        |  2 +-
 config/openim-rpc-user.yml         |  2 +-
 config/prometheus.yml              | 26 +++++++++++++-------------
 internal/msggateway/client.go      |  2 ++
 internal/push/push_handler.go      |  4 ++--
 pkg/rpcclient/msg.go               |  4 +++-
 start-config.yml                   |  2 +-
 16 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/config/openim-api.yml b/config/openim-api.yml
index 78a688fcd..9f53038d8 100644
--- a/config/openim-api.yml
+++ b/config/openim-api.yml
@@ -8,6 +8,6 @@ prometheus:
   # Whether to enable prometheus
   enable: true
   # Prometheus listening ports, must match the number of api.ports
-  ports: [ 20113 ]
+  ports: [ 20502 ]
   # This address can be accessed via a browser
   grafanaURL: http://127.0.0.1:13000/
diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml
index d8068c443..63332740f 100644
--- a/config/openim-msggateway.yml
+++ b/config/openim-msggateway.yml
@@ -8,7 +8,7 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20112 ]
+  ports: [ 20640 ]
 
 # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP
 listenIP: 0.0.0.0
diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml
index 07a7dc1ab..e71a218ed 100644
--- a/config/openim-msgtransfer.yml
+++ b/config/openim-msgtransfer.yml
@@ -3,4 +3,4 @@ prometheus:
   enable: true
   # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly
   # Because four instances have been launched, four ports need to be specified
-  ports: [ 20108, 20109, 20110, 20111 ]
+  ports: [ 20600, 20601, 20602, 20603 ]
diff --git a/config/openim-push.yml b/config/openim-push.yml
index 1e55cdee8..70aa5997f 100644
--- a/config/openim-push.yml
+++ b/config/openim-push.yml
@@ -4,13 +4,13 @@ rpc:
   # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP
   listenIP: 0.0.0.0
   # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports
-  ports: [ 10170 ]
+  ports: [ 10170, 10171, 10172, 10173 ]
 
 prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20107 ]
+  ports: [ 20670, 20671, 20672, 20673 ]
 
 maxConcurrentWorkers: 3
 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified.
diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml
index 979eca06f..c55c745b6 100644
--- a/config/openim-rpc-auth.yml
+++ b/config/openim-rpc-auth.yml
@@ -10,7 +10,7 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20106 ]
+  ports: [ 20660 ]
 
 tokenPolicy:
   # Token validity period, in days
diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml
index d3f822501..00c9c5aab 100644
--- a/config/openim-rpc-conversation.yml
+++ b/config/openim-rpc-conversation.yml
@@ -10,4 +10,4 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20105 ]
+  ports: [ 20680 ]
diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml
index 8d5869ce8..afac3c5db 100644
--- a/config/openim-rpc-friend.yml
+++ b/config/openim-rpc-friend.yml
@@ -10,4 +10,4 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20104 ]
+  ports: [ 20620 ]
diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml
index 5fff8b9fe..d0243b5fb 100644
--- a/config/openim-rpc-group.yml
+++ b/config/openim-rpc-group.yml
@@ -1,6 +1,6 @@
 rpc:
   # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP
-  registerIP: 
+  registerIP:
   # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP
   listenIP: 0.0.0.0
   # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports
@@ -10,7 +10,7 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20103 ]
+  ports: [ 20650 ]
 
 
 enableHistoryForNewMembers: true
\ No newline at end of file
diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml
index 38cc46ecf..15840c7f3 100644
--- a/config/openim-rpc-msg.yml
+++ b/config/openim-rpc-msg.yml
@@ -10,7 +10,7 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20102 ]
+  ports: [ 20630 ]
 
 
 # Does sending messages require friend verification
diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml
index 408251f4d..512ca391f 100644
--- a/config/openim-rpc-third.yml
+++ b/config/openim-rpc-third.yml
@@ -10,7 +10,7 @@ prometheus:
   # Enable or disable Prometheus monitoring
   enable: true
   # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup
-  ports: [ 20101 ]
+  ports: [ 20690 ]
 
 
 object:
diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml
index ec9bb30dc..1958bbc6a 100644
--- a/config/openim-rpc-user.yml
+++ b/config/openim-rpc-user.yml
@@ -10,7 +10,7 @@ prometheus:
   # Whether to enable prometheus
   enable: true
   # Prometheus listening ports, must be consistent with the number of rpc.ports
-  ports: [ 20100 ]
+  ports: [ 20610 ]
 
 
 
diff --git a/config/prometheus.yml b/config/prometheus.yml
index c7ce4a489..627cf9411 100644
--- a/config/prometheus.yml
+++ b/config/prometheus.yml
@@ -19,65 +19,65 @@ rule_files:
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
 scrape_configs:
-  # The job name is added as a label job=job_name" to any timeseries scraped from this config.
+  # The job name is added as a label "job=job_name" to any timeseries scraped from this config.
   # Monitored information captured by prometheus
 
   # prometheus fetches application services
   - job_name: node_exporter
     static_configs:
-      - targets: [ internal_ip:20114 ]
+      - targets: [ internal_ip:20500 ]
   - job_name: openimserver-openim-api
     static_configs:
-      - targets: [ internal_ip:20113 ]
+      - targets: [ internal_ip:20502 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-msggateway
     static_configs:
-      - targets: [ internal_ip:20112 ]
+      - targets: [ internal_ip:20640 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-msgtransfer
     static_configs:
-      - targets: [ internal_ip:20111, internal_ip:20110, internal_ip:20109, internal_ip:20108 ]
+      - targets: [ internal_ip:20600, internal_ip:20601, internal_ip:20602, internal_ip:20603 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-push
     static_configs:
-      - targets: [ internal_ip:20107 ]
+      - targets: [ internal_ip:20670, internal_ip:20671, internal_ip:20672, internal_ip:20673]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-auth
     static_configs:
-      - targets: [ internal_ip:20106 ]
+      - targets: [ internal_ip:20600 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-conversation
     static_configs:
-      - targets: [ internal_ip:20105 ]
+      - targets: [ internal_ip:20680 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-friend
     static_configs:
-      - targets: [ internal_ip:20104 ]
+      - targets: [ internal_ip:20620 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-group
     static_configs:
-      - targets: [ internal_ip:20103 ]
+      - targets: [ internal_ip:20650 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-msg
     static_configs:
-      - targets: [ internal_ip:20102 ]
+      - targets: [ internal_ip:20630 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-third
     static_configs:
-      - targets: [ internal_ip:20101 ]
+      - targets: [ internal_ip:20690 ]
         labels:
           namespace: default
   - job_name: openimserver-openim-rpc-user
     static_configs:
-      - targets: [ internal_ip:20100 ]
+      - targets: [ internal_ip:20610 ]
         labels:
           namespace: default
\ No newline at end of file
diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go
index a4902570a..dcb15b70d 100644
--- a/internal/msggateway/client.go
+++ b/internal/msggateway/client.go
@@ -271,11 +271,13 @@ func (c *Client) replyMessage(ctx context.Context, binaryReq *Req, err error, re
 		ErrMsg:        errResp.ErrMsg,
 		Data:          resp,
 	}
+	t := time.Now()
 	log.ZDebug(ctx, "gateway reply message", "resp", mReply.String())
 	err = c.writeBinaryMsg(mReply)
 	if err != nil {
 		log.ZWarn(ctx, "wireBinaryMsg replyMessage", err, "resp", mReply.String())
 	}
+	log.ZDebug(ctx, "wireBinaryMsg end", "time cost", time.Since(t))
 
 	if binaryReq.ReqIdentifier == WsLogoutMsg {
 		return errs.New("user logout", "operationID", binaryReq.OperationID).Wrap()
diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go
index 8ecb3dad1..79d3a9296 100644
--- a/internal/push/push_handler.go
+++ b/internal/push/push_handler.go
@@ -91,9 +91,9 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) {
 	}
 	sec := msgFromMQ.MsgData.SendTime / 1000
 	nowSec := timeutil.GetCurrentTimestampBySecond()
-	log.ZDebug(ctx, "push msg", "msg", pbData.String(), "sec", sec, "nowSec", nowSec)
+
 	if nowSec-sec > 10 {
-		return
+		log.ZWarn(ctx, "long time push msg", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec)
 	}
 	var err error
 	switch msgFromMQ.MsgData.SessionType {
diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go
index 72d5aab95..958cb69a6 100644
--- a/pkg/rpcclient/msg.go
+++ b/pkg/rpcclient/msg.go
@@ -355,7 +355,9 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co
 }
 
 func (s *NotificationSender) NotificationWithSessionType(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) {
-	s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) })
+	if err := s.queue.Push(func() { s.send(ctx, sendID, recvID, contentType, sessionType, m, opts...) }); err != nil {
+		log.ZWarn(ctx, "Push to queue failed", err, "sendID", sendID, "recvID", recvID, "msg", jsonutil.StructToJsonString(m))
+	}
 }
 
 func (s *NotificationSender) Notification(ctx context.Context, sendID, recvID string, contentType int32, m proto.Message, opts ...NotificationOptions) {
diff --git a/start-config.yml b/start-config.yml
index 21436d7a9..a6d3e47af 100644
--- a/start-config.yml
+++ b/start-config.yml
@@ -3,7 +3,7 @@ serviceBinaries:
   openim-crontask: 1
   openim-rpc-user: 1
   openim-msggateway: 1
-  openim-push: 1
+  openim-push: 4
   openim-msgtransfer: 4
   openim-rpc-conversation: 1
   openim-rpc-auth: 1