Merge remote-tracking branch 'upstream/main' into main

# Conflicts:
#	config/alertmanager.yml
#	config/email.tmpl
#	deployments/templates/prometheus.yml
#	docker-compose.yml
#	scripts/install/environment.sh
pull/1455/head
lin.huang 2 years ago
commit 053c918e46

@ -98,3 +98,14 @@ jobs:
- name: Exec OpenIM System uninstall
run: |
sudo ./scripts/install/install.sh -u
- name: gobenchdata publish
uses: bobheadxi/gobenchdata@v1
with:
PRUNE_COUNT: 30
GO_TEST_FLAGS: -cpu 1,2
PUBLISH: true
PUBLISH_BRANCH: gh-pages
env:
GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}
continue-on-error: true

@ -41,7 +41,7 @@ jobs:
# ./*.md all markdown files in the root directory
args: --verbose -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md'
env:
GITHUB_TOKEN: ${{secrets.GH_PAT}}
GITHUB_TOKEN: ${{secrets.BOT_GITHUB_TOKEN}}
- name: Create Issue From File
if: env.lychee_exit_code != 0

@ -30,7 +30,7 @@
</p>
## 🟢 扫描微信进群交流
<img src="https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg" width="300">
<img src="./docs/images/Wechat.jpg" width="300">
## Ⓜ️ 关于 OpenIM

@ -231,7 +231,7 @@ Before you start, please make sure your changes are in demand. The best for that
- [OpenIM Makefile Utilities](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-makefile.md)
- [OpenIM Script Utilities](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-scripts.md)
- [OpenIM Versioning](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/version.md)
- [Manage backend and monitor deployment](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md)
## :busts_in_silhouette: Community

@ -1,25 +1,32 @@
###################### AlertManager Configuration ######################
# AlertManager configuration using environment variables
#
# Resolve timeout
# SMTP configuration for sending alerts
# Templates for email notifications
# Routing configurations for alerts
# Receiver configurations
global:
resolve_timeout: 5m
smtp_from: 'openimalert@163.com'
smtp_smarthost: 'smtp.163.com:465'
smtp_auth_username: 'openimalert@163.com'
smtp_auth_password: 'YOURAUTHPASSWORD'
smtp_from: alert@openim.io
smtp_smarthost: smtp.163.com:465
smtp_auth_username: alert@openim.io
smtp_auth_password: YOURAUTHPASSWORD
smtp_require_tls: false
smtp_hello: 'xxx监控告警'
smtp_hello: xxx监控告警
templates:
- '/etc/alertmanager/email.tmpl'
- /etc/alertmanager/email.tmpl
route:
group_by: ['alertname']
group_wait: 5s
group_interval: 5s
repeat_interval: 5m
receiver: 'email'
receiver: email
receivers:
- name: 'email'
- name: email
email_configs:
- to: '239374037@qq.com'
- to: {EMAIL_TO:-'alert@example.com'}
html: '{{ template "email.to.html" . }}'
headers: { Subject: "[OPENIM-SERVER]告警" }
headers: { Subject: "[OPENIM-SERVER]Alarm" }
send_resolved: true

@ -1,15 +1,16 @@
{{ define "email.to.html" }}
{{ range .Alerts }}
-------------------------openim alert---------------------<br>
告警程序: prometheus_alert<br>
告警级别: {{ .Labels.severity }} 级<br>
告警类型: {{ .Labels.alertname }}<br>
故障主机: {{ .Labels.instance }}<br>
故障服务: {{ .Labels.job }}<br>
告警主题: {{ .Annotations.summary }}<br>
触发时间: {{ .StartsAt.Format "2020-01-02 15:04:05"}} <br>
----------------------------------------------------------<br>
<br>
<br>
<!-- Begin of OpenIM Alert -->
<div style="border:1px solid #ccc; padding:10px; margin-bottom:10px;">
<h3>OpenIM Alert</h3>
<p><strong>Alert Program:</strong> Prometheus Alert</p>
<p><strong>Severity Level:</strong> {{ .Labels.severity }}</p>
<p><strong>Alert Type:</strong> {{ .Labels.alertname }}</p>
<p><strong>Affected Host:</strong> {{ .Labels.instance }}</p>
<p><strong>Affected Service:</strong> {{ .Labels.job }}</p>
<p><strong>Alert Subject:</strong> {{ .Annotations.summary }}</p>
<p><strong>Trigger Time:</strong> {{ .StartsAt.Format "2006-01-02 15:04:05" }}</p>
</div>
<!-- End of OpenIM Alert -->
{{ end }}
{{ end }}

@ -0,0 +1,11 @@
groups:
- name: instance_down
rules:
- alert: InstanceDown
expr: up == 0
for: 1m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."

@ -6,13 +6,13 @@ global:
# Alertmanager configuration
alerting:
#alertmanagers:
# - static_configs:
# - targets: ['172.29.166.17:9093'] #alertmanager地址
alertmanagers:
- static_configs:
- targets: ['172.28.0.1:19093']
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "node_down.yml"
- "instance-down-rules.yml"
# - "first_rules.yml"
# - "second_rules.yml"

@ -0,0 +1,32 @@
###################### AlertManager Configuration ######################
# AlertManager configuration using environment variables
#
# Resolve timeout
# SMTP configuration for sending alerts
# Templates for email notifications
# Routing configurations for alerts
# Receiver configurations
global:
resolve_timeout: ${ALERTMANAGER_RESOLVE_TIMEOUT}
smtp_from: ${ALERTMANAGER_SMTP_FROM}
smtp_smarthost: ${ALERTMANAGER_SMTP_SMARTHOST}
smtp_auth_username: ${ALERTMANAGER_SMTP_AUTH_USERNAME}
smtp_auth_password: ${ALERTMANAGER_SMTP_AUTH_PASSWORD}
smtp_require_tls: ${ALERTMANAGER_SMTP_REQUIRE_TLS}
smtp_hello: ${ALERTMANAGER_SMTP_HELLO}
templates:
- /etc/alertmanager/email.tmpl
route:
group_wait: 5s
group_interval: 5s
repeat_interval: 5m
receiver: email
receivers:
- name: email
email_configs:
- to: ${ALERTMANAGER_EMAIL_TO}
html: '{{ template "email.to.html" . }}'
headers: { Subject: "[OPENIM-SERVER]Alarm" }
send_resolved: true

@ -225,7 +225,7 @@ PROMETHEUS_PORT=${PROMETHEUS_PORT}
GRAFANA_ADDRESS=${GRAFANA_NETWORK_ADDRESS}
# Port on which Grafana service is running.
# Default: GRAFANA_PORT=3000
# Default: GRAFANA_PORT=13000
GRAFANA_PORT=${GRAFANA_PORT}
# ======================================
@ -303,7 +303,7 @@ NODE_EXPORTER_PORT=${NODE_EXPORTER_PORT}
PROMETHEUS_PORT=${PROMETHEUS_PORT}
# Port for the grafana.
# Default: GRAFANA_PORT=3000
# Default: GRAFANA_PORT=13000
GRAFANA_PORT=${GRAFANA_PORT}
# Port for the admin front.

@ -12,7 +12,7 @@ alerting:
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
- "instanceDown_rules.yml"
- "instance-down-rules.yml"
# - "first_rules.yml"
# - "second_rules.yml"

@ -100,6 +100,7 @@ services:
- KAFKA_CFG_NODE_ID=0
- KAFKA_CFG_PROCESS_ROLES=controller,broker
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@<your_host>:9093
- KAFKA_HEAP_OPTS:"-Xmx256m -Xms256m"
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://${DOCKER_BRIDGE_GATEWAY}:${KAFKA_PORT}
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
@ -162,7 +163,7 @@ services:
restart: always
volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
- ./config/instanceDown_rules.yml:/etc/prometheus/instanceDown_rules.yml
- ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml
ports:
- "${PROMETHEUS_PORT}:9090"
networks:

@ -150,7 +150,7 @@ For convenience, configuration through modifying environment variables is recomm
+ **Description**: API address.
+ **Note**: If the server has an external IP, it will be automatically obtained. For internal networks, set this variable to the IP serving internally.
```
```bash
export API_URL="http://ip:10002"
```
@ -412,7 +412,7 @@ Configuration for Grafana, including its port and address.
| Parameter | Example Value | Description |
| --------------- | -------------------------- | --------------------- |
| GRAFANA_PORT | "3000" | Port used by Grafana. |
| GRAFANA_PORT | "13000" | Port used by Grafana. |
| GRAFANA_ADDRESS | "${DOCKER_BRIDGE_GATEWAY}" | Address for Grafana. |
### 2.16. <a name='RPCPortConfigurationVariables'></a>RPC Port Configuration Variables

@ -4,15 +4,42 @@
* 1. [Source Code & Docker](#SourceCodeDocker)
* 1.1. [Deployment](#Deployment)
* 1.2. [Configuration](#Configuration)
* 1.3. [Monitoring Running in Docker Guide](#MonitoringRunninginDockerGuide)
* 1.3.1. [Introduction](#Introduction)
* 1.3.2. [Prerequisites](#Prerequisites)
* 1.3.3. [Step 1: Clone the Repository](#Step1:ClonetheRepository)
* 1.3.4. [Step 2: Start Docker Compose](#Step2:StartDockerCompose)
* 1.3.5. [Step 3: Use the OpenIM Web Interface](#Step3:UsetheOpenIMWebInterface)
* 1.3.6. [Running Effect](#RunningEffect)
* 1.3.7. [Step 4: Access the Admin Panel](#Step4:AccesstheAdminPanel)
* 1.3.8. [Step 5: Access the Monitoring Interface](#Step5:AccesstheMonitoringInterface)
* 1.3.9. [Next Steps](#NextSteps)
* 1.3.10. [Troubleshooting](#Troubleshooting)
* 2. [Kubernetes](#Kubernetes)
* 2.1. [Middleware Monitoring](#MiddlewareMonitoring)
* 2.2. [Custom OpenIM Metrics](#CustomOpenIMMetrics)
* 2.3. [Node Exporter](#NodeExporter)
* 3. [Setting Up and Configuring AlertManager Using Environment Variables and `make init`](#SettingUpandConfiguringAlertManagerUsingEnvironmentVariablesandmakeinit)
* 3.1. [Introduction](#Introduction-1)
* 3.2. [Prerequisites](#Prerequisites-1)
* 3.3. [Configuration Steps](#ConfigurationSteps)
* 3.3.1. [Exporting Environment Variables](#ExportingEnvironmentVariables)
* 3.3.2. [Initializing AlertManager](#InitializingAlertManager)
* 3.3.3. [Key Configuration Fields](#KeyConfigurationFields)
* 3.3.4. [Configuring SMTP Authentication Password](#ConfiguringSMTPAuthenticationPassword)
* 3.3.5. [Useful Links for Common Email Servers](#UsefulLinksforCommonEmailServers)
* 3.4. [Conclusion](#Conclusion)
<!-- vscode-markdown-toc-config
numbering=true
autoSave=true
/vscode-markdown-toc-config -->
<!-- /vscode-markdown-toc -->
OpenIM offers various flexible deployment options to suit different environments and requirements. Here is a simplified and optimized description of these deployment options:
1. Source Code Deployment:
+ **Regular Source Code Deployment**: Deployment using the `nohup` method. This is a basic deployment method suitable for development and testing environments. For details, refer to the [Regular Source Code Deployment Guide](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment).
+ **Regular Source Code Deployment**: Deployment using the `nohup` method. This is a basic deployment method suitable for development and testing environments. For details, refer to the [Regular Source Code Deployment Guide](https://docs.openim.io/).
+ **Production-Level Deployment**: Deployment using the `system` method, more suitable for production environments. This method provides higher stability and reliability. For details, refer to the [Production-Level Deployment Guide](https://docs.openim.io/guides/gettingStarted/install-openim-linux-system).
2. Cluster Deployment:
+ **Kubernetes Deployment**: Provides two deployment methods, including deployment through Helm and sealos. This is suitable for environments that require high availability and scalability. Specific methods can be found in the [Kubernetes Deployment Guide](https://docs.openim.io/guides/gettingStarted/k8s-deployment).
@ -22,9 +49,9 @@ OpenIM offers various flexible deployment options to suit different environments
Next, we will introduce the specific steps, monitoring, and management backend configuration for each of these deployment methods, as well as usage tips to help you choose the most suitable deployment option according to your needs.
## Source Code & Docker
## 1. <a name='SourceCodeDocker'></a>Source Code & Docker
### Deployment
### 1.1. <a name='Deployment'></a>Deployment
OpenIM deploys openim-server and openim-chat from source code, while other components are deployed via Docker.
@ -43,11 +70,11 @@ make init
docker compose up -d
```
### Configuration
### 1.2. <a name='Configuration'></a>Configuration
To configure Prometheus data sources in Grafana, follow these steps:
1. **Log in to Grafana**: First, open your web browser and access the Grafana URL. If you haven't changed the port, the address is typically [http://localhost:3000](http://localhost:3000/).
1. **Log in to Grafana**: First, open your web browser and access the Grafana URL. If you haven't changed the port, the address is typically [http://localhost:13000](http://localhost:13000/).
2. **Log in with default credentials**: Grafana's default username and password are both `admin`. You will be prompted to change the password on your first login.
@ -135,18 +162,18 @@ To monitor OpenIM in Grafana, you need to focus on three categories of key metri
### Monitoring Running in Docker Guide
### 1.3. <a name='MonitoringRunninginDockerGuide'></a>Monitoring Running in Docker Guide
#### Introduction
#### 1.3.1. <a name='Introduction'></a>Introduction
This guide provides the steps to run OpenIM using Docker. OpenIM is an open-source instant messaging solution that can be quickly deployed using Docker. For more information, please refer to the [OpenIM Docker GitHub](https://github.com/openimsdk/openim-docker).
#### Prerequisites
#### 1.3.2. <a name='Prerequisites'></a>Prerequisites
+ Ensure that Docker and Docker Compose are installed.
+ Basic understanding of Docker and containerization technology.
#### Step 1: Clone the Repository
#### 1.3.3. <a name='Step1:ClonetheRepository'></a>Step 1: Clone the Repository
First, clone the OpenIM Docker repository:
@ -156,7 +183,7 @@ git clone https://github.com/openimsdk/openim-docker.git
Navigate to the repository directory and check the `README` file for more information and configuration options.
#### Step 2: Start Docker Compose
#### 1.3.4. <a name='Step2:StartDockerCompose'></a>Step 2: Start Docker Compose
In the repository directory, run the following command to start the service:
@ -166,17 +193,17 @@ docker-compose up -d
This will download the required Docker images and start the OpenIM service.
#### Step 3: Use the OpenIM Web Interface
#### 1.3.5. <a name='Step3:UsetheOpenIMWebInterface'></a>Step 3: Use the OpenIM Web Interface
+ Open a browser in private mode and access [OpenIM Web](http://localhost:11001/).
+ Register two users and try adding friends.
+ Test sending messages and pictures.
#### Running Effect
#### 1.3.6. <a name='RunningEffect'></a>Running Effect
![image-20231115100811208](http://sm.nsddd.top/sm202311151008639.png)
#### Step 4: Access the Admin Panel
#### 1.3.7. <a name='Step4:AccesstheAdminPanel'></a>Step 4: Access the Admin Panel
+ Access the [OpenIM Admin Panel](http://localhost:11002/).
+ Log in using the default username and password (`admin1:admin1`).
@ -185,29 +212,29 @@ Running Effect Image:
![image-20231115101039837](http://sm.nsddd.top/sm202311151010116.png)
#### Step 5: Access the Monitoring Interface
#### 1.3.8. <a name='Step5:AccesstheMonitoringInterface'></a>Step 5: Access the Monitoring Interface
+ Log in to the [Monitoring Interface](http://localhost:3000/login) using the credentials (`admin:admin`).
#### Next Steps
#### 1.3.9. <a name='NextSteps'></a>Next Steps
+ Configure and manage the services following the steps provided in the OpenIM source code.
+ Refer to the `README` file for advanced configuration and management.
#### Troubleshooting
#### 1.3.10. <a name='Troubleshooting'></a>Troubleshooting
+ If you encounter any issues, please check the documentation on [OpenIM Docker GitHub](https://github.com/openimsdk/openim-docker) or search for related issues in the Issues section.
+ If the problem persists, you can create an issue on the [openim-docker](https://github.com/openimsdk/openim-docker/issues/new/choose) repository or the [openim-server](https://github.com/openimsdk/open-im-server/issues/new/choose) repository.
## Kubernetes
## 2. <a name='Kubernetes'></a>Kubernetes
Refer to [openimsdk/helm-charts](https://github.com/openimsdk/helm-charts).
When deploying and monitoring OpenIM in a Kubernetes environment, you will focus on three main metrics: middleware, custom OpenIM metrics, and Node Exporter. Here are detailed steps and guidelines:
### Middleware Monitoring
### 2.1. <a name='MiddlewareMonitoring'></a>Middleware Monitoring
Middleware monitoring is crucial to ensure the overall system's stability. Typically, this includes monitoring the following components:
@ -219,16 +246,78 @@ Middleware monitoring is crucial to ensure the overall system's stability. Typic
For Kubernetes environments, you can use the corresponding Prometheus Exporters to collect monitoring data for these middleware components.
### Custom OpenIM Metrics
### 2.2. <a name='CustomOpenIMMetrics'></a>Custom OpenIM Metrics
Custom OpenIM metrics provide essential information about the OpenIM application itself, such as user activity, message traffic, system performance, and more. To monitor these metrics in Kubernetes:
+ Ensure OpenIM application configurations expose Prometheus metrics.
+ When deploying using Helm charts (refer to [OpenIM Helm Charts](https://github.com/openimsdk/helm-charts)), pay attention to configuring relevant monitoring settings.
### Node Exporter
### 2.3. <a name='NodeExporter'></a>Node Exporter
Node Exporter is used to collect hardware and operating system-level metrics for Kubernetes nodes, such as CPU, memory, disk usage, and more. To integrate Node Exporter in Kubernetes:
+ Deploy Node Exporter using the appropriate Helm chart. You can find information and guides on [Prometheus Community](https://prometheus.io/docs/guides/node-exporter/).
+ Ensure Node Exporter's data is collected by Prometheus instances within your cluster.
## 3. <a name='SettingUpandConfiguringAlertManagerUsingEnvironmentVariablesandmakeinit'></a>Setting Up and Configuring AlertManager Using Environment Variables and `make init`
### 3.1. <a name='Introduction-1'></a>Introduction
AlertManager, a component of the Prometheus monitoring system, handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver. This document outlines how to set up and configure AlertManager using environment variables and the `make init` command. We will focus on configuring key fields like the sender's email, SMTP settings, and SMTP authentication password.
### 3.2. <a name='Prerequisites-1'></a>Prerequisites
+ Basic knowledge of terminal and command-line operations.
+ AlertManager installed on your system.
+ Access to an SMTP server for sending emails.
### 3.3. <a name='ConfigurationSteps'></a>Configuration Steps
#### 3.3.1. <a name='ExportingEnvironmentVariables'></a>Exporting Environment Variables
Before initializing AlertManager, you need to set environment variables. These variables are used to configure the AlertManager settings without altering the code. Use the `export` command in your terminal. Here are some key variables you might set:
+ `export ALERTMANAGER_RESOLVE_TIMEOUT='5m'`
+ `export ALERTMANAGER_SMTP_FROM='alert@example.com'`
+ `export ALERTMANAGER_SMTP_SMARTHOST='smtp.example.com:465'`
+ `export ALERTMANAGER_SMTP_AUTH_USERNAME='alert@example.com'`
+ `export ALERTMANAGER_SMTP_AUTH_PASSWORD='your_password'`
+ `export ALERTMANAGER_SMTP_REQUIRE_TLS='false'`
#### 3.3.2. <a name='InitializingAlertManager'></a>Initializing AlertManager
After setting the necessary environment variables, you can initialize AlertManager by running the `make init` command. This command typically runs a script that prepares AlertManager with the provided configuration.
#### 3.3.3. <a name='KeyConfigurationFields'></a>Key Configuration Fields
##### a. Sender's Email (`ALERTMANAGER_SMTP_FROM`)
This variable sets the email address that will appear as the sender in the notifications sent by AlertManager.
##### b. SMTP Configuration
+ **SMTP Server (`ALERTMANAGER_SMTP_SMARTHOST`):** Specifies the address and port of the SMTP server used for sending emails.
+ **SMTP Authentication Username (`ALERTMANAGER_SMTP_AUTH_USERNAME`):** The username for authenticating with the SMTP server.
+ **SMTP Authentication Password (`ALERTMANAGER_SMTP_AUTH_PASSWORD`):** The password for SMTP server authentication. It's crucial to keep this value secure.
#### 3.3.4. <a name='ConfiguringSMTPAuthenticationPassword'></a>Configuring SMTP Authentication Password
The SMTP authentication password can be set using the `ALERTMANAGER_SMTP_AUTH_PASSWORD` environment variable. It's recommended to use a secure method to set this variable to avoid exposing sensitive information. For instance, you might read the password from a secure file or a secret management tool.
#### 3.3.5. <a name='UsefulLinksforCommonEmailServers'></a>Useful Links for Common Email Servers
For specific configurations related to common email servers, you may refer to their respective documentation:
+ Gmail SMTP Settings:
+ [Gmail SMTP Configuration](https://support.google.com/mail/answer/7126229?hl=en)
+ Microsoft Outlook SMTP Settings:
+ [Outlook Email Settings](https://support.microsoft.com/en-us/office/pop-imap-and-smtp-settings-8361e398-8af4-4e97-b147-6c6c4ac95353)
+ Yahoo Mail SMTP Settings:
+ [Yahoo SMTP Configuration](https://help.yahoo.com/kb/SLN4724.html)
### 3.4. <a name='Conclusion'></a>Conclusion
Setting up and configuring AlertManager with environment variables provides a flexible and secure way to manage alert settings. By following the above steps, you can easily configure AlertManager for your monitoring needs. Always ensure to secure sensitive information, especially when dealing with SMTP authentication credentials.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 KiB

After

Width:  |  Height:  |  Size: 118 KiB

@ -18,8 +18,9 @@ import (
"context"
"encoding/json"
"errors"
"sync"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"golang.org/x/sync/errgroup"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/conversation"
@ -40,6 +41,7 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"
"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
)
@ -285,18 +287,44 @@ func (p *Pusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData,
if err != nil {
return nil, err
}
var (
mu sync.Mutex
wg = errgroup.Group{}
input = &msggateway.OnlineBatchPushOneMsgReq{MsgData: msg, PushToUserIDs: pushToUserIDs}
maxWorkers = config.Config.Push.MaxConcurrentWorkers
)
if maxWorkers < 3 {
maxWorkers = 3
}
wg.SetLimit(maxWorkers)
// Online push message
for _, v := range conns {
msgClient := msggateway.NewMsgGatewayClient(v)
reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, &msggateway.OnlineBatchPushOneMsgReq{MsgData: msg, PushToUserIDs: pushToUserIDs})
if err != nil {
continue
}
log.ZDebug(ctx, "push result", "reply", reply)
if reply != nil && reply.SinglePushResult != nil {
wsResults = append(wsResults, reply.SinglePushResult...)
}
for _, conn := range conns {
conn := conn // loop var safe
wg.Go(func() error {
msgClient := msggateway.NewMsgGatewayClient(conn)
reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input)
if err != nil {
return nil
}
log.ZDebug(ctx, "push result", "reply", reply)
if reply != nil && reply.SinglePushResult != nil {
mu.Lock()
wsResults = append(wsResults, reply.SinglePushResult...)
mu.Unlock()
}
return nil
})
}
_ = wg.Wait()
// always return nil
return wsResults, nil
}

@ -199,8 +199,9 @@ type configStruct struct {
} `yaml:"longConnSvr"`
Push struct {
Enable string `yaml:"enable"`
GeTui struct {
MaxConcurrentWorkers int `yaml:"maxConcurrentWorkers"`
Enable string `yaml:"enable"`
GeTui struct {
PushUrl string `yaml:"pushUrl"`
AppKey string `yaml:"appKey"`
Intent string `yaml:"intent"`

@ -20,11 +20,8 @@ import (
"strconv"
"time"
"github.com/dtm-labs/rockscache"
"golang.org/x/sync/errgroup"
unrelationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/OpenIMSDK/tools/errs"
@ -136,10 +133,7 @@ func NewMsgCacheModel(client redis.UniversalClient) MsgModel {
type msgCache struct {
metaCache
rdb redis.UniversalClient
expireTime time.Duration
rcClient *rockscache.Client
msgDocDatabase unrelationtb.MsgDocModelInterface
rdb redis.UniversalClient
}
func (c *msgCache) getMaxSeqKey(conversationID string) string {
@ -176,29 +170,6 @@ func (c *msgCache) getSeqs(ctx context.Context, items []string, getkey func(s st
}
return m, nil
//pipe := c.rdb.Pipeline()
//for _, v := range items {
// if err := pipe.Get(ctx, getkey(v)).Err(); err != nil && err != redis.Nil {
// return nil, errs.Wrap(err)
// }
//}
//result, err := pipe.Exec(ctx)
//if err != nil && err != redis.Nil {
// return nil, errs.Wrap(err)
//}
//m = make(map[string]int64, len(items))
//for i, v := range result {
// seq := v.(*redis.StringCmd)
// if seq.Err() != nil && seq.Err() != redis.Nil {
// return nil, errs.Wrap(v.Err())
// }
// val := utils.StringToInt64(seq.Val())
// if val != 0 {
// m[items[i]] = val
// }
//}
//return m, nil
}
func (c *msgCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error {
@ -224,15 +195,6 @@ func (c *msgCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey fu
}
}
return nil
//pipe := c.rdb.Pipeline()
//for k, seq := range seqs {
// err := pipe.Set(ctx, getkey(k), seq, 0).Err()
// if err != nil {
// return errs.Wrap(err)
// }
//}
//_, err := pipe.Exec(ctx)
//return err
}
func (c *msgCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error {
@ -637,20 +599,49 @@ func (c *msgCache) DelUserDeleteMsgsList(ctx context.Context, conversationID str
}
func (c *msgCache) DeleteMessages(ctx context.Context, conversationID string, seqs []int64) error {
if config.Config.Redis.EnablePipeline {
return c.PipeDeleteMessages(ctx, conversationID, seqs)
}
return c.ParallelDeleteMessages(ctx, conversationID, seqs)
}
func (c *msgCache) ParallelDeleteMessages(ctx context.Context, conversationID string, seqs []int64) error {
wg := errgroup.Group{}
wg.SetLimit(concurrentLimit)
for _, seq := range seqs {
if err := c.rdb.Del(ctx, c.getMessageCacheKey(conversationID, seq)).Err(); err != nil {
seq := seq
wg.Go(func() error {
err := c.rdb.Del(ctx, c.getMessageCacheKey(conversationID, seq)).Err()
if err != nil {
return errs.Wrap(err)
}
return nil
})
}
return wg.Wait()
}
func (c *msgCache) PipeDeleteMessages(ctx context.Context, conversationID string, seqs []int64) error {
pipe := c.rdb.Pipeline()
for _, seq := range seqs {
_ = pipe.Del(ctx, c.getMessageCacheKey(conversationID, seq))
}
results, err := pipe.Exec(ctx)
if err != nil {
return errs.Wrap(err, "pipe.del")
}
for _, res := range results {
if res.Err() != nil {
return errs.Wrap(err)
}
}
return nil
//pipe := c.rdb.Pipeline()
//for _, seq := range seqs {
// if err := pipe.Del(ctx, c.getMessageCacheKey(conversationID, seq)).Err(); err != nil {
// return errs.Wrap(err)
// }
//}
//_, err := pipe.Exec(ctx)
//return errs.Wrap(err)
}
func (c *msgCache) CleanUpOneConversationAllMsg(ctx context.Context, conversationID string) error {
@ -667,14 +658,6 @@ func (c *msgCache) CleanUpOneConversationAllMsg(ctx context.Context, conversatio
}
}
return nil
//pipe := c.rdb.Pipeline()
//for _, v := range vals {
// if err := pipe.Del(ctx, v).Err(); err != nil {
// return errs.Wrap(err)
// }
//}
//_, err = pipe.Exec(ctx)
//return errs.Wrap(err)
}
func (c *msgCache) DelMsgFromCache(ctx context.Context, userID string, seqs []int64) error {

@ -249,3 +249,139 @@ func testPipeGetMessagesBySeqWithLostHalfSeqs(t *testing.T, cid string, seqs []i
assert.Equal(t, msg.Seq, seqs[idx])
}
}
func TestPipeDeleteMessages(t *testing.T) {
var (
cid = fmt.Sprintf("cid-%v", rand.Int63())
seqFirst = rand.Int63()
msgs = []*sdkws.MsgData{}
)
var seqs []int64
for i := 0; i < 100; i++ {
msgs = append(msgs, &sdkws.MsgData{
Seq: seqFirst + int64(i),
})
seqs = append(seqs, msgs[i].Seq)
}
testPipeSetMessageToCache(t, cid, msgs)
testPipeDeleteMessagesOK(t, cid, seqs, msgs)
// set again
testPipeSetMessageToCache(t, cid, msgs)
testPipeDeleteMessagesMix(t, cid, seqs[:90], msgs)
}
func testPipeDeleteMessagesOK(t *testing.T, cid string, seqs []int64, inputMsgs []*sdkws.MsgData) {
rdb := redis.NewClient(&redis.Options{})
defer rdb.Close()
cacher := msgCache{rdb: rdb}
err := cacher.PipeDeleteMessages(context.Background(), cid, seqs)
assert.Nil(t, err)
// validate
for _, msg := range inputMsgs {
key := cacher.getMessageCacheKey(cid, msg.Seq)
val := rdb.Exists(context.Background(), key).Val()
assert.EqualValues(t, 0, val)
}
}
func testPipeDeleteMessagesMix(t *testing.T, cid string, seqs []int64, inputMsgs []*sdkws.MsgData) {
rdb := redis.NewClient(&redis.Options{})
defer rdb.Close()
cacher := msgCache{rdb: rdb}
err := cacher.PipeDeleteMessages(context.Background(), cid, seqs)
assert.Nil(t, err)
// validate
for idx, msg := range inputMsgs {
key := cacher.getMessageCacheKey(cid, msg.Seq)
val, err := rdb.Exists(context.Background(), key).Result()
assert.Nil(t, err)
if idx < 90 {
assert.EqualValues(t, 0, val) // not exists
continue
}
assert.EqualValues(t, 1, val) // exists
}
}
func TestParallelDeleteMessages(t *testing.T) {
var (
cid = fmt.Sprintf("cid-%v", rand.Int63())
seqFirst = rand.Int63()
msgs = []*sdkws.MsgData{}
)
var seqs []int64
for i := 0; i < 100; i++ {
msgs = append(msgs, &sdkws.MsgData{
Seq: seqFirst + int64(i),
})
seqs = append(seqs, msgs[i].Seq)
}
randSeqs := []int64{}
for i := seqFirst + 100; i < seqFirst+200; i++ {
randSeqs = append(randSeqs, i)
}
testParallelSetMessageToCache(t, cid, msgs)
testParallelDeleteMessagesOK(t, cid, seqs, msgs)
// set again
testParallelSetMessageToCache(t, cid, msgs)
testParallelDeleteMessagesMix(t, cid, seqs[:90], msgs, 90)
testParallelDeleteMessagesOK(t, cid, seqs[90:], msgs[:90])
// set again
testParallelSetMessageToCache(t, cid, msgs)
testParallelDeleteMessagesMix(t, cid, randSeqs, msgs, 0)
}
func testParallelDeleteMessagesOK(t *testing.T, cid string, seqs []int64, inputMsgs []*sdkws.MsgData) {
rdb := redis.NewClient(&redis.Options{})
defer rdb.Close()
cacher := msgCache{rdb: rdb}
err := cacher.PipeDeleteMessages(context.Background(), cid, seqs)
assert.Nil(t, err)
// validate
for _, msg := range inputMsgs {
key := cacher.getMessageCacheKey(cid, msg.Seq)
val := rdb.Exists(context.Background(), key).Val()
assert.EqualValues(t, 0, val)
}
}
func testParallelDeleteMessagesMix(t *testing.T, cid string, seqs []int64, inputMsgs []*sdkws.MsgData, lessValNonExists int) {
rdb := redis.NewClient(&redis.Options{})
defer rdb.Close()
cacher := msgCache{rdb: rdb}
err := cacher.PipeDeleteMessages(context.Background(), cid, seqs)
assert.Nil(t, err)
// validate
for idx, msg := range inputMsgs {
key := cacher.getMessageCacheKey(cid, msg.Seq)
val, err := rdb.Exists(context.Background(), key).Result()
assert.Nil(t, err)
if idx < lessValNonExists {
assert.EqualValues(t, 0, val) // not exists
continue
}
assert.EqualValues(t, 1, val) // exists
}
}

@ -32,6 +32,7 @@ declare -A TEMPLATES=(
["${OPENIM_ROOT}/deployments/templates/env_template.yaml"]="${OPENIM_ROOT}/.env"
["${OPENIM_ROOT}/deployments/templates/openim.yaml"]="${OPENIM_ROOT}/config/config.yaml"
["${OPENIM_ROOT}/deployments/templates/prometheus.yml"]="${OPENIM_ROOT}/config/prometheus.yml"
["${OPENIM_ROOT}/deployments/templates/alertmanager.yml"]="${OPENIM_ROOT}/config/alertmanager.yml"
)
for template in "${!TEMPLATES[@]}"; do

@ -266,8 +266,30 @@ def "NODE_EXPORTER_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # node-exporter的地址
def "ALERT_MANAGER_PORT" "19093" # node-exporter的端口
def "ALERT_MANAGER_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # node-exporter的地址
###################### AlertManager Configuration Script ######################
# 解析超时
readonly ALERTMANAGER_RESOLVE_TIMEOUT=${ALERTMANAGER_RESOLVE_TIMEOUT:-'5m'}
# 发件人邮箱
readonly ALERTMANAGER_SMTP_FROM=${ALERTMANAGER_SMTP_FROM:-'alert@openim.io'}
# SMTP服务器地址和端口
readonly ALERTMANAGER_SMTP_SMARTHOST=${ALERTMANAGER_SMTP_SMARTHOST:-'smtp.163.com:465'}
# SMTP认证用户名
readonly ALERTMANAGER_SMTP_AUTH_USERNAME=${SMTP_USERNAME:-"alert@openim.io"}
# SMTP认证密码
readonly ALERTMANAGER_SMTP_AUTH_PASSWORD=${SMTP_PASSWORD:-"YOURAUTHPASSWORD"}
# SMTP是否需要TLS
readonly ALERTMANAGER_SMTP_REQUIRE_TLS=${ALERTMANAGER_SMTP_REQUIRE_TLS:-"false"}
# SMTP HELO/EHLO标识符
readonly ALERTMANAGER_SMTP_HELLO=${ALERTMANAGER_SMTP_HELLO:-"xxx监控告警"}
# 邮箱接收人
readonly ALERTMANAGER_EMAIL_TO=${ALERTMANAGER_EMAIL_TO:-"{EMAIL_TO:-'alert@example.com'}"}
# 邮箱主题
readonly ALERTMANAGER_EMAIL_SUBJECT=${ALERTMANAGER_EMAIL_SUBJECT:-"{EMAIL_SUBJECT:-'[Alert] Notification'}"}
# 是否发送已解决的告警
readonly ALERTMANAGER_SEND_RESOLVED=${ALERTMANAGER_SEND_RESOLVED:-"{SEND_RESOLVED:-'true'}"}
###################### Grafana 配置信息 ######################
def "GRAFANA_PORT" "3000" # Grafana的端口
def "GRAFANA_PORT" "13000" # Grafana的端口
def "GRAFANA_ADDRESS" "${DOCKER_BRIDGE_GATEWAY}" # Grafana的地址
###################### RPC Port Configuration Variables ######################

@ -22,6 +22,8 @@
# It can be downloaded from the following link:
# https://github.com/OpenIMSDK/Open-IM-Protoc/releases/tag/v1.0.0
#
# About the tool:
# https://github.com/openimsdk/open-im-server/blob/main/docs/contrib/protoc-tools.md
# Download link (Windows): https://github.com/OpenIMSDK/Open-IM-Protoc/releases/download/v1.0.0/windows.zip
# Download link (Linux): https://github.com/OpenIMSDK/Open-IM-Protoc/releases/download/v1.0.0/linux.zip
#

Loading…
Cancel
Save