build: update mongo and kafka start logic. (#2858)

* build: update mongo and kafka start logic.

* build: update go version image in dockerfile.

* build: remove zookeeper image.

* add authSource comment.

* update tools version.

* add created sucess print.

* remove unused script.

* format.
pull/2861/head
Monet Lee 1 month ago committed by GitHub
parent 12790e141d
commit e0284724b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -1,6 +1,5 @@
MONGO_IMAGE=mongo:6.0.2
MONGO_IMAGE=mongo:7.0
REDIS_IMAGE=redis:7.0.0
ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8
KAFKA_IMAGE=bitnami/kafka:3.5.1
MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z
ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13
@ -16,4 +15,3 @@ OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.2
#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.2
DATA_DIR=./

@ -1,5 +1,5 @@
# Use Go 1.21 Alpine as the base image for building the application
FROM golang:1.21-alpine AS builder
# Use Go 1.22 Alpine as the base image for building the application
FROM golang:1.22-alpine AS builder
# Define the base directory for the application as an environment variable
ENV SERVER_DIR=/openim-server
@ -22,7 +22,7 @@ RUN go install github.com/magefile/mage@v1.15.0
RUN mage build
# Using Alpine Linux with Go environment for the final image
FROM golang:1.21-alpine
FROM golang:1.22-alpine
# Install necessary packages, such as bash
RUN apk add --no-cache bash

@ -8,6 +8,8 @@ database: openim_v3
username: openIM
# Password for database authentication
password: openIM123
# Authentication source for database authentication, if use root user, set it to admin
authSource: openim_v3
# Maximum number of connections in the connection pool
maxPoolSize: 100
# Maximum number of retry attempts for a failed database connection

@ -8,12 +8,35 @@ services:
ports:
- "37017:27017"
container_name: mongo
command: ["/bin/bash", "-c", "/docker-entrypoint-initdb.d/mongo-init.sh; docker-entrypoint.sh mongod --wiredTigerCacheSizeGB 1 --auth"]
command: >
bash -c '
docker-entrypoint.sh mongod --wiredTigerCacheSizeGB $$wiredTigerCacheSizeGB --auth &
until mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval "db.runCommand({ ping: 1 })" &>/dev/null; do
echo "Waiting for MongoDB to start..."
sleep 1
done &&
mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval "
db = db.getSiblingDB(\"$$MONGO_INITDB_DATABASE\");
if (!db.getUser(\"$$MONGO_OPENIM_USERNAME\")) {
db.createUser({
user: \"$$MONGO_OPENIM_USERNAME\",
pwd: \"$$MONGO_OPENIM_PASSWORD\",
roles: [{role: \"readWrite\", db: \"$$MONGO_INITDB_DATABASE\"}]
});
print(\"User created successfully: \");
print(\"Username: $$MONGO_OPENIM_USERNAME\");
print(\"Password: $$MONGO_OPENIM_PASSWORD\");
print(\"Database: $$MONGO_INITDB_DATABASE\");
} else {
print(\"User already exists in database: $$MONGO_INITDB_DATABASE, Username: $$MONGO_OPENIM_USERNAME\");
}
" &&
tail -f /dev/null
'
volumes:
- "${DATA_DIR}/components/mongodb/data/db:/data/db"
- "${DATA_DIR}/components/mongodb/data/logs:/data/logs"
- "${DATA_DIR}/components/mongodb/data/conf:/etc/mongo"
- "./scripts/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh:ro"
environment:
- TZ=Asia/Shanghai
- wiredTigerCacheSizeGB=1
@ -71,10 +94,7 @@ services:
ports:
- "19094:9094"
volumes:
- ./scripts/create-topic.sh:/opt/bitnami/kafka/create-topic.sh
- "${DATA_DIR}/components/kafka:/bitnami/kafka"
command: >
bash -c "/opt/bitnami/scripts/kafka/run.sh & /opt/bitnami/kafka/create-topic.sh; wait"
environment:
#KAFKA_HEAP_OPTS: "-Xms128m -Xmx256m"
TZ: Asia/Shanghai
@ -85,10 +105,11 @@ services:
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_NUM_PARTITIONS: 8
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true"
networks:
- openim
minio:
image: "${MINIO_IMAGE}"
ports:
@ -171,4 +192,3 @@ services:
# - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana
# networks:
# - openim

@ -1,6 +1,8 @@
module github.com/openimsdk/open-im-server/v3
go 1.21.2
go 1.22.0
toolchain go1.23.2
require (
firebase.google.com/go/v4 v4.14.1
@ -13,7 +15,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/mitchellh/mapstructure v1.5.0
github.com/openimsdk/protocol v0.0.72-alpha.54
github.com/openimsdk/tools v0.0.50-alpha.16
github.com/openimsdk/tools v0.0.50-alpha.32
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.18.0
github.com/stretchr/testify v1.9.0
@ -92,7 +94,7 @@ require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect

@ -126,8 +126,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF
github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s=
github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc=
github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4=
github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw=
github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
@ -356,8 +356,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

@ -69,6 +69,7 @@ type Mongo struct {
Database string `mapstructure:"database"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
AuthSource string `mapstructure:"authSource"`
MaxPoolSize int `mapstructure:"maxPoolSize"`
MaxRetry int `mapstructure:"maxRetry"`
}
@ -490,6 +491,7 @@ func (m *Mongo) Build() *mongoutil.Config {
Database: m.Database,
Username: m.Username,
Password: m.Password,
AuthSource: m.AuthSource,
MaxPoolSize: m.MaxPoolSize,
MaxRetry: m.MaxRetry,
}

@ -1,55 +0,0 @@
#!/usr/bin/env bash
# Copyright © 2023 OpenIM. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Wait for Kafka to be ready
KAFKA_SERVER=localhost:9092
MAX_ATTEMPTS=300
attempt_num=1
echo "Waiting for Kafka to be ready..."
until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server $KAFKA_SERVER; do
echo "Attempt $attempt_num of $MAX_ATTEMPTS: Kafka not ready yet..."
if [ $attempt_num -eq $MAX_ATTEMPTS ]; then
echo "Kafka not ready after $MAX_ATTEMPTS attempts, exiting"
exit 1
fi
attempt_num=$((attempt_num+1))
sleep 1
done
echo "Kafka is ready. Creating topics..."
topics=("toRedis" "toMongo" "toPush" "toOfflinePush")
partitions=8
replicationFactor=1
for topic in "${topics[@]}"; do
if /opt/bitnami/kafka/bin/kafka-topics.sh --create \
--bootstrap-server $KAFKA_SERVER \
--replication-factor $replicationFactor \
--partitions $partitions \
--topic $topic
then
echo "Topic $topic created."
else
echo "Failed to create topic $topic."
fi
done
echo "All topics created."

@ -1,66 +0,0 @@
# Copyright © 2023 OpenIM. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
mongosh <<EOF
var maxRetries = 300;
var connected = false;
var rootUsername = '$MONGO_INITDB_ROOT_USERNAME';
var rootPassword = '$MONGO_INITDB_ROOT_PASSWORD';
var dbName = '$MONGO_INITDB_DATABASE';
var openimUsername = '$MONGO_OPENIM_USERNAME';
var openimPassword = '$MONGO_OPENIM_PASSWORD';
while (!connected && maxRetries > 0) {
try {
db = connect('mongodb://127.0.0.1:27017/admin');
var authResult = db.auth(rootUsername, rootPassword);
if (authResult) {
print('Authentication successful for root user: ' + rootUsername);
connected = true;
} else {
print('Authentication failed for root user: ' + rootUsername + ' with password: ' + rootPassword);
quit(1);
}
} catch (e) {
maxRetries--;
print('Connection failed, retrying... Remaining attempts: ' + maxRetries);
sleep(1000); // Sleep for 1 second
}
}
if (connected) {
db = db.getSiblingDB(dbName);
var createUserResult = db.createUser({
user: openimUsername,
pwd: openimPassword,
roles: [{
role: 'readWrite',
db: dbName
}]
});
if (createUserResult.ok == 1) {
print('User creation successful. User: ' + openimUsername + ', Database: ' + dbName);
} else {
print('User creation failed for user: ' + openimUsername + ' in database: ' + dbName);
quit(1);
}
} else {
print('Failed to connect to MongoDB after 300 retries.');
quit(1);
}
EOF

@ -66,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error {
}
func CheckKafka(ctx context.Context, conf *config.Kafka) error {
return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic})
return kafka.CheckHealth(ctx, conf.Build())
}
func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) {

Loading…
Cancel
Save