From e0284724b5e4dafef311e1e513a90e8615d4c4f4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 18 Nov 2024 11:32:47 +0800 Subject: [PATCH] build: update mongo and kafka start logic. (#2858) * build: update mongo and kafka start logic. * build: update go version image in dockerfile. * build: remove zookeeper image. * add authSource comment. * update tools version. * add created sucess print. * remove unused script. * format. --- .env | 4 +-- Dockerfile | 6 ++-- config/mongodb.yml | 2 ++ docker-compose.yml | 36 ++++++++++++++----- go.mod | 8 +++-- go.sum | 12 +++---- pkg/common/config/config.go | 2 ++ scripts/create-topic.sh | 55 ----------------------------- scripts/mongo-init.sh | 66 ----------------------------------- tools/check-component/main.go | 2 +- 10 files changed, 48 insertions(+), 145 deletions(-) delete mode 100755 scripts/create-topic.sh delete mode 100755 scripts/mongo-init.sh diff --git a/.env b/.env index 0c02bf9f6..2b903c949 100644 --- a/.env +++ b/.env @@ -1,6 +1,5 @@ -MONGO_IMAGE=mongo:6.0.2 +MONGO_IMAGE=mongo:7.0 REDIS_IMAGE=redis:7.0.0 -ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 @@ -16,4 +15,3 @@ OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.2 #OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.2 DATA_DIR=./ - diff --git a/Dockerfile b/Dockerfile index f8cfbda9e..4b38d711b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Use Go 1.21 Alpine as the base image for building the application -FROM golang:1.21-alpine AS builder +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder # Define the base directory for the application as an environment variable ENV SERVER_DIR=/openim-server @@ -22,7 +22,7 @@ RUN go install github.com/magefile/mage@v1.15.0 RUN mage build # Using Alpine Linux with Go environment for the final image -FROM golang:1.21-alpine +FROM golang:1.22-alpine # Install necessary packages, such as bash RUN apk add --no-cache bash diff --git a/config/mongodb.yml b/config/mongodb.yml index 78f85992c..072cb4b8f 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -8,6 +8,8 @@ database: openim_v3 username: openIM # Password for database authentication password: openIM123 +# Authentication source for database authentication, if use root user, set it to admin +authSource: openim_v3 # Maximum number of connections in the connection pool maxPoolSize: 100 # Maximum number of retry attempts for a failed database connection diff --git a/docker-compose.yml b/docker-compose.yml index 6d88bac10..8d25383bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,35 @@ services: ports: - "37017:27017" container_name: mongo - command: ["/bin/bash", "-c", "/docker-entrypoint-initdb.d/mongo-init.sh; docker-entrypoint.sh mongod --wiredTigerCacheSizeGB 1 --auth"] + command: > + bash -c ' + docker-entrypoint.sh mongod --wiredTigerCacheSizeGB $$wiredTigerCacheSizeGB --auth & + until mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval "db.runCommand({ ping: 1 })" &>/dev/null; do + echo "Waiting for MongoDB to start..." + sleep 1 + done && + mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval " + db = db.getSiblingDB(\"$$MONGO_INITDB_DATABASE\"); + if (!db.getUser(\"$$MONGO_OPENIM_USERNAME\")) { + db.createUser({ + user: \"$$MONGO_OPENIM_USERNAME\", + pwd: \"$$MONGO_OPENIM_PASSWORD\", + roles: [{role: \"readWrite\", db: \"$$MONGO_INITDB_DATABASE\"}] + }); + print(\"User created successfully: \"); + print(\"Username: $$MONGO_OPENIM_USERNAME\"); + print(\"Password: $$MONGO_OPENIM_PASSWORD\"); + print(\"Database: $$MONGO_INITDB_DATABASE\"); + } else { + print(\"User already exists in database: $$MONGO_INITDB_DATABASE, Username: $$MONGO_OPENIM_USERNAME\"); + } + " && + tail -f /dev/null + ' volumes: - "${DATA_DIR}/components/mongodb/data/db:/data/db" - "${DATA_DIR}/components/mongodb/data/logs:/data/logs" - "${DATA_DIR}/components/mongodb/data/conf:/etc/mongo" - - "./scripts/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh:ro" environment: - TZ=Asia/Shanghai - wiredTigerCacheSizeGB=1 @@ -71,10 +94,7 @@ services: ports: - "19094:9094" volumes: - - ./scripts/create-topic.sh:/opt/bitnami/kafka/create-topic.sh - "${DATA_DIR}/components/kafka:/bitnami/kafka" - command: > - bash -c "/opt/bitnami/scripts/kafka/run.sh & /opt/bitnami/kafka/create-topic.sh; wait" environment: #KAFKA_HEAP_OPTS: "-Xms128m -Xmx256m" TZ: Asia/Shanghai @@ -85,10 +105,11 @@ services: KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_NUM_PARTITIONS: 8 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" networks: - openim - minio: image: "${MINIO_IMAGE}" ports: @@ -124,7 +145,7 @@ services: - "11002:80" networks: - openim - + # prometheus: # image: ${PROMETHEUS_IMAGE} # container_name: prometheus @@ -171,4 +192,3 @@ services: # - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana # networks: # - openim - diff --git a/go.mod b/go.mod index 051beb403..8a05b016c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/openimsdk/open-im-server/v3 -go 1.21.2 +go 1.22.0 + +toolchain go1.23.2 require ( firebase.google.com/go/v4 v4.14.1 @@ -13,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.54 - github.com/openimsdk/tools v0.0.50-alpha.16 + github.com/openimsdk/tools v0.0.50-alpha.32 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -92,7 +94,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect diff --git a/go.sum b/go.sum index 816c82094..3cae53b0d 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s= github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= -github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw= +github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -356,8 +356,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da6c63d60..10285b565 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -69,6 +69,7 @@ type Mongo struct { Database string `mapstructure:"database"` Username string `mapstructure:"username"` Password string `mapstructure:"password"` + AuthSource string `mapstructure:"authSource"` MaxPoolSize int `mapstructure:"maxPoolSize"` MaxRetry int `mapstructure:"maxRetry"` } @@ -490,6 +491,7 @@ func (m *Mongo) Build() *mongoutil.Config { Database: m.Database, Username: m.Username, Password: m.Password, + AuthSource: m.AuthSource, MaxPoolSize: m.MaxPoolSize, MaxRetry: m.MaxRetry, } diff --git a/scripts/create-topic.sh b/scripts/create-topic.sh deleted file mode 100755 index bbc739287..000000000 --- a/scripts/create-topic.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Wait for Kafka to be ready - -KAFKA_SERVER=localhost:9092 - -MAX_ATTEMPTS=300 -attempt_num=1 - -echo "Waiting for Kafka to be ready..." - -until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server $KAFKA_SERVER; do - echo "Attempt $attempt_num of $MAX_ATTEMPTS: Kafka not ready yet..." - if [ $attempt_num -eq $MAX_ATTEMPTS ]; then - echo "Kafka not ready after $MAX_ATTEMPTS attempts, exiting" - exit 1 - fi - attempt_num=$((attempt_num+1)) - sleep 1 -done - -echo "Kafka is ready. Creating topics..." - - -topics=("toRedis" "toMongo" "toPush" "toOfflinePush") -partitions=8 -replicationFactor=1 - -for topic in "${topics[@]}"; do - if /opt/bitnami/kafka/bin/kafka-topics.sh --create \ - --bootstrap-server $KAFKA_SERVER \ - --replication-factor $replicationFactor \ - --partitions $partitions \ - --topic $topic - then - echo "Topic $topic created." - else - echo "Failed to create topic $topic." - fi -done - -echo "All topics created." diff --git a/scripts/mongo-init.sh b/scripts/mongo-init.sh deleted file mode 100755 index 25bb2d654..000000000 --- a/scripts/mongo-init.sh +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -mongosh < 0) { - try { - db = connect('mongodb://127.0.0.1:27017/admin'); - var authResult = db.auth(rootUsername, rootPassword); - if (authResult) { - print('Authentication successful for root user: ' + rootUsername); - connected = true; - } else { - print('Authentication failed for root user: ' + rootUsername + ' with password: ' + rootPassword); - quit(1); - } - } catch (e) { - maxRetries--; - print('Connection failed, retrying... Remaining attempts: ' + maxRetries); - sleep(1000); // Sleep for 1 second - } -} - -if (connected) { - db = db.getSiblingDB(dbName); - var createUserResult = db.createUser({ - user: openimUsername, - pwd: openimPassword, - roles: [{ - role: 'readWrite', - db: dbName - }] - }); - - if (createUserResult.ok == 1) { - print('User creation successful. User: ' + openimUsername + ', Database: ' + dbName); - } else { - print('User creation failed for user: ' + openimUsername + ' in database: ' + dbName); - quit(1); - } -} else { - print('Failed to connect to MongoDB after 300 retries.'); - quit(1); -} -EOF - - - - diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 4f4c08c16..94dbd613c 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -66,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error { } func CheckKafka(ctx context.Context, conf *config.Kafka) error { - return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic}) + return kafka.CheckHealth(ctx, conf.Build()) } func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) {