diff --git a/.github/code-language-detector.yml b/.github/code-language-detector.yml index 32a8c1f54..194c2474a 100644 --- a/.github/code-language-detector.yml +++ b/.github/code-language-detector.yml @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# https://github.com/marketplace/actions/code-language-detector directory: ./ file_types: - .go diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 154386ff5..55ee241d7 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -48,4 +48,4 @@ template: | ## Contributors to this $REPOSITORY release - $CONTRIBUTORS \ No newline at end of file + $CONTRIBUTORS diff --git a/.github/standardizer.yml b/.github/standardizer.yml new file mode 100644 index 000000000..fceb69df1 --- /dev/null +++ b/.github/standardizer.yml @@ -0,0 +1,50 @@ +# https://github.com/marketplace/actions/conformity-checker-for-project +baseConfig: + searchDirectory: "./" + ignoreCase: false + +directoryNaming: + allowHyphens: true + allowUnderscores: false + mustBeLowercase: true + +fileNaming: + allowHyphens: true + allowUnderscores: true + mustBeLowercase: true + +ignoreFormats: + - "\\.log$" + - "\\.env$" + - "README\\.md$" + - "_test\\.go$" + - "\\.md$" + - _test\\.txt$ + - LICENSE + - Dockerfile + - CODEOWNERS + - Makefile + +ignoreDirectories: + - "vendor" + - ".git" + - "deployments" + - "node_modules" + - "logs" + - "CHANGELOG" + - "components" + - "_output" + - "tools/openim-web" + - "CHANGELOG" + - "examples/Test_directory" + - test/testdata + +fileTypeSpecificNaming: + ".yaml": + allowHyphens: true + allowUnderscores: false + mustBeLowercase: true + ".go": + allowHyphens: false + allowUnderscores: true + mustBeLowercase: true \ No newline at end of file diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index 5aeddd09a..f47283997 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -73,6 +73,9 @@ jobs: - name: Code Typecheck Detector uses: kubecub/typecheck@main + - name: Conformity Checker for Project + uses: kubecub/standardizer@main + - name: Module Operations run: | sudo make tidy diff --git a/Makefile b/Makefile index 89b9e4152..818372fb8 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ VERSION_PACKAGE=github.com/openimsdk/open-im-server/v3/pkg/version # Includes include scripts/make-rules/common.mk # make sure include common.mk at the first include line +include scripts/make-rules/common-versions.mk include scripts/make-rules/golang.mk include scripts/make-rules/image.mk include scripts/make-rules/copyright.mk diff --git a/README_zh_CN.md b/README_zh_CN.md index 7eabfa509..f42031165 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -52,7 +52,7 @@

## 🟢 扫描微信进群交流 - + ## Ⓜ️ 关于 OpenIM diff --git a/docker-compose-1.yml b/docker-compose-1.yml index 39fc944ce..ed852fd29 100644 --- a/docker-compose-1.yml +++ b/docker-compose-1.yml @@ -1,4 +1,4 @@ -#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/OpenIMSDK/Open-IM-Server.git +#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/openimsdk/open-im-server.git # The command that triggers this file to pull the image is "docker compose up -f" version: '3' diff --git a/docker-compose.yml b/docker-compose.yml index ef0714329..a983acbaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/OpenIMSDK/Open-IM-Server.git +#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/openimsdk/open-im-server.git # The command that triggers this file to pull the image is "docker compose up -d". version: '3' diff --git a/docs/contrib/images.md b/docs/contrib/images.md index d1a83d639..396ffb2e3 100644 --- a/docs/contrib/images.md +++ b/docs/contrib/images.md @@ -96,4 +96,21 @@ When pulling OpenIM's Docker images, you can choose the most suitable source bas 3. Run the `docker images` command to confirm that the image has been successfully pulled. -This concludes OpenIM's image management strategy and the steps for pulling images. If you have any questions, please feel free to ask. +### Accelerating Deployment for Users in China with Aliyun Mirror or Alternative Image Addresses + +For users in China looking to speed up the deployment process of OpenIM, leveraging a mirror image address is a highly recommended practice. After executing the `make init` command, a `.env` file is generated, which you'll need to edit to configure the image registry source. This configuration is crucial for optimizing download speeds and ensuring a smoother setup process. + +Within the generated `.env` file, you'll find a section dedicated to choosing the image address. It includes options for GitHub (`ghcr.io/openimsdk`), Docker Hub (`openim`), and Ali Cloud (`registry.cn-hangzhou.aliyuncs.com/openimsdk`). To achieve the best performance within China, it is advised to use the Aliyun image address. + +To do this, you need to comment out the current `IMAGE_REGISTRY` setting and uncomment the Aliyun option. Here is how you can adjust it for Aliyun: + +```bash +# Choose the image address: GitHub (ghcr.io/openimsdk), Docker Hub (openim), +# or Ali Cloud (registry.cn-hangzhou.aliyuncs.com/openimsdk). +# Uncomment one of the following three options. Aliyun is recommended for users in China. +# IMAGE_REGISTRY="ghcr.io/openimsdk" +# IMAGE_REGISTRY="openim" +IMAGE_REGISTRY="registry.cn-hangzhou.aliyuncs.com/openimsdk" +``` + +This change directs the deployment process to fetch the required images from the Aliyun registry, significantly improving download and installation speeds due to the geographical and network advantages within China. If, for any reason, you prefer not to use Aliyun or encounter issues, consider switching to another mirror address listed in the `.env` file by following the same uncommenting process. This flexibility ensures that users can select the most suitable image source for their specific situation, leading to a more efficient deployment of OpenIM. diff --git a/docs/contrib/kafka.md b/docs/contrib/kafka.md new file mode 100644 index 000000000..4547c9480 --- /dev/null +++ b/docs/contrib/kafka.md @@ -0,0 +1,162 @@ +# OpenIM Kafka Guide + +This document aims to provide a set of concise guidelines to help you quickly install and use Kafka through Docker Compose. + +## Installing Kafka + +With the Docker Compose script provided by OpenIM, you can easily install Kafka. Use the following command to start Kafka: + +```bash +docker compose up -d +``` + +After executing this command, Kafka will be installed and started. You can confirm the Kafka container is running with the following command: + +```bash +docker ps | grep kafka +``` + +The output of this command, as shown below, displays the status information of the Kafka container: + +``` +be416b5a0851 bitnami/kafka:3.5.1 "/opt/bitnami/script…" 3 days ago Up 2 days 9092/tcp, 0.0.0.0:19094->9094/tcp, :::19094->9094/tcp kafka +``` + +### References + +- Official Docker installation documentation: [Click here](http://events.jianshu.io/p/b60afa35303a) +- Detailed installation guide: [Tutorial on Towards Data Science](https://towardsdatascience.com/how-to-install-apache-kafka-using-docker-the-easy-way-4ceb00817d8b) + +## Using Kafka + +### Entering the Kafka Container + +To execute Kafka commands, you first need to enter the Kafka container. Use the following command: + +```bash +docker exec -it kafka bash +``` + +### Kafka Command Tools + +Inside the Kafka container, you can use various command-line tools to manage Kafka. These tools include but are not limited to: + +- `kafka-topics.sh`: For creating, deleting, listing, or altering topics. +- `kafka-console-producer.sh`: Allows sending messages to a specified topic from the command line. +- `kafka-console-consumer.sh`: Allows reading messages from the command line, with the ability to specify topics. +- `kafka-consumer-groups.sh`: For managing consumer group information. + +### Kafka Client Tool Installation + +For easier Kafka management, you can install Kafka client tools. If you installed Kafka through OpenIM's Docker Compose, you can install the Kafka client tools with the following command: + +```bash +make install.kafkactl +``` + +### Automatic Topic Creation + +When installing Kafka through OpenIM's Docker Compose method, OpenIM automatically creates the following topics: + +- `latestMsgToRedis` +- `msgToPush` +- `offlineMsgToMongoMysql` + +These topics are created using the `scripts/create-topic.sh` script. The script waits for Kafka to be ready before executing the commands to create topics: + +```bash +# Wait for Kafka to be ready +until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092; do + echo "Waiting for Kafka to be ready..." + sleep 2 +done + +# Create topics +/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic latestMsgToRedis +/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic msgToPush +/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic offlineMsgToMongoMysql + +echo "Topics created." +``` + +The optimized and expanded documentation further details some basic commands for operations inside the Kafka container, as well as basic commands for managing Kafka using `kafkactl`. Here is a more detailed guide. + + +## Basic Commands in the Kafka Container + +### Listing Topics + +To list all existing topics, you can use the following command: + +```bash +kafka-topics.sh --list --bootstrap-server localhost:9092 +``` + +### Creating a New Topic + +When creating a new topic, you can specify the number of partitions and the replication factor. Here is the command to create a new topic: + +```bash +kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic your_topic_name +``` + +### Producing Messages + +To send messages to a specific topic, you can use the producer command. The following command prompts you to enter messages, which are sent to the specified topic with each press of the Enter key: + +```bash +kafka-console-producer.sh --broker-list localhost:9092 --topic your_topic_name +``` + +### Consuming Messages + +To read messages from a specific topic, you can use the consumer command. The following command reads new messages from the specified topic and outputs them on the console: + +```bash +kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic your_topic_name --from-beginning +``` + +The ` + +--from-beginning` parameter reads messages from the beginning of the topic. If this parameter is omitted, only new messages will be read. + + +## Basic Commands Using `kafkactl` + +`kafkactl` is a command-line tool for managing and operating Kafka clusters. It offers a more modern way to interact with Kafka. + +### Listing Topics + +To list all topics, you can use: + +```bash +kafkactl get topics +``` + +### Creating a New Topic + +To create a new topic with `kafkactl`, use: + +```bash +kafkactl create topic your_topic_name --partitions 1 --replication-factor 1 +``` + +### Producing Messages + +To send messages to a topic, you can use: + +```bash +kafkactl produce your_topic_name --value "your message" +``` + +Here, `"your message"` is the content of the message you want to send. + +### Consuming Messages + +To consume messages from a topic, use: + +```bash +kafkactl consume your_topic_name --from-beginning +``` + +Again, the `--from-beginning` parameter will start consuming messages from the beginning of the topic. If you do not wish to start from the beginning, you can omit this parameter. \ No newline at end of file diff --git a/docs/contrib/logging.md b/docs/contrib/logging.md index e25ac92c2..b88fbeb8d 100644 --- a/docs/contrib/logging.md +++ b/docs/contrib/logging.md @@ -2,7 +2,7 @@ ## Script Logging Documentation Link -If you wish to view the script's logging documentation, you can click on this link: [Logging Documentation](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/contrib/bash-log.md). +If you wish to view the script's logging documentation, you can click on this link: [Logging Documentation](https://github.com/openimsdk/open-im-server/blob/main/docs/contrib/bash-log.md). Below is the documentation for logging and error handling in the OpenIM Go project. diff --git a/docs/contrib/test.md b/docs/contrib/test.md index 37c1792cf..2470a0537 100644 --- a/docs/contrib/test.md +++ b/docs/contrib/test.md @@ -2,14 +2,94 @@ This document serves as a comprehensive guide to understanding and utilizing the `test.sh` script for testing OpenIM RPC services. The `test.sh` script is a collection of bash functions designed to test various aspects of the OpenIM RPC services, ensuring that each part of the API is functioning as expected. -+ Scripts:https://github.com/OpenIMSDK/Open-IM-Server/tree/main/scripts/install/test.sh ++ Scripts:https://github.com/openimsdk/open-im-server/tree/main/scripts/install/test.sh -For some complex, bulky functional tests, performance tests, and various e2e tests, We are all in the current warehouse to https://github.com/OpenIMSDK/Open-IM-Server/tree/main/test or https://github.com/openim-sigs/test-infra directory In the. +For some complex, bulky functional tests, performance tests, and various e2e tests, We are all in the current warehouse to https://github.com/openimsdk/open-im-server/tree/main/test or https://github.com/openim-sigs/test-infra directory In the. + About OpenIM Feature [Test Docs](https://docs.google.com/spreadsheets/d/1zELWkwxgOOZ7u5pmYCqqaFnvZy2SVajv/edit?usp=sharing&ouid=103266350914914783293&rtpof=true&sd=true) +## Util Test -## Usage +Let's restructure and enhance the document under a unified second-level heading, adding clarity and details for better comprehension and visual appeal. + +--- + +## Development Guide + +### Comprehensive Testing Instructions + +#### Running Unit Tests + +- **Command**: To execute unit tests, input the following in your terminal: + ``` + make test + ``` + +#### Evaluating Test Coverage + +- **Overview**: It's crucial to assess how much of your code is covered by tests. +- **Command**: + ```bash + make cover + ``` + This command generates a report detailing the percentage of your code tested, ensuring adherence to quality standards. + +#### Conducting API Tests + +- **Purpose**: API tests validate the interaction and functionality of your application's interfaces. +- **How to Run**: + ``` + make test-api + ``` + Use this to check the integrity and reliability of your API endpoints. + +#### End-to-End (E2E) Testing + +- **Scope**: E2E tests simulate real-user scenarios from start to finish. +- **Execution**: + ``` + make test-e2e + ``` + This comprehensive testing ensures your application performs as expected in real-world situations. + +### Crafting Unit Test Cases + +#### Setup for Test Case Generation + +- **Installation**: Install the `gotests` tool to generate test cases automatically. + ```bash + make install.gotests + ``` + This command installs the `gotests` tool for test case generation. + +- **Environment Preparation**: Define your test template environment variable and generate test cases as shown below: + ```bash + export GOTESTS_TEMPLATE=testify + gotests -i -w -only keyFunc . + ``` + This prepares your environment for test case generation using the `testify` template. + +#### Isolating Function Tests + +- **Single Function Testing**: When you need to focus on testing a single function for detailed examination. +- **Method**: + ```bash + go test -v -run TestKeyFunc + ``` + This command specifically runs tests for `TestKeyFunc`, allowing targeted debugging and validation. + +### Important Note + +- **Quality Assurance**: Throughout your development process, it is imperative to ensure that the unit test coverage meets or surpasses the standards set by OpenIM. +- **Maintaining Standards**: Regularly running your tests with + ```make test``` + supports maintaining high code quality and adherence to OpenIM's rigorous testing benchmarks. + +## E2E Test + +TODO + +## Api Test The `test.sh` script is located within the `./scripts/install/` directory of the OpenIM service's codebase. To use the script, navigate to this directory from your terminal: diff --git a/docs/images/Open-IM.png b/docs/images/Open-IM.png deleted file mode 100644 index b61706550..000000000 Binary files a/docs/images/Open-IM.png and /dev/null differ diff --git a/docs/images/Architecture.jpg b/docs/images/architecture.jpg similarity index 100% rename from docs/images/Architecture.jpg rename to docs/images/architecture.jpg diff --git a/docs/images/Wechat.jpg b/docs/images/wechat.jpg similarity index 100% rename from docs/images/Wechat.jpg rename to docs/images/wechat.jpg diff --git a/pkg/common/db/table/relation/msg.go b/pkg/common/db/table/relation/msg.go index fcdaf3299..14b8dbb30 100644 --- a/pkg/common/db/table/relation/msg.go +++ b/pkg/common/db/table/relation/msg.go @@ -21,6 +21,7 @@ import ( "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/mongo" ) diff --git a/pkg/common/discoveryregister/direct/directResolver.go b/pkg/common/discoveryregister/direct/direct_resolver.go similarity index 100% rename from pkg/common/discoveryregister/direct/directResolver.go rename to pkg/common/discoveryregister/direct/direct_resolver.go diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index fdebd79cc..3ac8710aa 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -15,8 +15,6 @@ package discoveryregister import ( - "os" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/direct" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" @@ -28,10 +26,6 @@ import ( // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(config *config.GlobalConfig) (discovery.SvcDiscoveryRegistry, error) { - if os.Getenv("ENVS_DISCOVERY") != "" { - config.Envs.Discovery = os.Getenv("ENVS_DISCOVERY") - } - switch config.Envs.Discovery { case "zookeeper": return zookeeper.NewZookeeperDiscoveryRegister(&config.Zookeeper) diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discoveryregister/kubernetes/kubernetes.go index 90fc314db..f1ce0bbdc 100644 --- a/pkg/common/discoveryregister/kubernetes/kubernetes.go +++ b/pkg/common/discoveryregister/kubernetes/kubernetes.go @@ -104,6 +104,7 @@ func getSelfHost(ctx context.Context, gatewayName string) string { } // like openimserver-openim-msggateway-0.openimserver-openim-msggateway-headless.openim-lin.svc.cluster.local:88. +// Replica set in kubernetes environment func getMsgGatewayHost(ctx context.Context, gatewayName string) []string { port := 88 instance := "openimserver" @@ -189,9 +190,10 @@ func (cli *K8sDR) CloseConn(conn *grpc.ClientConn) { // do not use this method for call rpc. func (cli *K8sDR) GetClientLocalConns() map[string][]*grpc.ClientConn { - fmt.Println("should not call this function!!!!!!!!!!!!!!!!!!!!!!!!!") + log.ZError(context.Background(), "should not call this function!", nil) return nil } + func (cli *K8sDR) Close() { } diff --git a/scripts-new/bricks.sh b/scripts-new/bricks.sh index d2077ec1e..9f730e3d9 100644 --- a/scripts-new/bricks.sh +++ b/scripts-new/bricks.sh @@ -1,9 +1,8 @@ -OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/ -source "${OPENIM_ROOT}/lib/util.sh" -source "${OPENIM_ROOT}/define/binaries.sh" -source "${OPENIM_ROOT}/lib/path.sh" - +source "$(dirname "${BASH_SOURCE[0]}")/lib/util.sh" +source "$(dirname "${BASH_SOURCE[0]}")/define/binaries.sh" +source "$(dirname "${BASH_SOURCE[0]}")/lib/path.sh" +source "$(dirname "${BASH_SOURCE[0]}")/lib/logging.sh" #停止所有的二进制对应的进程 @@ -19,17 +18,14 @@ ERR_LOG_FILE=err.log.file #启动所有的二进制 start_binaries() { - local project_dir="$OPENIM_ROOT" # You should adjust this path as necessary + echo $ # Iterate over binaries defined in binary_path.sh for binary in "${!binaries[@]}"; do local count=${binaries[$binary]} local bin_full_path=$(get_bin_full_path "$binary") # Loop to start binary the specified number of times for ((i=0; i "test.log" 2>&1 & - #nohup sh -c '"$bin_full_path" -i "$i" -c "$OPENIM_OUTPUT_CONFIG" 2>&1 | tee test.log' & - #nohup ./run_my_command.sh "$bin_full_path" -i "$i" -c "$OPENIM_OUTPUT_CONFIG" > test.log 2>&1 & + echo "Starting $binary instance $i: $bin_full_path -i $i -c $OPENIM_OUTPUT_CONFIG" cmd=("$bin_full_path" -i "$i" -c "$OPENIM_OUTPUT_CONFIG") nohup "${cmd[@]}" >> "${LOG_FILE}" 2> >(tee -a "$ERR_LOG_FILE" | while read line; do echo -e "\e[31m${line}\e[0m"; done >&2) & done @@ -80,9 +76,7 @@ check_binaries_running(){ result=$(openim::util::check_process_names "$full_path" "$expected_count") ret_val=$? - if [ "$ret_val" -eq 0 ]; then - echo "$binary is running normally." - else + if [ "$ret_val" -ne 0 ]; then no_running_binaries=$((no_running_binaries + 1)) echo $result fi diff --git a/scripts-new/check.sh b/scripts-new/check.sh index 256293023..128c5fe83 100644 --- a/scripts-new/check.sh +++ b/scripts-new/check.sh @@ -32,10 +32,10 @@ source "$OPENIM_SCRIPTS/bricks.sh" result=$(check_binaries_running) ret_val=$? if [ $ret_val -eq 0 ]; then - echo "All binaries are running." + echo "All services are running normally." else - echo "$result" - echo "abort..." + openim::log::error "Some services are not running as expected. Details are as follows:" + openim::log::error "$result" exit 1 fi diff --git a/scripts-new/lib/util.sh b/scripts-new/lib/util.sh index b2edb758b..543d2c684 100644 --- a/scripts-new/lib/util.sh +++ b/scripts-new/lib/util.sh @@ -387,7 +387,7 @@ openim::util::check_process_names() { if [ "$running_count" -eq "$expected_count" ]; then return 0 else - echo "$process_path Expected $expected_count processes, but $running_count are present" + echo "$process_path Expected $expected_count processes, but $running_count running" return 1 fi } diff --git a/scripts-new/start.sh b/scripts-new/start.sh index 7a2d27404..792fbeca3 100644 --- a/scripts-new/start.sh +++ b/scripts-new/start.sh @@ -20,27 +20,14 @@ ret_val=$? if [ $ret_val -ne 0 ]; then echo "$result" - echo "abort..." + echo "Some services running, abort start" exit 1 fi # Call the main function result=$(start_binaries) +openim::log::success $result - -result=$(check_binaries_running) -ret_val=$? -if [ $ret_val -eq 0 ]; then - echo "All binaries are running." -else - echo "$result" - echo "abort..." - exit 1 -fi - - -print_listened_ports_by_binaries - - +$OPENIM_SCRIPTS/check.sh diff --git a/scripts/check-all.sh b/scripts/check-all.sh index 2fc773b2f..54296b22d 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -20,9 +20,6 @@ # READ: https://github.com/openimsdk/open-im-server/tree/main/scripts/install/environment.sh - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/install/common.sh" @@ -114,7 +111,3 @@ else openim::util::find_ports_for_all_services ${OPENIM_MSGTRANSFER_BINARY[@]} openim::log::success "All OpenIM services are running normally! " fi - - - - diff --git a/scripts/make-rules/common-versions.mk b/scripts/make-rules/common-versions.mk new file mode 100644 index 000000000..572585fce --- /dev/null +++ b/scripts/make-rules/common-versions.mk @@ -0,0 +1,58 @@ +# Copyright © 2023 OpenIMSDK. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# OpenIM Makefile Versions used +# +# Define the latest version for each tool to ensure consistent versioning across installations +GOLANGCI_LINT_VERSION ?= latest +GOIMPORTS_VERSION ?= latest +ADDLICENSE_VERSION ?= latest +DEEPCOPY_GEN_VERSION ?= latest +CONVERSION_GEN_VERSION ?= latest +GINKGO_VERSION ?= v1.16.2 +GO_GITLINT_VERSION ?= latest +GO_JUNIT_REPORT_VERSION ?= latest +GOTESTS_VERSION ?= latest +SWAGGER_VERSION ?= latest +KUBE_SCORE_VERSION ?= latest +KUBECONFORM_VERSION ?= latest +GSEMVER_VERSION ?= latest +GIT_CHGLOG_VERSION ?= latest +KO_VERSION ?= latest +GITHUB_RELEASE_VERSION ?= latest +COSCLI_VERSION ?= v0.19.0-beta +MINIO_VERSION ?= latest +DELVE_VERSION ?= latest +AIR_VERSION ?= latest +GOLINES_VERSION ?= latest +GO_MOD_OUTDATED_VERSION ?= latest +CFSSL_VERSION ?= latest +DEPTH_VERSION ?= latest +GO_CALLVIS_VERSION ?= latest +MISSPELL_VERSION ?= latest +GOTHANKS_VERSION ?= latest +RICHGO_VERSION ?= latest +RTS_VERSION ?= latest +TYPECHECK_VERSION ?= latest +COMMENT_LANG_DETECTOR_VERSION ?= latest +STANDARDIZER_VERSION ?= latest +GO_TESTS_VERSION ?= v1.6.0 +GO_APIDIFF_VERSION ?= v0.8.2 +KAFKACTL_VERSION ?= latest +GOTESTSUM_VERSION ?= latest + +WIRE_VERSION ?= latest +# WIRE_VERSION ?= $(call get_go_version,github.com/google/wire) +MOCKGEN_VERSION ?= $(call get_go_version,github.com/golang/mock) +PROTOC_GEN_GO_VERSION ?= $(call get_go_version,github.com/golang/protobuf/protoc-gen-go) \ No newline at end of file diff --git a/scripts/make-rules/common.mk b/scripts/make-rules/common.mk index f8537b6ca..ffbb69a55 100644 --- a/scripts/make-rules/common.mk +++ b/scripts/make-rules/common.mk @@ -78,6 +78,13 @@ VERSION := $(shell git describe --tags --always --match='v*') # v2.3.3: git tag endif +# Helper function to get dependency version from go.mod +get_gomod_version = $(shell go list -m $1 | awk '{print $$2}') +define go_install +$(info ===========> Installing $(1)@$(2)) +$(GO) install $(1)@$(2) +endef + # Check if the tree is dirty. default to dirty(maybe u should commit?) GIT_TREE_STATE:="dirty" ifeq (, $(shell git status --porcelain 2>/dev/null)) diff --git a/scripts/make-rules/tools.mk b/scripts/make-rules/tools.mk index 917c18cfe..5335d094d 100644 --- a/scripts/make-rules/tools.mk +++ b/scripts/make-rules/tools.mk @@ -64,81 +64,101 @@ tools.verify.%: ## install.golangci-lint: Install golangci-lint .PHONY: install.golangci-lint install.golangci-lint: - @$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + @$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) ## install.goimports: Install goimports, used to format go source files .PHONY: install.goimports install.goimports: - @$(GO) install golang.org/x/tools/cmd/goimports@latest + @$(GO) install golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) ## install.addlicense: Install addlicense, used to add license header to source files .PHONY: install.addlicense install.addlicense: - @$(GO) install github.com/google/addlicense@latest + @$(GO) install github.com/google/addlicense@$(ADDLICENSE_VERSION) ## install.deepcopy-gen: Install deepcopy-gen, used to generate deep copy functions .PHONY: install.deepcopy-gen install.deepcopy-gen: - @$(GO) install k8s.io/code-generator/cmd/deepcopy-gen@latest + @$(GO) install k8s.io/code-generator/cmd/deepcopy-gen@$(DEEPCOPY_GEN_VERSION) ## install.conversion-gen: Install conversion-gen, used to generate conversion functions .PHONY: install.conversion-gen install.conversion-gen: - @$(GO) install k8s.io/code-generator/cmd/conversion-gen@latest + @$(GO) install k8s.io/code-generator/cmd/conversion-gen@$(CONVERSION_GEN_VERSION) ## install.ginkgo: Install ginkgo to run a single test or set of tests .PHONY: install.ginkgo install.ginkgo: - @$(GO) install github.com/onsi/ginkgo/ginkgo@v1.16.2 + @$(GO) install github.com/onsi/ginkgo/ginkgo@$(GINKGO_VERSION) -## Install go-gitlint: Install go-gitlint, used to check git commit message +## install.go-gitlint: Install go-gitlint, used to check git commit message .PHONY: install.go-gitlint install.go-gitlint: - @$(GO) install github.com/marmotedu/go-gitlint/cmd/go-gitlint@latest + @$(GO) install github.com/marmotedu/go-gitlint/cmd/go-gitlint@$(GO_GITLINT_VERSION) ## install.go-junit-report: Install go-junit-report, used to convert go test output to junit xml .PHONY: install.go-junit-report install.go-junit-report: - @$(GO) install github.com/jstemmer/go-junit-report@latest + @$(GO) install github.com/jstemmer/go-junit-report@$(GO_JUNIT_REPORT_VERSION) ## install.gotests: Install gotests, used to generate go tests +.PHONY: install.gotests +install.gotests: + @$(GO) install github.com/cweill/gotests/gotests@$(GO_TESTS_VERSION) + +## install.kafkactl: Install kafkactl command line tool. +.PHONY: install.kafkactl +install.kafkactl: + @$(GO) install github.com/deviceinsight/kafkactl@$(KAFKACTL_VERSION) + +## install.go-apidiff: Install go-apidiff, used to check api changes +.PHONY: install.go-apidiff +install.go-apidiff: + @$(GO) install github.com/joelanford/go-apidiff@$(GO_APIDIFF_VERSION) + +## install.swagger: Install swagger, used to generate swagger documentation .PHONY: install.swagger install.swagger: - @$(GO) install github.com/go-swagger/go-swagger/cmd/swagger@latest + @$(GO) install github.com/go-swagger/go-swagger/cmd/swagger@$(SWAGGER_VERSION) # ============================================================================== # Tools that might be used include go gvm # +## install.gotestsum: Install gotestsum, used to run go tests +.PHONY: install.gotestsum +install.gotestsum: + @$(GO) install gotest.tools/gotestsum@$(GOTESTSUM_VERSION) + ## install.kube-score: Install kube-score, used to check kubernetes yaml files .PHONY: install.kube-score install.kube-score: - @$(GO) install github.com/zegl/kube-score/cmd/kube-score@latest + @$(GO) install github.com/zegl/kube-score/cmd/kube-score@$(KUBE_SCORE_VERSION) ## install.kubeconform: Install kubeconform, used to check kubernetes yaml files .PHONY: install.kubeconform install.kubeconform: - @$(GO) install github.com/yannh/kubeconform/cmd/kubeconform@latest + @$(GO) install github.com/yannh/kubeconform/cmd/kubeconform@$(KUBECONFORM_VERSION) ## install.gsemver: Install gsemver, used to generate semver .PHONY: install.gsemver install.gsemver: - @$(GO) install github.com/arnaud-deprez/gsemver@latest + @$(GO) install github.com/arnaud-deprez/gsemver@$(GSEMVER_VERSION) ## install.git-chglog: Install git-chglog, used to generate changelog .PHONY: install.git-chglog install.git-chglog: - @$(GO) install github.com/git-chglog/git-chglog/cmd/git-chglog@latest + @$(GO) install github.com/git-chglog/git-chglog/cmd/git-chglog@$(GIT_CHGLOG_VERSION) ## install.ko: Install ko, used to build go program into container images .PHONY: install.ko install.ko: - @$(GO) install github.com/google/ko@latest + @$(GO) install github.com/google/ko@$(KO_VERSION) ## install.github-release: Install github-release, used to create github release .PHONY: install.github-release install.github-release: - @$(GO) install github.com/github-release/github-release@latest + @$(GO) install github.com/github-release/github-release@$(GITHUB_RELEASE_VERSION) ## install.coscli: Install coscli, used to upload files to cos # example: ./coscli cp/sync -r /home/off-line/docker-off-line/ cos://openim-1306374445/openim/image/amd/off-line/off-line/ -e cos.ap-guangzhou.myqcloud.com @@ -146,7 +166,7 @@ install.github-release: # amd64 .PHONY: install.coscli install.coscli: - @wget -q https://github.com/tencentyun/coscli/releases/download/v0.19.0-beta/coscli-linux -O ${TOOLS_DIR}/coscli + @wget -q https://github.com/tencentyun/coscli/releases/download/$(COSCLI_VERSION)/coscli-linux -O ${TOOLS_DIR}/coscli @chmod +x ${TOOLS_DIR}/coscli ## install.coscmd: Install coscmd, used to upload files to cos @@ -157,50 +177,50 @@ install.coscmd: ## install.minio: Install minio, used to upload files to minio .PHONY: install.minio install.minio: - @$(GO) install github.com/minio/minio@latest + @$(GO) install github.com/minio/minio@$(MINIO_VERSION) ## install.delve: Install delve, used to debug go program .PHONY: install.delve install.delve: - @$(GO) install github.com/go-delve/delve/cmd/dlv@latest + @$(GO) install github.com/go-delve/delve/cmd/dlv@$(DELVE_VERSION) ## install.air: Install air, used to hot reload go program .PHONY: install.air install.air: - @$(GO) install github.com/cosmtrek/air@latest + @$(GO) install github.com/cosmtrek/air@$(AIR_VERSION) ## install.gvm: Install gvm, gvm is a Go version manager, built on top of the official go tool. -# github: https://github.com/moovweb/gvm .PHONY: install.gvm install.gvm: - @echo "===========> Installing gvm,The default installation path is ~/.gvm/scripts/gvm" + @echo "===========> Installing gvm, The default installation path is ~/.gvm/scripts/gvm" @bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer) - @$(shell source /root/.gvm/scripts/gvm) + @source /root/.gvm/scripts/gvm ## install.golines: Install golines, used to format long lines .PHONY: install.golines install.golines: - @$(GO) install github.com/segmentio/golines@latest + @$(GO) install github.com/segmentio/golines@$(GOLINES_VERSION) ## install.go-mod-outdated: Install go-mod-outdated, used to check outdated dependencies .PHONY: install.go-mod-outdated install.go-mod-outdated: - @$(GO) install github.com/psampaz/go-mod-outdated@latest + @$(GO) install github.com/psampaz/go-mod-outdated@$(GO_MOD_OUTDATED_VERSION) ## install.mockgen: Install mockgen, used to generate mock functions .PHONY: install.mockgen install.mockgen: - @$(GO) install github.com/golang/mock/mockgen@latest + @$(GO) install github.com/golang/mock/mockgen@$(MOCKGEN_VERSION) + +## install.wire: Install wire, used to generate wire files +.PHONY: install.wire +install.wire: + @$(GO) install github.com/google/wire/cmd/wire@$(WIRE_VERSION) -## install.gotests: Install gotests, used to generate test functions -.PHONY: install.gotests -install.gotests: - @$(GO) install github.com/cweill/gotests/gotests@latest ## install.protoc-gen-go: Install protoc-gen-go, used to generate go source files from protobuf files .PHONY: install.protoc-gen-go install.protoc-gen-go: - @$(GO) install github.com/golang/protobuf/protoc-gen-go@latest + @$(GO) install github.com/golang/protobuf/protoc-gen-go@$(PROTOC_GEN_GO_VERSION) ## install.cfssl: Install cfssl, used to generate certificates .PHONY: install.cfssl @@ -210,43 +230,49 @@ install.cfssl: ## install.depth: Install depth, used to check dependency tree .PHONY: install.depth install.depth: - @$(GO) install github.com/KyleBanks/depth/cmd/depth@latest + @$(GO) install github.com/KyleBanks/depth/cmd/depth@$(DEPTH_VERSION) ## install.go-callvis: Install go-callvis, used to visualize call graph .PHONY: install.go-callvis install.go-callvis: - @$(GO) install github.com/ofabry/go-callvis@latest + @$(GO) install github.com/ofabry/go-callvis@$(GO_CALLVIS_VERSION) -## install.misspell +## install.misspell: Install misspell .PHONY: install.misspell install.misspell: - @$(GO) install github.com/client9/misspell/cmd/misspell@latest + @$(GO) install github.com/client9/misspell/cmd/misspell@$(MISSPELL_VERSION) ## install.gothanks: Install gothanks, used to thank go dependencies .PHONY: install.gothanks install.gothanks: - @$(GO) install github.com/psampaz/gothanks@latest + @$(GO) install github.com/psampaz/gothanks@$(GOTHANKS_VERSION) ## install.richgo: Install richgo .PHONY: install.richgo install.richgo: - @$(GO) install github.com/kyoh86/richgo@latest + @$(GO) install github.com/kyoh86/richgo@$(RICHGO_VERSION) ## install.rts: Install rts .PHONY: install.rts install.rts: - @$(GO) install github.com/galeone/rts/cmd/rts@latest + @$(GO) install github.com/galeone/rts/cmd/rts@$(RTS_VERSION) # ================= kubecub openim tools ========================================= -## install.typecheck: install kubecub typecheck check for go code +# https://github.com/kubecub +## install.typecheck: Install kubecub typecheck, checks for go code .PHONY: install.typecheck install.typecheck: - @$(GO) install github.com/kubecub/typecheck@latest + @$(GO) install github.com/kubecub/typecheck@$(TYPECHECK_VERSION) -## install.comment-lang-detector: install kubecub comment-lang-detector check for go code comment language +## install.comment-lang-detector: Install kubecub comment-lang-detector, checks for go code comment language .PHONY: install.comment-lang-detector install.comment-lang-detector: - @$(GO) install github.com/kubecub/comment-lang-detector/cmd/cld@latest + @$(GO) install github.com/kubecub/comment-lang-detector/cmd/cld@$(COMMENT_LANG_DETECTOR_VERSION) + +## install.standardizer: Install kubecub standardizer, checks for go code standardization +.PHONY: install.standardizer +install.standardizer: + @$(GO) install github.com/kubecub/standardizer@$(STANDARDIZER_VERSION) ## tools.help: Display help information about the tools package .PHONY: tools.help diff --git a/scripts/start-all.sh b/scripts/start-all.sh index acc926f59..6f4a6c574 100755 --- a/scripts/start-all.sh +++ b/scripts/start-all.sh @@ -94,7 +94,6 @@ if [[ $? -ne 0 ]]; then exit 1 fi - result=$(openim::util::check_process_names ${OPENIM_ALL_SERVICE_LIBRARIES_NO_TRANSFER[@]}) if [[ $? -ne 0 ]]; then openim::log::error "The OpenIM services may fail to start.\n $result" diff --git a/scripts/verify-standardizer.sh b/scripts/verify-standardizer.sh new file mode 100755 index 000000000..08a13b9a2 --- /dev/null +++ b/scripts/verify-standardizer.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Copyright © 2023 OpenIM. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script does a fast type check of script srnetes code for all platforms. +# Usage: `scripts/verify-standardizer.sh`. + +OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${OPENIM_ROOT}/scripts/lib/init.sh" + +openim::golang::verify_go_version + +cd "${OPENIM_ROOT}" +ret=0 +scripts/run-in-gopath.sh \ +make tools.verify.standardizer +${OPENIM_ROOT}/_output/tools/standardizer || ret=$? +if [[ $ret -ne 0 ]]; then + openim::log::error "Failed to check the directory name or file name. Your name may not meet the specification. Please check the configuration file and the directory or file name." >&2 + openim::log::error "Please see https://github.com/kubecub/standardizer for more information." >&2 + exit 1 +fi diff --git a/test/testdata/requests/sendMessage.json b/test/testdata/requests/send-message.json similarity index 100% rename from test/testdata/requests/sendMessage.json rename to test/testdata/requests/send-message.json diff --git a/tools/component/component.go b/tools/component/component.go index 6cea8352d..a625cdad0 100644 --- a/tools/component/component.go +++ b/tools/component/component.go @@ -115,7 +115,7 @@ func main() { allSuccess := true for _, check := range serviceChecks { - if err := check(ctx, conf); err != nil { + if err := check(ctx, &config.Config); err != nil { util.ColorErrPrint(fmt.Sprintf("Check failed: %v", err)) allSuccess = false break diff --git a/tools/component/util/env.go b/tools/component/util/env.go deleted file mode 100644 index 92f072f76..000000000 --- a/tools/component/util/env.go +++ /dev/null @@ -1,107 +0,0 @@ -package util - -import ( - "os" - "strconv" - "strings" - - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/errs" -) - -func ConfigGetEnv(config *config.GlobalConfig) error { - config.Mongo.Uri = getEnv("MONGO_URI", config.Mongo.Uri) - config.Mongo.Username = getEnv("MONGO_OPENIM_USERNAME", config.Mongo.Username) - config.Mongo.Password = getEnv("MONGO_OPENIM_PASSWORD", config.Mongo.Password) - config.Mongo.Address = getArrEnv("MONGO_ADDRESS", "MONGO_PORT", config.Mongo.Address) - config.Mongo.Database = getEnv("MONGO_DATABASE", config.Mongo.Database) - maxPoolSize, err := getEnvInt("MONGO_MAX_POOL_SIZE", config.Mongo.MaxPoolSize) - if err != nil { - return errs.WrapMsg(err, "MONGO_MAX_POOL_SIZE") - } - config.Mongo.MaxPoolSize = maxPoolSize - - config.Redis.Username = getEnv("REDIS_USERNAME", config.Redis.Username) - config.Redis.Password = getEnv("REDIS_PASSWORD", config.Redis.Password) - config.Redis.Address = getArrEnv("REDIS_ADDRESS", "REDIS_PORT", config.Redis.Address) - - config.Object.ApiURL = getEnv("OBJECT_APIURL", config.Object.ApiURL) - config.Object.Minio.Endpoint = getEnv("MINIO_ENDPOINT", config.Object.Minio.Endpoint) - config.Object.Minio.AccessKeyID = getEnv("MINIO_ACCESS_KEY_ID", config.Object.Minio.AccessKeyID) - config.Object.Minio.SecretAccessKey = getEnv("MINIO_SECRET_ACCESS_KEY", config.Object.Minio.SecretAccessKey) - config.Object.Minio.SignEndpoint = getEnv("MINIO_SIGN_ENDPOINT", config.Object.Minio.SignEndpoint) - - config.Zookeeper.Schema = getEnv("ZOOKEEPER_SCHEMA", config.Zookeeper.Schema) - config.Zookeeper.ZkAddr = getArrEnv("ZOOKEEPER_ADDRESS", "ZOOKEEPER_PORT", config.Zookeeper.ZkAddr) - config.Zookeeper.Username = getEnv("ZOOKEEPER_USERNAME", config.Zookeeper.Username) - config.Zookeeper.Password = getEnv("ZOOKEEPER_PASSWORD", config.Zookeeper.Password) - - config.Kafka.Username = getEnv("KAFKA_USERNAME", config.Kafka.Username) - config.Kafka.Password = getEnv("KAFKA_PASSWORD", config.Kafka.Password) - config.Kafka.Addr = getArrEnv("KAFKA_ADDRESS", "KAFKA_PORT", config.Kafka.Addr) - config.Object.Minio.Endpoint = getMinioAddr("MINIO_ENDPOINT", "MINIO_ADDRESS", "MINIO_PORT", config.Object.Minio.Endpoint) - return nil -} - -// Helper function to get environment variable or default value -func getEnv(key, fallback string) string { - if value, exists := os.LookupEnv(key); exists { - return value - } - return fallback -} - -// Helper function to get environment variable or default value -func getEnvInt(key string, fallback int) (int, error) { - if value, exists := os.LookupEnv(key); exists { - val, err := strconv.Atoi(value) - if err != nil { - return 0, errs.WrapMsg(err, "string to int failed") - } - return val, nil - } - return fallback, nil -} - -func getArrEnv(key1, key2 string, fallback []string) []string { - address, addrExists := os.LookupEnv(key1) - port, portExists := os.LookupEnv(key2) - - if addrExists && portExists { - addresses := strings.Split(address, ",") - for i, addr := range addresses { - addresses[i] = addr + ":" + port - } - return addresses - } - - if addrExists && !portExists { - addresses := strings.Split(address, ",") - for i, addr := range addresses { - addresses[i] = addr + ":" + "0" - } - return addresses - } - - if !addrExists && portExists { - result := make([]string, len(fallback)) - for i, addr := range fallback { - add := strings.Split(addr, ":") - result[i] = add[0] + ":" + port - } - return result - } - return fallback -} - -func getMinioAddr(key1, key2, key3, fallback string) string { - // Prioritize environment variables - endpoint := getEnv(key1, fallback) - address, addressExist := os.LookupEnv(key2) - port, portExist := os.LookupEnv(key3) - if portExist && addressExist { - endpoint = "http://" + address + ":" + port - return endpoint - } - return endpoint -}