pull/2148/head
Gordon
2 years ago
commit
7e2a55ce55
@ -0,0 +1,50 @@
|
||||
# https://github.com/marketplace/actions/conformity-checker-for-project
|
||||
baseConfig:
|
||||
searchDirectory: "./"
|
||||
ignoreCase: false
|
||||
|
||||
directoryNaming:
|
||||
allowHyphens: true
|
||||
allowUnderscores: false
|
||||
mustBeLowercase: true
|
||||
|
||||
fileNaming:
|
||||
allowHyphens: true
|
||||
allowUnderscores: true
|
||||
mustBeLowercase: true
|
||||
|
||||
ignoreFormats:
|
||||
- "\\.log$"
|
||||
- "\\.env$"
|
||||
- "README\\.md$"
|
||||
- "_test\\.go$"
|
||||
- "\\.md$"
|
||||
- _test\\.txt$
|
||||
- LICENSE
|
||||
- Dockerfile
|
||||
- CODEOWNERS
|
||||
- Makefile
|
||||
|
||||
ignoreDirectories:
|
||||
- "vendor"
|
||||
- ".git"
|
||||
- "deployments"
|
||||
- "node_modules"
|
||||
- "logs"
|
||||
- "CHANGELOG"
|
||||
- "components"
|
||||
- "_output"
|
||||
- "tools/openim-web"
|
||||
- "CHANGELOG"
|
||||
- "examples/Test_directory"
|
||||
- test/testdata
|
||||
|
||||
fileTypeSpecificNaming:
|
||||
".yaml":
|
||||
allowHyphens: true
|
||||
allowUnderscores: false
|
||||
mustBeLowercase: true
|
||||
".go":
|
||||
allowHyphens: false
|
||||
allowUnderscores: true
|
||||
mustBeLowercase: true
|
||||
@ -0,0 +1,162 @@
|
||||
# OpenIM Kafka Guide
|
||||
|
||||
This document aims to provide a set of concise guidelines to help you quickly install and use Kafka through Docker Compose.
|
||||
|
||||
## Installing Kafka
|
||||
|
||||
With the Docker Compose script provided by OpenIM, you can easily install Kafka. Use the following command to start Kafka:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
After executing this command, Kafka will be installed and started. You can confirm the Kafka container is running with the following command:
|
||||
|
||||
```bash
|
||||
docker ps | grep kafka
|
||||
```
|
||||
|
||||
The output of this command, as shown below, displays the status information of the Kafka container:
|
||||
|
||||
```
|
||||
be416b5a0851 bitnami/kafka:3.5.1 "/opt/bitnami/script…" 3 days ago Up 2 days 9092/tcp, 0.0.0.0:19094->9094/tcp, :::19094->9094/tcp kafka
|
||||
```
|
||||
|
||||
### References
|
||||
|
||||
- Official Docker installation documentation: [Click here](http://events.jianshu.io/p/b60afa35303a)
|
||||
- Detailed installation guide: [Tutorial on Towards Data Science](https://towardsdatascience.com/how-to-install-apache-kafka-using-docker-the-easy-way-4ceb00817d8b)
|
||||
|
||||
## Using Kafka
|
||||
|
||||
### Entering the Kafka Container
|
||||
|
||||
To execute Kafka commands, you first need to enter the Kafka container. Use the following command:
|
||||
|
||||
```bash
|
||||
docker exec -it kafka bash
|
||||
```
|
||||
|
||||
### Kafka Command Tools
|
||||
|
||||
Inside the Kafka container, you can use various command-line tools to manage Kafka. These tools include but are not limited to:
|
||||
|
||||
- `kafka-topics.sh`: For creating, deleting, listing, or altering topics.
|
||||
- `kafka-console-producer.sh`: Allows sending messages to a specified topic from the command line.
|
||||
- `kafka-console-consumer.sh`: Allows reading messages from the command line, with the ability to specify topics.
|
||||
- `kafka-consumer-groups.sh`: For managing consumer group information.
|
||||
|
||||
### Kafka Client Tool Installation
|
||||
|
||||
For easier Kafka management, you can install Kafka client tools. If you installed Kafka through OpenIM's Docker Compose, you can install the Kafka client tools with the following command:
|
||||
|
||||
```bash
|
||||
make install.kafkactl
|
||||
```
|
||||
|
||||
### Automatic Topic Creation
|
||||
|
||||
When installing Kafka through OpenIM's Docker Compose method, OpenIM automatically creates the following topics:
|
||||
|
||||
- `latestMsgToRedis`
|
||||
- `msgToPush`
|
||||
- `offlineMsgToMongoMysql`
|
||||
|
||||
These topics are created using the `scripts/create-topic.sh` script. The script waits for Kafka to be ready before executing the commands to create topics:
|
||||
|
||||
```bash
|
||||
# Wait for Kafka to be ready
|
||||
until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092; do
|
||||
echo "Waiting for Kafka to be ready..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Create topics
|
||||
/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic latestMsgToRedis
|
||||
/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic msgToPush
|
||||
/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic offlineMsgToMongoMysql
|
||||
|
||||
echo "Topics created."
|
||||
```
|
||||
|
||||
The optimized and expanded documentation further details some basic commands for operations inside the Kafka container, as well as basic commands for managing Kafka using `kafkactl`. Here is a more detailed guide.
|
||||
|
||||
|
||||
## Basic Commands in the Kafka Container
|
||||
|
||||
### Listing Topics
|
||||
|
||||
To list all existing topics, you can use the following command:
|
||||
|
||||
```bash
|
||||
kafka-topics.sh --list --bootstrap-server localhost:9092
|
||||
```
|
||||
|
||||
### Creating a New Topic
|
||||
|
||||
When creating a new topic, you can specify the number of partitions and the replication factor. Here is the command to create a new topic:
|
||||
|
||||
```bash
|
||||
kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic your_topic_name
|
||||
```
|
||||
|
||||
### Producing Messages
|
||||
|
||||
To send messages to a specific topic, you can use the producer command. The following command prompts you to enter messages, which are sent to the specified topic with each press of the Enter key:
|
||||
|
||||
```bash
|
||||
kafka-console-producer.sh --broker-list localhost:9092 --topic your_topic_name
|
||||
```
|
||||
|
||||
### Consuming Messages
|
||||
|
||||
To read messages from a specific topic, you can use the consumer command. The following command reads new messages from the specified topic and outputs them on the console:
|
||||
|
||||
```bash
|
||||
kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic your_topic_name --from-beginning
|
||||
```
|
||||
|
||||
The `
|
||||
|
||||
--from-beginning` parameter reads messages from the beginning of the topic. If this parameter is omitted, only new messages will be read.
|
||||
|
||||
|
||||
## Basic Commands Using `kafkactl`
|
||||
|
||||
`kafkactl` is a command-line tool for managing and operating Kafka clusters. It offers a more modern way to interact with Kafka.
|
||||
|
||||
### Listing Topics
|
||||
|
||||
To list all topics, you can use:
|
||||
|
||||
```bash
|
||||
kafkactl get topics
|
||||
```
|
||||
|
||||
### Creating a New Topic
|
||||
|
||||
To create a new topic with `kafkactl`, use:
|
||||
|
||||
```bash
|
||||
kafkactl create topic your_topic_name --partitions 1 --replication-factor 1
|
||||
```
|
||||
|
||||
### Producing Messages
|
||||
|
||||
To send messages to a topic, you can use:
|
||||
|
||||
```bash
|
||||
kafkactl produce your_topic_name --value "your message"
|
||||
```
|
||||
|
||||
Here, `"your message"` is the content of the message you want to send.
|
||||
|
||||
### Consuming Messages
|
||||
|
||||
To consume messages from a topic, use:
|
||||
|
||||
```bash
|
||||
kafkactl consume your_topic_name --from-beginning
|
||||
```
|
||||
|
||||
Again, the `--from-beginning` parameter will start consuming messages from the beginning of the topic. If you do not wish to start from the beginning, you can omit this parameter.
|
||||
|
Before Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 207 KiB After Width: | Height: | Size: 207 KiB |
|
Before Width: | Height: | Size: 118 KiB After Width: | Height: | Size: 118 KiB |
@ -0,0 +1,58 @@
|
||||
# Copyright © 2023 OpenIMSDK.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
# OpenIM Makefile Versions used
|
||||
#
|
||||
# Define the latest version for each tool to ensure consistent versioning across installations
|
||||
GOLANGCI_LINT_VERSION ?= latest
|
||||
GOIMPORTS_VERSION ?= latest
|
||||
ADDLICENSE_VERSION ?= latest
|
||||
DEEPCOPY_GEN_VERSION ?= latest
|
||||
CONVERSION_GEN_VERSION ?= latest
|
||||
GINKGO_VERSION ?= v1.16.2
|
||||
GO_GITLINT_VERSION ?= latest
|
||||
GO_JUNIT_REPORT_VERSION ?= latest
|
||||
GOTESTS_VERSION ?= latest
|
||||
SWAGGER_VERSION ?= latest
|
||||
KUBE_SCORE_VERSION ?= latest
|
||||
KUBECONFORM_VERSION ?= latest
|
||||
GSEMVER_VERSION ?= latest
|
||||
GIT_CHGLOG_VERSION ?= latest
|
||||
KO_VERSION ?= latest
|
||||
GITHUB_RELEASE_VERSION ?= latest
|
||||
COSCLI_VERSION ?= v0.19.0-beta
|
||||
MINIO_VERSION ?= latest
|
||||
DELVE_VERSION ?= latest
|
||||
AIR_VERSION ?= latest
|
||||
GOLINES_VERSION ?= latest
|
||||
GO_MOD_OUTDATED_VERSION ?= latest
|
||||
CFSSL_VERSION ?= latest
|
||||
DEPTH_VERSION ?= latest
|
||||
GO_CALLVIS_VERSION ?= latest
|
||||
MISSPELL_VERSION ?= latest
|
||||
GOTHANKS_VERSION ?= latest
|
||||
RICHGO_VERSION ?= latest
|
||||
RTS_VERSION ?= latest
|
||||
TYPECHECK_VERSION ?= latest
|
||||
COMMENT_LANG_DETECTOR_VERSION ?= latest
|
||||
STANDARDIZER_VERSION ?= latest
|
||||
GO_TESTS_VERSION ?= v1.6.0
|
||||
GO_APIDIFF_VERSION ?= v0.8.2
|
||||
KAFKACTL_VERSION ?= latest
|
||||
GOTESTSUM_VERSION ?= latest
|
||||
|
||||
WIRE_VERSION ?= latest
|
||||
# WIRE_VERSION ?= $(call get_go_version,github.com/google/wire)
|
||||
MOCKGEN_VERSION ?= $(call get_go_version,github.com/golang/mock)
|
||||
PROTOC_GEN_GO_VERSION ?= $(call get_go_version,github.com/golang/protobuf/protoc-gen-go)
|
||||
@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright © 2023 OpenIM. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script does a fast type check of script srnetes code for all platforms.
|
||||
# Usage: `scripts/verify-standardizer.sh`.
|
||||
|
||||
OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
|
||||
source "${OPENIM_ROOT}/scripts/lib/init.sh"
|
||||
|
||||
openim::golang::verify_go_version
|
||||
|
||||
cd "${OPENIM_ROOT}"
|
||||
ret=0
|
||||
scripts/run-in-gopath.sh \
|
||||
make tools.verify.standardizer
|
||||
${OPENIM_ROOT}/_output/tools/standardizer || ret=$?
|
||||
if [[ $ret -ne 0 ]]; then
|
||||
openim::log::error "Failed to check the directory name or file name. Your name may not meet the specification. Please check the configuration file and the directory or file name." >&2
|
||||
openim::log::error "Please see https://github.com/kubecub/standardizer for more information." >&2
|
||||
exit 1
|
||||
fi
|
||||
@ -1,107 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/tools/errs"
|
||||
)
|
||||
|
||||
func ConfigGetEnv(config *config.GlobalConfig) error {
|
||||
config.Mongo.Uri = getEnv("MONGO_URI", config.Mongo.Uri)
|
||||
config.Mongo.Username = getEnv("MONGO_OPENIM_USERNAME", config.Mongo.Username)
|
||||
config.Mongo.Password = getEnv("MONGO_OPENIM_PASSWORD", config.Mongo.Password)
|
||||
config.Mongo.Address = getArrEnv("MONGO_ADDRESS", "MONGO_PORT", config.Mongo.Address)
|
||||
config.Mongo.Database = getEnv("MONGO_DATABASE", config.Mongo.Database)
|
||||
maxPoolSize, err := getEnvInt("MONGO_MAX_POOL_SIZE", config.Mongo.MaxPoolSize)
|
||||
if err != nil {
|
||||
return errs.WrapMsg(err, "MONGO_MAX_POOL_SIZE")
|
||||
}
|
||||
config.Mongo.MaxPoolSize = maxPoolSize
|
||||
|
||||
config.Redis.Username = getEnv("REDIS_USERNAME", config.Redis.Username)
|
||||
config.Redis.Password = getEnv("REDIS_PASSWORD", config.Redis.Password)
|
||||
config.Redis.Address = getArrEnv("REDIS_ADDRESS", "REDIS_PORT", config.Redis.Address)
|
||||
|
||||
config.Object.ApiURL = getEnv("OBJECT_APIURL", config.Object.ApiURL)
|
||||
config.Object.Minio.Endpoint = getEnv("MINIO_ENDPOINT", config.Object.Minio.Endpoint)
|
||||
config.Object.Minio.AccessKeyID = getEnv("MINIO_ACCESS_KEY_ID", config.Object.Minio.AccessKeyID)
|
||||
config.Object.Minio.SecretAccessKey = getEnv("MINIO_SECRET_ACCESS_KEY", config.Object.Minio.SecretAccessKey)
|
||||
config.Object.Minio.SignEndpoint = getEnv("MINIO_SIGN_ENDPOINT", config.Object.Minio.SignEndpoint)
|
||||
|
||||
config.Zookeeper.Schema = getEnv("ZOOKEEPER_SCHEMA", config.Zookeeper.Schema)
|
||||
config.Zookeeper.ZkAddr = getArrEnv("ZOOKEEPER_ADDRESS", "ZOOKEEPER_PORT", config.Zookeeper.ZkAddr)
|
||||
config.Zookeeper.Username = getEnv("ZOOKEEPER_USERNAME", config.Zookeeper.Username)
|
||||
config.Zookeeper.Password = getEnv("ZOOKEEPER_PASSWORD", config.Zookeeper.Password)
|
||||
|
||||
config.Kafka.Username = getEnv("KAFKA_USERNAME", config.Kafka.Username)
|
||||
config.Kafka.Password = getEnv("KAFKA_PASSWORD", config.Kafka.Password)
|
||||
config.Kafka.Addr = getArrEnv("KAFKA_ADDRESS", "KAFKA_PORT", config.Kafka.Addr)
|
||||
config.Object.Minio.Endpoint = getMinioAddr("MINIO_ENDPOINT", "MINIO_ADDRESS", "MINIO_PORT", config.Object.Minio.Endpoint)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper function to get environment variable or default value
|
||||
func getEnv(key, fallback string) string {
|
||||
if value, exists := os.LookupEnv(key); exists {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// Helper function to get environment variable or default value
|
||||
func getEnvInt(key string, fallback int) (int, error) {
|
||||
if value, exists := os.LookupEnv(key); exists {
|
||||
val, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return 0, errs.WrapMsg(err, "string to int failed")
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
return fallback, nil
|
||||
}
|
||||
|
||||
func getArrEnv(key1, key2 string, fallback []string) []string {
|
||||
address, addrExists := os.LookupEnv(key1)
|
||||
port, portExists := os.LookupEnv(key2)
|
||||
|
||||
if addrExists && portExists {
|
||||
addresses := strings.Split(address, ",")
|
||||
for i, addr := range addresses {
|
||||
addresses[i] = addr + ":" + port
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
|
||||
if addrExists && !portExists {
|
||||
addresses := strings.Split(address, ",")
|
||||
for i, addr := range addresses {
|
||||
addresses[i] = addr + ":" + "0"
|
||||
}
|
||||
return addresses
|
||||
}
|
||||
|
||||
if !addrExists && portExists {
|
||||
result := make([]string, len(fallback))
|
||||
for i, addr := range fallback {
|
||||
add := strings.Split(addr, ":")
|
||||
result[i] = add[0] + ":" + port
|
||||
}
|
||||
return result
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func getMinioAddr(key1, key2, key3, fallback string) string {
|
||||
// Prioritize environment variables
|
||||
endpoint := getEnv(key1, fallback)
|
||||
address, addressExist := os.LookupEnv(key2)
|
||||
port, portExist := os.LookupEnv(key3)
|
||||
if portExist && addressExist {
|
||||
endpoint = "http://" + address + ":" + port
|
||||
return endpoint
|
||||
}
|
||||
return endpoint
|
||||
}
|
||||
Loading…
Reference in new issue