fix: fix the component check func

pull/1868/head
luhaoling 2 years ago
parent d95fde0528
commit e729fe62e1

@ -77,9 +77,10 @@ func NewRedis() (redis.UniversalClient, error) {
defer cancel() defer cancel()
err = rdb.Ping(ctx).Err() err = rdb.Ping(ctx).Err()
if err != nil { if err != nil {
return nil, errs.Wrap(fmt.Errorf("redis ping %w", err)) uriFormat := "address:%s, username:%s, password:%s, clusterMode:%t, enablePipeline:%t"
errMsg := fmt.Sprintf(uriFormat, config.Config.Redis.Address, config.Config.Redis.Username, config.Config.Redis.Password, config.Config.Redis.ClusterMode, config.Config.Redis.EnablePipeline)
return nil, errs.Wrap(err, errMsg)
} }
redisClient = rdb redisClient = rdb
return rdb, err return rdb, err
} }

@ -61,9 +61,9 @@ func NewMongo() (*Mongo, error) {
time.Sleep(time.Second) // exponential backoff could be implemented here time.Sleep(time.Second) // exponential backoff could be implemented here
continue continue
} }
return nil, errs.Wrap(err) return nil, errs.Wrap(err, uri)
} }
return nil, errs.Wrap(err) return nil, errs.Wrap(err, uri)
} }
func buildMongoURI() string { func buildMongoURI() string {

@ -15,6 +15,8 @@
package zookeeper package zookeeper
import ( import (
"fmt"
"github.com/OpenIMSDK/tools/errs"
"os" "os"
"strings" "strings"
"time" "time"
@ -33,7 +35,7 @@ func NewZookeeperDiscoveryRegister() (discoveryregistry.SvcDiscoveryRegistry, er
username := getEnv("ZOOKEEPER_USERNAME", config.Config.Zookeeper.Username) username := getEnv("ZOOKEEPER_USERNAME", config.Config.Zookeeper.Username)
password := getEnv("ZOOKEEPER_PASSWORD", config.Config.Zookeeper.Password) password := getEnv("ZOOKEEPER_PASSWORD", config.Config.Zookeeper.Password)
return openkeeper.NewClient( zk, err := openkeeper.NewClient(
zkAddr, zkAddr,
schema, schema,
openkeeper.WithFreq(time.Hour), openkeeper.WithFreq(time.Hour),
@ -42,6 +44,16 @@ func NewZookeeperDiscoveryRegister() (discoveryregistry.SvcDiscoveryRegistry, er
openkeeper.WithTimeout(10), openkeeper.WithTimeout(10),
openkeeper.WithLogger(log.NewZkLogger()), openkeeper.WithLogger(log.NewZkLogger()),
) )
if err != nil {
uriFormat := "address:%s, username :%s, password :%s, schema:%s."
errInfo := fmt.Sprintf(uriFormat,
config.Config.Zookeeper.ZkAddr,
config.Config.Zookeeper.Username,
config.Config.Zookeeper.Password,
config.Config.Zookeeper.Schema)
return nil, errs.Wrap(err, errInfo)
}
return zk, nil
} }
// getEnv returns the value of an environment variable if it exists, otherwise it returns the fallback value. // getEnv returns the value of an environment variable if it exists, otherwise it returns the fallback value.

@ -51,7 +51,7 @@ func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, topics, addrs []str
SetupTLSConfig(consumerGroupConfig) SetupTLSConfig(consumerGroupConfig)
consumerGroup, err := sarama.NewConsumerGroup(addrs, groupID, consumerGroupConfig) consumerGroup, err := sarama.NewConsumerGroup(addrs, groupID, consumerGroupConfig)
if err != nil { if err != nil {
return nil, errs.Wrap(err, strings.Join(topics, ","), strings.Join(addrs, ","), groupID) return nil, errs.Wrap(err, strings.Join(topics, ","), strings.Join(addrs, ","), groupID, config.Config.Kafka.Username, config.Config.Kafka.Password)
} }
return &MConsumerGroup{ return &MConsumerGroup{
consumerGroup, consumerGroup,

@ -15,16 +15,15 @@
package main package main
import ( import (
"errors"
"flag" "flag"
"fmt" "fmt"
"github.com/IBM/sarama" "github.com/IBM/sarama"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/zookeeper" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/zookeeper"
"github.com/openimsdk/open-im-server/v3/pkg/common/kafka" "github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
"log"
"os" "os"
"strings" "strings"
"time" "time"
@ -40,12 +39,16 @@ const (
// defaultCfgPath is the default path of the configuration file. // defaultCfgPath is the default path of the configuration file.
defaultCfgPath = "../../../../../config/config.yaml" defaultCfgPath = "../../../../../config/config.yaml"
maxRetry = 300 maxRetry = 300
componentStartErrCode = 6000 colorRed = 31
) )
var ( var (
cfgPath = flag.String("c", defaultCfgPath, "Path to the configuration file") cfgPath = flag.String("c", defaultCfgPath, "Path to the configuration file")
ErrComponentStart = errs.NewCodeError(componentStartErrCode, "ComponentStartErr") MongoAuthFailed = "Authentication failed."
RedisAuthFailed = "NOAUTH Authentication required."
MinioAuthFailed = "Minio Authentication failed"
ZkAuthFailed = "zk Authentication failed"
KafkaAuthFailed = "SASL Authentication failed"
) )
func initCfg() error { func initCfg() error {
@ -59,7 +62,8 @@ func initCfg() error {
type checkFunc struct { type checkFunc struct {
name string name string
function func() (string, error) function func() error
authInfo string
} }
func main() { func main() {
@ -75,11 +79,11 @@ func main() {
checks := []checkFunc{ checks := []checkFunc{
//{name: "Mysql", function: checkMysql}, //{name: "Mysql", function: checkMysql},
{name: "Mongo", function: checkMongo}, {name: "Mongo", function: checkMongo, authInfo: MongoAuthFailed},
{name: "Redis", function: checkRedis}, {name: "Redis", function: checkRedis, authInfo: RedisAuthFailed},
{name: "Minio", function: checkMinio}, {name: "Minio", function: checkMinio, authInfo: MinioAuthFailed},
{name: "Zookeeper", function: checkZookeeper}, {name: "Zookeeper", function: checkZookeeper, authInfo: ZkAuthFailed},
{name: "Kafka", function: checkKafka}, {name: "Kafka", function: checkKafka, authInfo: KafkaAuthFailed},
} }
for i := 0; i < maxRetry; i++ { for i := 0; i < maxRetry; i++ {
@ -91,23 +95,22 @@ func main() {
var ( var (
err error err error
errInfo string errInfo string
disruptions bool
) )
allSuccess := true allSuccess := true
disruptions := true
for _, check := range checks { for _, check := range checks {
errInfo, err = check.function() err = check.function()
if err != nil { if err != nil {
component.ErrorPrint(fmt.Sprintf("Starting %s failed, %v, the conneted info is:%s", check.name, err, errInfo)) if errorJudge(err, check.authInfo) {
fmt.Fprintln(os.Stderr, errInfo, err) disruptions = true
}
ErrorPrint(fmt.Sprintf("Starting %s failed, %v, the conneted info is:%s", check.name, err, errInfo))
allSuccess = false allSuccess = false
break break
} else { } else {
component.SuccessPrint(fmt.Sprintf("%s connected successfully, the addr is:%s", check.name, errInfo)) component.SuccessPrint(fmt.Sprintf("%s connected successfully, the addr is:%s", check.name, errInfo))
fmt.Fprintln(os.Stderr, errInfo, err)
}
if check.name == "kafka" && errs.Unwrap(err) == ErrComponentStart {
disruptions = false
} }
} }
if allSuccess { if allSuccess {
@ -117,76 +120,50 @@ func main() {
if disruptions { if disruptions {
component.ErrorPrint(fmt.Sprintf("component check exit,err: %v", err)) component.ErrorPrint(fmt.Sprintf("component check exit,err: %v", err))
fmt.Fprintln(os.Stderr, errInfo, err)
return return
} }
} }
} }
// checkMongo checks the MongoDB connection without retries // checkMongo checks the MongoDB connection without retries
func checkMongo() (string, error) { func checkMongo() error {
_, err := unrelation.NewMongo() _, err := unrelation.NewMongo()
if err != nil { return err
if config.Config.Mongo.Uri != "" {
return config.Config.Mongo.Uri, err
}
uriFormat := "mongodb://%s/%s?maxPoolSize=%s"
if config.Config.Mongo.Username != "" && config.Config.Mongo.Password != "" {
uriFormat = "mongodb://%s:%s@%s/%s?maxPoolSize=%s"
return fmt.Sprintf(uriFormat, config.Config.Mongo.Username, config.Config.Mongo.Password, config.Config.Mongo.Address, config.Config.Mongo.Database, config.Config.Mongo.MaxPoolSize), err
}
return fmt.Sprintf(uriFormat, config.Config.Mongo.Address, config.Config.Mongo.Database, config.Config.Mongo.MaxPoolSize), err
}
return strings.Join(config.Config.Mongo.Address, ","), nil
} }
// checkRedis checks the Redis connection // checkRedis checks the Redis connection
func checkRedis() (string, error) { func checkRedis() error {
_, err := cache.NewRedis() _, err := cache.NewRedis()
if err != nil { return err
uriFormat := "The username is:%s, the password is:%s, the address is:%s, the clusterMode is:%t"
return fmt.Sprintf(uriFormat, config.Config.Redis.Username, config.Config.Redis.Password, config.Config.Redis.Address, config.Config.Redis.ClusterMode), err
}
return strings.Join(config.Config.Redis.Address, ","), err
} }
// checkMinio checks the MinIO connection // checkMinio checks the MinIO connection
func checkMinio() (string, error) { func checkMinio() error {
rdb, err := cache.NewRedis() // Check if MinIO is enabled
if config.Config.Object.Enable != "minio" {
enable := config.Config.Object.Enable return nil
switch config.Config.Object.Enable {
case "minio":
_, err = minio.NewMinio(cache.NewMinioCache(rdb))
case "cos":
_, err = cos.NewCos()
case "oss":
_, err = oss.NewOSS()
default:
err = fmt.Errorf("invalid object enable: %s", enable)
} }
if err != nil { minio := &component.Minio{
uriFormat := "The apiURL is:%s, the endpoint is:%s, the signEndpoint is:%s." ApiURL: config.Config.Object.ApiURL,
return fmt.Sprintf(uriFormat, config.Config.Object.ApiURL, config.Config.Object.Minio.Endpoint, config.Config.Object.Minio.SignEndpoint), err Endpoint: config.Config.Object.Minio.Endpoint,
AccessKeyID: config.Config.Object.Minio.AccessKeyID,
SecretAccessKey: config.Config.Object.Minio.SecretAccessKey,
SignEndpoint: config.Config.Object.Minio.SignEndpoint,
UseSSL: getEnv("MINIO_USE_SSL", "false"),
} }
return config.Config.Object.Minio.Endpoint, nil _, err := component.CheckMinio(minio)
return err
} }
// checkZookeeper checks the Zookeeper connection // checkZookeeper checks the Zookeeper connection
func checkZookeeper() (string, error) { func checkZookeeper() error {
_, err := zookeeper.NewZookeeperDiscoveryRegister() _, err := zookeeper.NewZookeeperDiscoveryRegister()
if err != nil { return err
if config.Config.Zookeeper.Username != "" && config.Config.Zookeeper.Password != "" {
return fmt.Sprintf("The addr is:%s,the schema is:%s, the username is:%s, the password is:%s.", config.Config.Zookeeper.ZkAddr, config.Config.Zookeeper.Schema, config.Config.Zookeeper.Username, config.Config.Zookeeper.Password), err
}
return fmt.Sprintf("The addr is:%s,the schema is:%s", config.Config.Zookeeper.ZkAddr, config.Config.Zookeeper.Schema), err
}
return strings.Join(config.Config.Zookeeper.ZkAddr, ","), nil
} }
// checkKafka checks the Kafka connection // checkKafka checks the Kafka connection
func checkKafka() (string, error) { func checkKafka() error {
// Prioritize environment variables // Prioritize environment variables
kafkaStu := &component.Kafka{ kafkaStu := &component.Kafka{
@ -195,16 +172,16 @@ func checkKafka() (string, error) {
Addr: config.Config.Kafka.Addr, Addr: config.Config.Kafka.Addr,
} }
str, kafkaClient, err := component.CheckKafka(kafkaStu) _, kafkaClient, err := component.CheckKafka(kafkaStu)
if err != nil { if err != nil {
return "", err return err
} }
defer kafkaClient.Close() defer kafkaClient.Close()
// Verify if necessary topics exist // Verify if necessary topics exist
topics, err := kafkaClient.Topics() topics, err := kafkaClient.Topics()
if err != nil { if err != nil {
return "", errs.Wrap(err) return errs.Wrap(err)
} }
requiredTopics := []string{ requiredTopics := []string{
@ -215,29 +192,38 @@ func checkKafka() (string, error) {
for _, requiredTopic := range requiredTopics { for _, requiredTopic := range requiredTopics {
if !isTopicPresent(requiredTopic, topics) { if !isTopicPresent(requiredTopic, topics) {
return "", ErrComponentStart.Wrap(fmt.Sprintf("Kafka doesn't contain topic: %v", requiredTopic)) return errs.Wrap(err, fmt.Sprintf("Kafka doesn't contain topic: %v", requiredTopic))
} }
} }
kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{ _, err = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
KafkaVersion: sarama.V2_0_0_0, KafkaVersion: sarama.V2_0_0_0,
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
}, []string{config.Config.Kafka.LatestMsgToRedis.Topic}, }, []string{config.Config.Kafka.LatestMsgToRedis.Topic},
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToRedis) config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToRedis)
if err != nil {
return err
}
kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{ _, err = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
KafkaVersion: sarama.V2_0_0_0, KafkaVersion: sarama.V2_0_0_0,
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
}, []string{config.Config.Kafka.MsgToMongo.Topic}, }, []string{config.Config.Kafka.MsgToPush.Topic},
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongo) config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToMongo)
if err != nil {
return err
}
kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{ kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
KafkaVersion: sarama.V2_0_0_0, KafkaVersion: sarama.V2_0_0_0,
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false, OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
}, []string{config.Config.Kafka.MsgToPush.Topic}, config.Config.Kafka.Addr, }, []string{config.Config.Kafka.MsgToPush.Topic}, config.Config.Kafka.Addr,
config.Config.Kafka.ConsumerGroupID.MsgToPush) config.Config.Kafka.ConsumerGroupID.MsgToPush)
if err != nil {
return err
}
return str, nil return nil
} }
// isTopicPresent checks if a topic is present in the list of topics // isTopicPresent checks if a topic is present in the list of topics
@ -251,13 +237,27 @@ func isTopicPresent(topic string, topics []string) bool {
} }
func configGetEnv() { func configGetEnv() {
config.Config.Object.Minio.AccessKeyID = getEnv("MINIO_ACCESS_KEY_ID", config.Config.Object.Minio.AccessKeyID)
config.Config.Object.Minio.SecretAccessKey = getEnv("MINIO_SECRET_ACCESS_KEY", config.Config.Object.Minio.SecretAccessKey)
config.Config.Mongo.Uri = getEnv("MONGO_URI", config.Config.Mongo.Uri) config.Config.Mongo.Uri = getEnv("MONGO_URI", config.Config.Mongo.Uri)
config.Config.Mongo.Username = getEnv("MONGO_OPENIM_USERNAME", config.Config.Mongo.Username) config.Config.Mongo.Username = getEnv("MONGO_OPENIM_USERNAME", config.Config.Mongo.Username)
config.Config.Mongo.Password = getEnv("MONGO_OPENIM_PASSWORD", config.Config.Mongo.Password) config.Config.Mongo.Password = getEnv("MONGO_OPENIM_PASSWORD", config.Config.Mongo.Password)
config.Config.Kafka.Username = getEnv("KAFKA_USERNAME", config.Config.Kafka.Username) config.Config.Kafka.Username = getEnv("KAFKA_USERNAME", config.Config.Kafka.Username)
config.Config.Kafka.Password = getEnv("KAFKA_PASSWORD", config.Config.Kafka.Password) config.Config.Kafka.Password = getEnv("KAFKA_PASSWORD", config.Config.Kafka.Password)
config.Config.Kafka.Addr = strings.Split(getEnv("KAFKA_ADDRESS", strings.Join(config.Config.Kafka.Addr, ",")), ",") config.Config.Kafka.Addr = strings.Split(getEnv("KAFKA_ADDRESS", strings.Join(config.Config.Kafka.Addr, ",")), ",")
config.Config.Object.Minio.Endpoint = getMinioAddr("MINIO_ENDPOINT", "MINIO_ADDRESS", "MINIO_PORT", config.Config.Object.Minio.Endpoint)
}
func getMinioAddr(key1, key2, key3, fallback string) string {
// Prioritize environment variables
endpoint := getEnv(key1, fallback)
address, addressExist := os.LookupEnv(key2)
port, portExist := os.LookupEnv(key3)
if portExist && addressExist {
endpoint = "http://" + address + ":" + port
return endpoint
}
return endpoint
} }
// Helper function to get environment variable or default value // Helper function to get environment variable or default value
@ -267,3 +267,18 @@ func getEnv(key, fallback string) string {
} }
return fallback return fallback
} }
func ErrorPrint(s string) {
colorPrint(colorRed, "%v", s)
}
func colorPrint(colorCode int, format string, a ...interface{}) {
log.Printf("\x1b[%dm%s\x1b[0m\n", colorCode, fmt.Sprintf(format, a...))
}
func errorJudge(err error, errMsg string) bool {
if strings.Contains(errors.Unwrap(err).Error(), errMsg) {
return true
}
return false
}

Loading…
Cancel
Save