From 230c0dbb8be41909f9f0e692e6322abb7232b993 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:02:46 +0800 Subject: [PATCH 1/5] fix: minio config build (#2263) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error --------- Co-authored-by: withchao --- pkg/common/config/config.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index caf31036c..24d04d8cc 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,14 +15,13 @@ package config import ( - "fmt" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "net" + "strings" "time" ) @@ -473,25 +472,23 @@ func (k *Kafka) Build() *kafka.Config { }, } } + func (m *Minio) Build() *minio.Config { - conf := minio.Config{ + formatEndpoint := func(address string) string { + if strings.HasPrefix(address, "http://") || strings.HasPrefix(address, "https://") { + return address + } + return "http://" + address + } + return &minio.Config{ Bucket: m.Bucket, AccessKeyID: m.AccessKeyID, SecretAccessKey: m.SecretAccessKey, SessionToken: m.SessionToken, PublicRead: m.PublicRead, + Endpoint: formatEndpoint(m.InternalAddress), + SignEndpoint: formatEndpoint(m.ExternalAddress), } - if _, _, err := net.SplitHostPort(m.InternalAddress); err == nil { - conf.Endpoint = fmt.Sprintf("http://%s", m.InternalAddress) - } else { - conf.Endpoint = m.InternalAddress - } - if _, _, err := net.SplitHostPort(m.ExternalAddress); err == nil { - conf.SignEndpoint = fmt.Sprintf("http://%s", m.ExternalAddress) - } else { - conf.SignEndpoint = m.ExternalAddress - } - return &conf } func (c *Cos) Build() *cos.Config { return &cos.Config{ From 691cf740af346ca17ebdc1cfc6a5f41a587e4d8a Mon Sep 17 00:00:00 2001 From: blooming <37789413+Bloomingg@users.noreply.github.com> Date: Mon, 6 May 2024 16:37:52 +0800 Subject: [PATCH 2/5] fix: avoid frequent scheduled task (#2274) --- .github/workflows/build-docker-image.yml | 3 +-- .github/workflows/{lock-issue.yml => lock-issue.bak} | 0 .github/workflows/stale.yml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) rename .github/workflows/{lock-issue.yml => lock-issue.bak} (100%) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 1e9711d2c..b4733116e 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -15,8 +15,6 @@ name: Publish Docker image on: - schedule: - - cron: '30 2 * * *' push: branches: - main @@ -31,6 +29,7 @@ env: jobs: build-dockerhub: + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true) runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/lock-issue.yml b/.github/workflows/lock-issue.bak similarity index 100% rename from .github/workflows/lock-issue.yml rename to .github/workflows/lock-issue.bak diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index da44cb7f3..4445f6dda 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -21,7 +21,7 @@ name: Mark stale issues and pull requests on: schedule: - - cron: '0 8 * * *' + - cron: '0 8 * * 1' jobs: stale: @@ -36,7 +36,7 @@ jobs: with: repo-token: ${{ secrets.BOT_GITHUB_TOKEN }} days-before-stale: 60 - days-before-close: 7 + days-before-close: 305 stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days.' stale-pr-message: 'This issue is stale because it has been open 60 days with no activity.' close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.' From 8d84e2f01f87e2f3e8d9338362ce5a7a4c286c2a Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 7 May 2024 21:05:41 +0800 Subject: [PATCH 3/5] Skip minio check (#2281) * If MinIO is not being used, then do not perform the MinIO check. * If MinIO is not being used, then do not perform the MinIO check. * If MinIO is not being used, then do not perform the MinIO check. * If MinIO is not being used, then do not perform the MinIO check. * kill binary before build * Stop the process before compiling. * Stop the process before compiling. --- docker-compose.yml | 3 +++ go.mod | 3 ++- go.sum | 4 ++-- internal/tools/cron_task.go | 14 +------------- magefile.go | 12 +----------- tools/check-component/main.go | 20 ++++++++++++++++---- 6 files changed, 25 insertions(+), 31 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4acd00cdd..aeb53a417 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,7 @@ services: ports: - "12181:2181" environment: + #JVMFLAGS: "-Xms32m -Xmx128m" TZ: "Asia/Shanghai" ALLOW_ANONYMOUS_LOGIN: "yes" restart: always @@ -70,6 +71,7 @@ services: command: > bash -c "/opt/bitnami/scripts/kafka/run.sh & /opt/bitnami/kafka/create-topic.sh; wait" environment: + #KAFKA_HEAP_OPTS: "-Xms128m -Xmx256m" TZ: Asia/Shanghai KAFKA_CFG_NODE_ID: 0 KAFKA_CFG_PROCESS_ROLES: controller,broker @@ -119,3 +121,4 @@ services: - openim + diff --git a/go.mod b/go.mod index cc0de7614..b522065da 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.11 + github.com/openimsdk/gomake v0.0.13 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible @@ -44,6 +44,7 @@ require ( golang.org/x/sync v0.6.0 ) + require ( cloud.google.com/go v0.112.0 // indirect cloud.google.com/go/compute v1.23.3 // indirect diff --git a/go.sum b/go.sum index 7c256c58d..e2ecf7284 100644 --- a/go.sum +++ b/go.sum @@ -279,8 +279,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/openimsdk/gomake v0.0.11 h1:jJ9286zKFfBeARkmfqMEcUYg9lJ+Cj9lylxP8W9uCFM= -github.com/openimsdk/gomake v0.0.11/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= +github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.2 h1:8IfV6o2ySU7C54sh/MG7ctEp1h3lSNe03OCUDWSk5Ws= diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 7161f55fc..20baeffaf 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -25,8 +25,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "os" - "os/signal" - "syscall" "time" "github.com/openimsdk/tools/errs" @@ -50,22 +48,12 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return errs.WrapMsg(err, "failed to register discovery service") } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) - ctx, exitBy := context.WithCancelCause(context.Background()) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) conn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) if err != nil { return err } cli := msg.NewMsgClient(conn) - go func() { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - select { - case <-ctx.Done(): - case s := <-sigs: - exitBy(fmt.Errorf("exit signal %s", s)) - } - }() crontab := cron.New() clearFunc := func() { now := time.Now() @@ -84,5 +72,5 @@ func Start(ctx context.Context, config *CronTaskConfig) error { log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) crontab.Start() <-ctx.Done() - return context.Cause(ctx) + return nil } diff --git a/magefile.go b/magefile.go index 98ffa48f3..a8a1c4040 100644 --- a/magefile.go +++ b/magefile.go @@ -6,22 +6,12 @@ package main import ( "github.com/openimsdk/gomake/mageutil" "os" - "strings" ) var Default = Build func Build() { - platforms := os.Getenv("PLATFORMS") - if platforms == "" { - platforms = mageutil.DetectPlatform() - } - - for _, platform := range strings.Split(platforms, " ") { - mageutil.CompileForPlatform(platform) - } - - mageutil.PrintGreen("All binaries under cmd and tools were successfully compiled.") + mageutil.Build() } func Start() { diff --git a/tools/check-component/main.go b/tools/check-component/main.go index d3225abb8..7fe64d3c5 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -66,6 +66,7 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, kafkaConfig = &config.Kafka{} minioConfig = &config.Minio{} zookeeperConfig = &config.ZooKeeper{} + thirdConfig = &config.Third{} ) err := config.LoadConfig(filepath.Join(configDir, cmd.MongodbConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], mongoConfig) if err != nil { @@ -82,11 +83,19 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.MinioConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], minioConfig) + err = config.LoadConfig(filepath.Join(configDir, cmd.OpenIMRPCThirdCfgFileName), cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } + if thirdConfig.Object.Enable == "minio" { + err = config.LoadConfig(filepath.Join(configDir, cmd.MinioConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], minioConfig) + if err != nil { + return nil, nil, nil, nil, nil, err + } + } else { + minioConfig = nil + } err = config.LoadConfig(filepath.Join(configDir, cmd.ZookeeperConfigFileName), cmd.ConfigEnvPrefixMap[cmd.ZookeeperConfigFileName], zookeeperConfig) if err != nil { return nil, nil, nil, nil, nil, err @@ -131,14 +140,17 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * "Redis": func() error { return CheckRedis(ctx, redisConfig) }, - "MinIO": func() error { - return CheckMinIO(ctx, minioConfig) - }, "Kafka": func() error { return CheckKafka(ctx, kafkaConfig) }, } + if minioConfig != nil { + checks["MinIO"] = func() error { + return CheckMinIO(ctx, minioConfig) + } + } + for i := 0; i < maxRetry; i++ { allSuccess := true for name, check := range checks { From 5ba5402bee04d73d46a04c3461ae15a711f9abce Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Wed, 8 May 2024 16:00:27 +0800 Subject: [PATCH 4/5] update gomake (#2286) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 746dddf65..3f765805c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.9 +RUN go get github.com/openimsdk/gomake@v0.0.13 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] From 66426cd98b5784e94340a5c85a950416af24b9a2 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 8 May 2024 19:38:27 +0800 Subject: [PATCH 5/5] fix: search for messag (#2288) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * fix: SearchMessage --------- Co-authored-by: withchao --- pkg/common/db/mgo/msg.go | 122 +++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 50 deletions(-) diff --git a/pkg/common/db/mgo/msg.go b/pkg/common/db/mgo/msg.go index 6fe24536b..17e493d33 100644 --- a/pkg/common/db/mgo/msg.go +++ b/pkg/common/db/mgo/msg.go @@ -267,58 +267,80 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do } func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*relation.MsgInfoModel, error) { - var pipe mongo.Pipeline - condition := bson.A{} - if req.SendTime != "" { - // Changed to keyed fields for bson.M to avoid govet errors - condition = append(condition, bson.M{"$eq": bson.A{bson.M{"$dateToString": bson.M{"format": "%Y-%m-%d", "date": bson.M{"$toDate": "$$item.msg.send_time"}}}, req.SendTime}}) + where := make(bson.A, 0, 6) + if req.RecvID != "" { + where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) + } + if req.SendID != "" { + where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) } if req.ContentType != 0 { - condition = append(condition, bson.M{"$eq": bson.A{"$$item.msg.content_type", req.ContentType}}) + where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) } if req.SessionType != 0 { - condition = append(condition, bson.M{"$eq": bson.A{"$$item.msg.session_type", req.SessionType}}) + where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) } - if req.RecvID != "" { - condition = append(condition, bson.M{"$regexFind": bson.M{"input": "$$item.msg.recv_id", "regex": req.RecvID}}) + if req.SendTime != "" { + sendTime, err := time.Parse(time.DateOnly, req.SendTime) + if err != nil { + return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) + } + where = append(where, + bson.M{ + "msgs.msg.send_time": bson.M{ + "$gte": sendTime.UnixMilli(), + }, + }, + bson.M{ + "msgs.msg.send_time": bson.M{ + "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), + }, + }, + ) } - if req.SendID != "" { - condition = append(condition, bson.M{"$regexFind": bson.M{"input": "$$item.msg.send_id", "regex": req.SendID}}) + pipeline := bson.A{ + bson.M{ + "$unwind": "$msgs", + }, } - - or := bson.A{ - bson.M{"doc_id": bson.M{"$regex": "^si_", "$options": "i"}}, - bson.M{"doc_id": bson.M{"$regex": "^g_", "$options": "i"}}, - bson.M{"doc_id": bson.M{"$regex": "^sg_", "$options": "i"}}, + if len(where) > 0 { + pipeline = append(pipeline, bson.M{ + "$match": bson.M{"$and": where}, + }) } - - // Use bson.D with keyed fields to specify the order explicitly - pipe = mongo.Pipeline{ - {{"$match", bson.D{{Key: "$or", Value: or}}}}, - {{"$project", bson.D{ - {Key: "msgs", Value: bson.D{ - {Key: "$filter", Value: bson.D{ - {Key: "input", Value: "$msgs"}, - {Key: "as", Value: "item"}, - {Key: "cond", Value: bson.D{{Key: "$and", Value: condition}}}, - }}, - }}, - {Key: "doc_id", Value: 1}, - }}}, - {{"$unwind", bson.M{"path": "$msgs"}}}, - {{"$sort", bson.M{"msgs.msg.send_time": -1}}}, + pipeline = append(pipeline, + bson.M{ + "$project": bson.M{ + "_id": 0, + "msg": "$msgs.msg", + }, + }, + bson.M{ + "$count": "count", + }, + ) + count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) + if err != nil { + return 0, nil, err } - type docModel struct { - DocID string `bson:"doc_id"` - Msg *relation.MsgInfoModel `bson:"msgs"` + if len(count) == 0 || count[0] == 0 { + return 0, nil, nil } - msgsDocs, err := mongoutil.Aggregate[*docModel](ctx, m.coll, pipe) + pipeline = pipeline[:len(pipeline)-1] + pipeline = append(pipeline, + bson.M{ + "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), + }, + bson.M{ + "$limit": req.Pagination.GetShowNumber(), + }, + ) + msgs, err := mongoutil.Aggregate[*relation.MsgInfoModel](ctx, m.coll, pipeline) if err != nil { return 0, nil, err } - msgs := make([]*relation.MsgInfoModel, 0) - for _, doc := range msgsDocs { - msgInfo := doc.Msg + for i := range msgs { + msgInfo := msgs[i] if msgInfo == nil || msgInfo.Msg == nil { continue } @@ -350,17 +372,17 @@ func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) ( } msgs = append(msgs, msgInfo) } - start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber - n := int32(len(msgs)) - if start >= n { - return n, []*relation.MsgInfoModel{}, nil - } - if start+req.Pagination.ShowNumber < n { - msgs = msgs[start : start+req.Pagination.ShowNumber] - } else { - msgs = msgs[start:] - } - return n, msgs, nil + //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber + //n := int32(len(msgs)) + //if start >= n { + // return n, []*relation.MsgInfoModel{}, nil + //} + //if start+req.Pagination.ShowNumber < n { + // msgs = msgs[start : start+req.Pagination.ShowNumber] + //} else { + // msgs = msgs[start:] + //} + return count[0], msgs, nil } func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*relation.UserCount, dateCount map[string]int64, err error) {