From 15648cd44d39f7db4bd474a5ccff4cdc66f0bcc6 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:35:39 +0800 Subject: [PATCH 01/12] fix: concurrent write to websocket connection (#2866) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- internal/msggateway/client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index af96e7d46..6e31751dc 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -446,6 +446,8 @@ func (c *Client) handlerTextMessage(b []byte) error { if err != nil { return err } + c.w.Lock() + defer c.w.Unlock() if err := c.conn.SetWriteDeadline(writeWait); err != nil { return err } From daba153d27f86af34630804d5d53144e67afa93d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:25:28 +0800 Subject: [PATCH 02/12] fix: admin token limit (#2871) --- internal/rpc/auth/auth.go | 10 +++- pkg/common/storage/controller/auth.go | 69 +++++++++++++++------------ pkg/common/storage/controller/msg.go | 2 +- 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 62df74d21..a1acfd931 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -16,6 +16,7 @@ package auth import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -66,6 +67,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config.Share.Secret, config.RpcConfig.TokenPolicy.Expire, config.Share.MultiLogin, + config.Share.IMAdminUserID, ), config: config, }) @@ -129,6 +131,10 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim if err != nil { return nil, errs.Wrap(err) } + isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + if isAdmin { + return claims, nil + } m, err := s.authDatabase.GetTokensWithoutError(ctx, claims.UserID, claims.PlatformID) if err != nil { return nil, err @@ -190,7 +196,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID } m, err := s.authDatabase.GetTokensWithoutError(ctx, userID, int(platformID)) - if err != nil && err != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return err } for k := range m { @@ -208,7 +214,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID func (s *authServer) InvalidateToken(ctx context.Context, req *pbauth.InvalidateTokenReq) (*pbauth.InvalidateTokenResp, error) { m, err := s.authDatabase.GetTokensWithoutError(ctx, req.UserID, int(req.PlatformID)) - if err != nil && err != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } if m == nil { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index df1274967..e7b5bc297 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -35,9 +35,10 @@ type authDatabase struct { accessSecret string accessExpire int64 multiLogin multiLoginConfig + adminUserIDs []string } -func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin) AuthDatabase { +func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin, adminUserIDs []string) AuthDatabase { return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{ Policy: multiLogin.Policy, MaxNumOneEnd: multiLogin.MaxNumOneEnd, @@ -53,7 +54,8 @@ func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire i constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad, constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin, }, - }} + }, adminUserIDs: adminUserIDs, + } } // If the result is empty. @@ -90,27 +92,31 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) - if err != nil { - return "", err - } - deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) - if err != nil { - return "", err - } - if len(deleteTokenKey) != 0 { - err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) + isAdmin := authverify.IsManagerUserID(userID, a.adminUserIDs) + if !isAdmin { + tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) if err != nil { return "", err } - } - if len(kickedTokenKey) != 0 { - for _, k := range kickedTokenKey { - err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) + + deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + if err != nil { + return "", err + } + if len(deleteTokenKey) != 0 { + err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) if err != nil { return "", err } - log.ZDebug(ctx, "kicked token in create token", "token", k) + } + if len(kickedTokenKey) != 0 { + for _, k := range kickedTokenKey { + err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) + if err != nil { + return "", err + } + log.ZDebug(ctx, "kicked token in create token", "token", k) + } } } @@ -121,9 +127,12 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", errs.WrapMsg(err, "token.SignedString") } - if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err + if !isAdmin { + if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { + return "", err + } } + return tokenString, nil } @@ -226,16 +235,16 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string return nil, nil, errs.New("unknown multiLogin policy").Wrap() } - var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd - if a.multiLogin.Policy == constant.Customize { - adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] - } - l := len(adminToken) - if platformID == constant.AdminPlatformID { - l++ - } - if l > adminTokenMaxNum { - kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) - } + //var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd + //if a.multiLogin.Policy == constant.Customize { + // adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] + //} + //l := len(adminToken) + //if platformID == constant.AdminPlatformID { + // l++ + //} + //if l > adminTokenMaxNum { + // kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) + //} return deleteToken, kickToken, nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 90b479064..9636f7a15 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -490,7 +490,7 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co } successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) if err != nil { - if err != redis.Nil { + if errors.Is(err, redis.Nil) { log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) } } From a37eb5ea0dc123ef4175ef3ad52bfdb62f5564a6 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 22 Nov 2024 12:26:29 +0800 Subject: [PATCH 03/12] build: create changelog tool and workflows. (#2869) --- .github/workflows/changelog.yml | 78 ++++++++ tools/changelog/changelog.go | 198 ++++++++++++++++++++ tools/changelog/main.go | 308 -------------------------------- 3 files changed, 276 insertions(+), 308 deletions(-) create mode 100644 .github/workflows/changelog.yml create mode 100644 tools/changelog/changelog.go delete mode 100644 tools/changelog/main.go diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 000000000..b97036d91 --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,78 @@ +name: Release Changelog + +on: + release: + types: [released] + +permissions: + contents: write + pull-requests: write + +jobs: + update-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Go Changelog Generator + run: | + # Run the Go changelog generator, passing the release tag if available + if [ "${{ github.event.release.tag_name }}" = "latest" ]; then + go run tools/changelog/changelog.go > "${{ github.event.release.tag_name }}-changelog.md" + else + go run tools/changelog/changelog.go "${{ github.event.release.tag_name }}" > "${{ github.event.release.tag_name }}-changelog.md" + fi + + - name: Handle changelog files + run: | + # Ensure that the CHANGELOG directory exists + mkdir -p CHANGELOG + + # Extract Major.Minor version by removing the 'v' prefix from the tag name + TAG_NAME=${{ github.event.release.tag_name }} + CHANGELOG_VERSION_NUMBER=$(echo "$TAG_NAME" | sed 's/^v//' | grep -oP '^\d+\.\d+') + + # Define the new changelog file path + CHANGELOG_FILENAME="CHANGELOG-$CHANGELOG_VERSION_NUMBER.md" + CHANGELOG_PATH="CHANGELOG/$CHANGELOG_FILENAME" + + # Check if the changelog file for the current release already exists + if [ -f "$CHANGELOG_PATH" ]; then + # If the file exists, append the new changelog to the existing one + cat "$CHANGELOG_PATH" >> "${TAG_NAME}-changelog.md" + # Overwrite the existing changelog with the updated content + mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH" + else + # If the changelog file doesn't exist, rename the temp changelog file to the new changelog file + mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH" + + # Ensure that README.md exists + if [ ! -f "CHANGELOG/README.md" ]; then + echo -e "# CHANGELOGs\n\n" > CHANGELOG/README.md + fi + + # Add the new changelog entry at the top of the README.md + if ! grep -q "\[$CHANGELOG_FILENAME\]" CHANGELOG/README.md; then + sed -i "3i- [$CHANGELOG_FILENAME](./$CHANGELOG_FILENAME)" CHANGELOG/README.md + # Remove the extra newline character added by sed + # sed -i '4d' CHANGELOG/README.md + fi + fi + + - name: Clean up + run: | + # Remove any temporary files that were created during the process + rm -f "${{ github.event.release.tag_name }}-changelog.md" + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7.0.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update CHANGELOG for release ${{ github.event.release.tag_name }}" + title: "Update CHANGELOG for release ${{ github.event.release.tag_name }}" + body: "This PR updates the CHANGELOG files for release ${{ github.event.release.tag_name }}" + branch: changelog-${{ github.event.release.tag_name }} + base: main + delete-branch: true + labels: changelog diff --git a/tools/changelog/changelog.go b/tools/changelog/changelog.go new file mode 100644 index 000000000..75d914a27 --- /dev/null +++ b/tools/changelog/changelog.go @@ -0,0 +1,198 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" +) + +// You can specify a tag as a command line argument to generate the changelog for a specific version. +// Example: go run tools/changelog/changelog.go v0.0.33 +// If no tag is provided, the latest release will be used. + +// Setting repo owner and repo name by generate changelog +const ( + repoOwner = "openimsdk" + repoName = "open-im-server" +) + +// GitHubRepo struct represents the repo details. +type GitHubRepo struct { + Owner string + Repo string + FullChangelog string +} + +// ReleaseData represents the JSON structure for release data. +type ReleaseData struct { + TagName string `json:"tag_name"` + Body string `json:"body"` + HtmlUrl string `json:"html_url"` + Published string `json:"published_at"` +} + +// Method to classify and format release notes. +func (g *GitHubRepo) classifyReleaseNotes(body string) map[string][]string { + result := map[string][]string{ + "feat": {}, + "fix": {}, + "chore": {}, + "refactor": {}, + "build": {}, + "other": {}, + } + + // Regular expression to extract PR number and URL (case insensitive) + rePR := regexp.MustCompile(`(?i)in (https://github\.com/[^\s]+/pull/(\d+))`) + + // Split the body into individual lines. + lines := strings.Split(body, "\n") + + for _, line := range lines { + // Skip lines that contain "deps: Merge" + if strings.Contains(strings.ToLower(line), "deps: merge #") { + continue + } + + // Use a regular expression to extract Full Changelog link and its title (case insensitive). + if strings.Contains(strings.ToLower(line), "**full changelog**") { + matches := regexp.MustCompile(`(?i)\*\*full changelog\*\*: (https://github\.com/[^\s]+/compare/([^\s]+))`).FindStringSubmatch(line) + if len(matches) > 2 { + // Format the Full Changelog link with title + g.FullChangelog = fmt.Sprintf("[%s](%s)", matches[2], matches[1]) + } + continue // Skip further processing for this line. + } + + if strings.HasPrefix(line, "*") { + var category string + + // Use strings.ToLower to make the matching case insensitive + lowerLine := strings.ToLower(line) + + // Determine the category based on the prefix (case insensitive). + if strings.HasPrefix(lowerLine, "* feat") { + category = "feat" + } else if strings.HasPrefix(lowerLine, "* fix") { + category = "fix" + } else if strings.HasPrefix(lowerLine, "* chore") { + category = "chore" + } else if strings.HasPrefix(lowerLine, "* refactor") { + category = "refactor" + } else if strings.HasPrefix(lowerLine, "* build") { + category = "build" + } else { + category = "other" + } + + // Extract PR number and URL (case insensitive) + matches := rePR.FindStringSubmatch(line) + if len(matches) == 3 { + prURL := matches[1] + prNumber := matches[2] + // Format the line with the PR link and use original content for the final result + formattedLine := fmt.Sprintf("* %s [#%s](%s)", strings.Split(line, " by ")[0][2:], prNumber, prURL) + result[category] = append(result[category], formattedLine) + } else { + // If no PR link is found, just add the line as is + result[category] = append(result[category], line) + } + } + } + + return result +} + +// Method to generate the final changelog. +func (g *GitHubRepo) generateChangelog(tag, date, htmlURL, body string) string { + sections := g.classifyReleaseNotes(body) + + // Convert ISO 8601 date to simpler format (YYYY-MM-DD) + formattedDate := date[:10] + + // Changelog header with tag, date, and links. + changelog := fmt.Sprintf("## [%s](%s) \t(%s)\n\n", tag, htmlURL, formattedDate) + + if len(sections["feat"]) > 0 { + changelog += "### New Features\n" + strings.Join(sections["feat"], "\n") + "\n\n" + } + if len(sections["fix"]) > 0 { + changelog += "### Bug Fixes\n" + strings.Join(sections["fix"], "\n") + "\n\n" + } + if len(sections["chore"]) > 0 { + changelog += "### Chores\n" + strings.Join(sections["chore"], "\n") + "\n\n" + } + if len(sections["refactor"]) > 0 { + changelog += "### Refactors\n" + strings.Join(sections["refactor"], "\n") + "\n\n" + } + if len(sections["build"]) > 0 { + changelog += "### Builds\n" + strings.Join(sections["build"], "\n") + "\n\n" + } + if len(sections["other"]) > 0 { + changelog += "### Others\n" + strings.Join(sections["other"], "\n") + "\n\n" + } + + if g.FullChangelog != "" { + changelog += fmt.Sprintf("**Full Changelog**: %s\n", g.FullChangelog) + } + + return changelog +} + +// Method to fetch release data from GitHub API. +func (g *GitHubRepo) fetchReleaseData(version string) (*ReleaseData, error) { + var apiURL string + + if version == "" { + // Fetch the latest release. + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", g.Owner, g.Repo) + } else { + // Fetch a specific version. + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", g.Owner, g.Repo, version) + } + + resp, err := http.Get(apiURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var releaseData ReleaseData + err = json.Unmarshal(body, &releaseData) + if err != nil { + return nil, err + } + + return &releaseData, nil +} + +func main() { + repo := &GitHubRepo{Owner: repoOwner, Repo: repoName} + + // Get the version from command line arguments, if provided + var version string // Default is use latest + + if len(os.Args) > 1 { + version = os.Args[1] // Use the provided version + } + + // Fetch release data (either for latest or specific version) + releaseData, err := repo.fetchReleaseData(version) + if err != nil { + fmt.Println("Error fetching release data:", err) + return + } + + // Generate and print the formatted changelog + changelog := repo.generateChangelog(releaseData.TagName, releaseData.Published, releaseData.HtmlUrl, releaseData.Body) + fmt.Println(changelog) +} diff --git a/tools/changelog/main.go b/tools/changelog/main.go deleted file mode 100644 index ff9a7eab9..000000000 --- a/tools/changelog/main.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "regexp" - "sort" - "strings" -) - -var ( - mergeRequest = regexp.MustCompile(`Merge pull request #([\d]+)`) - webconsoleBump = regexp.MustCompile(regexp.QuoteMeta("bump(github.com/openshift/origin-web-console): ") + `([\w]+)`) - upstreamKube = regexp.MustCompile(`^UPSTREAM: (\d+)+:(.+)`) - upstreamRepo = regexp.MustCompile(`^UPSTREAM: ([\w/-]+): (\d+)+:(.+)`) - prefix = regexp.MustCompile(`^[\w-]: `) - - assignments = []prefixAssignment{ - {"cluster up", "cluster"}, - {" pv ", "storage"}, - {"haproxy", "router"}, - {"router", "router"}, - {"route", "route"}, - {"authoriz", "auth"}, - {"rbac", "auth"}, - {"authent", "auth"}, - {"reconcil", "auth"}, - {"auth", "auth"}, - {"role", "auth"}, - {" dc ", "deploy"}, - {"deployment", "deploy"}, - {"rolling", "deploy"}, - {"security context constr", "security"}, - {"scc", "security"}, - {"pipeline", "build"}, - {"build", "build"}, - {"registry", "registry"}, - {"registries", "image"}, - {"image", "image"}, - {" arp ", "network"}, - {" cni ", "network"}, - {"egress", "network"}, - {"network", "network"}, - {"oc ", "cli"}, - {"template", "template"}, - {"etcd", "server"}, - {"pod", "node"}, - {"scripts/", "hack"}, - {"e2e", "test"}, - {"integration", "test"}, - {"cluster", "cluster"}, - {"master", "server"}, - {"packages", "hack"}, - {"api", "server"}, - } -) - -type prefixAssignment struct { - term string - prefix string -} - -type commit struct { - short string - parents []string - message string -} - -func contains(arr []string, value string) bool { - for _, s := range arr { - if s == value { - return true - } - } - return false -} - -func main() { - log.SetFlags(0) - if len(os.Args) != 3 { - log.Fatalf("Must specify two arguments, FROM and TO") - } - from := os.Args[1] - to := os.Args[2] - - out, err := exec.Command("git", "log", "--topo-order", "--pretty=tformat:%h %p|%s", "--reverse", fmt.Sprintf("%s..%s", from, to)).CombinedOutput() - if err != nil { - log.Fatal(err) - } - - hide := make(map[string]struct{}) - var apiChanges []string - var webconsole []string - var commits []commit - var upstreams []commit - var bumps []commit - for _, line := range strings.Split(string(out), "\n") { - if len(strings.TrimSpace(line)) == 0 { - continue - } - parts := strings.SplitN(line, "|", 2) - hashes := strings.Split(parts[0], " ") - c := commit{short: hashes[0], parents: hashes[1:], message: parts[1]} - - if strings.HasPrefix(c.message, "UPSTREAM: ") { - hide[c.short] = struct{}{} - upstreams = append(upstreams, c) - } - if strings.HasPrefix(c.message, "bump(") { - hide[c.short] = struct{}{} - bumps = append(bumps, c) - } - - if len(c.parents) == 1 { - commits = append(commits, c) - continue - } - - matches := mergeRequest.FindStringSubmatch(line) - if len(matches) == 0 { - // this may have been a human pressing the merge button, we'll just record this as a direct push - continue - } - - // split the accumulated commits into any that are force merges (assumed to be the initial set due - // to --topo-order) from the PR commits as soon as we see any of our merge parents. Then print - // any of the force merges - var first int - for i := range commits { - first = i - if contains(c.parents, commits[i].short) { - first++ - break - } - } - individual := commits[:first] - merged := commits[first:] - for _, commit := range individual { - if len(commit.parents) > 1 { - continue - } - if _, ok := hide[commit.short]; ok { - continue - } - fmt.Printf("force-merge: %s %s\n", commit.message, commit.short) - } - - // try to find either the PR title or the first commit title from the merge commit - out, err := exec.Command("git", "show", "--pretty=tformat:%b", c.short).CombinedOutput() - if err != nil { - log.Fatal(err) - } - var message string - para := strings.Split(string(out), "\n\n") - if len(para) > 0 && strings.HasPrefix(para[0], "Automatic merge from submit-queue") { - para = para[1:] - } - // this is no longer necessary with the submit queue in place - if len(para) > 0 && strings.HasPrefix(para[0], "Merged by ") { - para = para[1:] - } - // post submit-queue, the merge bot will add the PR title, which is usually pretty good - if len(para) > 0 { - message = strings.Split(para[0], "\n")[0] - } - if len(message) == 0 && len(merged) > 0 { - message = merged[0].message - } - if len(message) > 0 && len(merged) == 1 && message == merged[0].message { - merged = nil - } - - // try to calculate a prefix based on the diff - if len(message) > 0 && !prefix.MatchString(message) { - prefix, ok := findPrefixFor(message, merged) - if ok { - message = prefix + ": " + message - } - } - - // github merge - - // has api changes - display := fmt.Sprintf("%s [\\#%s](https://github.com/openimsdk/Open-IM-Server/pull/%s)", message, matches[1], matches[1]) - if hasFileChanges(c.short, "pkg/apistruct/") { - apiChanges = append(apiChanges, display) - } - - var filtered []commit - for _, commit := range merged { - if _, ok := hide[commit.short]; ok { - continue - } - filtered = append(filtered, commit) - } - if len(filtered) > 0 { - fmt.Printf("- %s\n", display) - for _, commit := range filtered { - fmt.Printf(" - %s (%s)\n", commit.message, commit.short) - } - } - - // stick the merge commit in at the beginning of the next list so we can anchor the previous parent - commits = []commit{c} - } - - // chunk the bumps - var lines []string - for _, commit := range bumps { - if m := webconsoleBump.FindStringSubmatch(commit.message); len(m) > 0 { - webconsole = append(webconsole, m[1]) - continue - } - lines = append(lines, commit.message) - } - lines = sortAndUniq(lines) - for _, line := range lines { - fmt.Printf("- %s\n", line) - } - - // chunk the upstreams - lines = nil - for _, commit := range upstreams { - lines = append(lines, commit.message) - } - lines = sortAndUniq(lines) - for _, line := range lines { - fmt.Printf("- %s\n", upstreamLinkify(line)) - } - - if len(webconsole) > 0 { - fmt.Printf("- web: from %s^..%s\n", webconsole[0], webconsole[len(webconsole)-1]) - } - - for _, apiChange := range apiChanges { - fmt.Printf(" - %s\n", apiChange) - } -} - -func findPrefixFor(message string, commits []commit) (string, bool) { - message = strings.ToLower(message) - for _, m := range assignments { - if strings.Contains(message, m.term) { - return m.prefix, true - } - } - for _, c := range commits { - if prefix, ok := findPrefixFor(c.message, nil); ok { - return prefix, ok - } - } - return "", false -} - -func hasFileChanges(commit string, prefixes ...string) bool { - out, err := exec.Command("git", "diff", "--name-only", fmt.Sprintf("%s^..%s", commit, commit)).CombinedOutput() - if err != nil { - log.Fatal(err) - } - for _, file := range strings.Split(string(out), "\n") { - for _, prefix := range prefixes { - if strings.HasPrefix(file, prefix) { - return true - } - } - } - return false -} - -func sortAndUniq(lines []string) []string { - sort.Strings(lines) - out := make([]string, 0, len(lines)) - last := "" - for _, s := range lines { - if last == s { - continue - } - last = s - out = append(out, s) - } - return out -} - -func upstreamLinkify(line string) string { - if m := upstreamKube.FindStringSubmatch(line); len(m) > 0 { - return fmt.Sprintf("UPSTREAM: [#%s](https://github.com/openimsdk/open-im-server/pull/%s):%s", m[1], m[1], m[2]) - } - if m := upstreamRepo.FindStringSubmatch(line); len(m) > 0 { - return fmt.Sprintf("UPSTREAM: [%s#%s](https://github.com/%s/pull/%s):%s", m[1], m[2], m[1], m[2], m[3]) - } - return line -} From bf8853580024e22aa48fb1b6731415cbdbe63f94 Mon Sep 17 00:00:00 2001 From: Sasaya Date: Fri, 22 Nov 2024 14:45:24 +0800 Subject: [PATCH 04/12] fix: single chat offline push webhook (#2862) --- internal/push/push_handler.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ae308dfe9..b5bbed3ed 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -166,17 +166,21 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * return nil } } - offlinePushUserID := []string{msg.RecvID} + needOfflinePushUserID := []string{msg.RecvID} + var offlinePushUserID []string //receiver offline push - if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, - offlinePushUserID, msg, nil); err != nil { + if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserID, msg, &offlinePushUserID); err != nil { return err } log.ZInfo(ctx, "webhookBeforeOfflinePush end") - err = c.offlinePushMsg(ctx, msg, offlinePushUserID) + + if len(offlinePushUserID) > 0 { + needOfflinePushUserID = offlinePushUserID + } + err = c.offlinePushMsg(ctx, msg, needOfflinePushUserID) if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) + log.ZWarn(ctx, "offlinePushMsg failed", err, "needOfflinePushUserID", needOfflinePushUserID, "msg", msg) return nil } From 334749c5a2549f5f5619ace17274cf76925bc0e2 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:05:39 +0800 Subject: [PATCH 05/12] feat: Print Panic Log (#2850) * feat: catch panic * feat: docker file * feat: cicd * feat: dockerfile --------- Co-authored-by: Monet Lee --- .github/workflows/go-build-test.yml | 2 +- internal/api/router.go | 2 +- internal/msggateway/client.go | 6 ++++++ internal/msgtransfer/init.go | 5 +++++ internal/msgtransfer/online_history_msg_handler.go | 7 +++++++ internal/rpc/msg/send.go | 8 ++++++++ pkg/rpccache/subscriber.go | 6 ++++++ 7 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 5b37bf47d..2ca960cc9 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -149,7 +149,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_version: ["1.21"] + go_version: ["1.22"] steps: - name: Checkout Repository diff --git a/internal/api/router.go b/internal/api/router.go index 4ac301b07..8e4d17ef1 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -74,7 +74,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 6e31751dc..0da7d7220 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/openimsdk/tools/mw" "runtime/debug" "sync" "sync/atomic" @@ -375,6 +376,11 @@ func (c *Client) writeBinaryMsg(resp Resp) error { func (c *Client) activeHeartbeat(ctx context.Context) { if c.PlatformID == constant.WebPlatformID { go func() { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() log.ZDebug(ctx, "server initiative send heartbeat start.") ticker := time.NewTicker(pingPeriod) defer ticker.Stop() diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index f11cfde1a..92053931c 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -136,6 +136,11 @@ func (m *MsgTransfer) Start(index int, config *Config) error { if config.MsgTransfer.Prometheus.Enable { go func() { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(m.ctx, r) + } + }() prometheusPort, err := datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) if err != nil { netErr = err diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 84453c8df..4a5d5ba89 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/tools/mw" "strconv" "strings" "sync" @@ -346,6 +347,12 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con } } func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() + defer och.wg.Done() for msg := range och.conversationUserHasReadChan { diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 4762f24de..5c5d081c3 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/tools/mw" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" @@ -83,8 +84,15 @@ func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq) } func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgData) { + log.ZDebug(nctx, "setConversationAtInfo", "msg", msg) + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(nctx, r) + } + }() + ctx := mcontext.NewCtx("@@@" + mcontext.GetOperationID(nctx)) var atUserID []string diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index 3046f84b1..3c73ef449 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,12 +17,18 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() for message := range client.Subscribe(ctx, channel).Channel() { log.ZDebug(ctx, "subscriberRedisDeleteCache", "channel", channel, "payload", message.Payload) var keys []string From b17f4f15e853fde7e7c574e671cdc0f4eb4f5fa3 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 22 Nov 2024 15:06:28 +0800 Subject: [PATCH 06/12] fix: move workflow to correct path (#2837) * build: implement version file update when release. * build: move file to correct path. * remove. --- .../{.github/workflows => }/update-version-file-on-release.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{.github/workflows => }/update-version-file-on-release.yml (100%) diff --git a/.github/workflows/.github/workflows/update-version-file-on-release.yml b/.github/workflows/update-version-file-on-release.yml similarity index 100% rename from .github/workflows/.github/workflows/update-version-file-on-release.yml rename to .github/workflows/update-version-file-on-release.yml From c2fd0061b042fe5153e885722374cd75f212d6c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:48:47 +0800 Subject: [PATCH 07/12] build(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 (#2851) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.1) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a05b016c..d39ad1d34 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-playground/validator/v10 v10.20.0 github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 diff --git a/go.sum b/go.sum index 3cae53b0d..df5a34516 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= From 2c612ba6d0b8a8d81d47929eb6013881b07c9dcd Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:49:16 +0800 Subject: [PATCH 08/12] fix: del login Policy (#2825) * fix: del login Policy * feat: offline push * feat: offline push * fix: err * fix: err --- config/openim-push.yml | 4 ++-- config/share.yml | 13 +------------ go.mod | 2 +- go.sum | 4 ++-- internal/push/offlinepush/dummy/push.go | 1 + internal/push/offlinepush/offlinepusher.go | 3 --- internal/rpc/msg/as_read.go | 5 +++-- internal/rpc/msg/seq.go | 4 ++-- pkg/common/config/config.go | 16 ++-------------- pkg/common/storage/controller/auth.go | 20 ++++---------------- pkg/common/storage/controller/msg.go | 2 +- 11 files changed, 19 insertions(+), 55 deletions(-) diff --git a/config/openim-push.yml b/config/openim-push.yml index 70e67add2..92f716ba2 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -13,8 +13,8 @@ prometheus: ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 -#Use geTui for offline push notifications, or choose fcm or jpush; corresponding configuration settings must be specified. -enable: geTui +#Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. +enable: geTui: pushUrl: https://restapi.getui.com/v2/$appId masterSecret: diff --git a/config/share.yml b/config/share.yml index 7d977ae15..1726af2dc 100644 --- a/config/share.yml +++ b/config/share.yml @@ -15,15 +15,4 @@ imAdminUserID: [ imAdmin ] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLogin: policy: 1 - maxNumOneEnd: 30 - customizeLoginNum: - ios: 1 - android: 1 - windows: 1 - osx: 1 - web: 1 - miniWeb: 1 - linux: 1 - aPad: 1 - iPad: 1 - admin: 1 + maxNumOneEnd: 30 \ No newline at end of file diff --git a/go.mod b/go.mod index d39ad1d34..96c229166 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.54 + github.com/openimsdk/protocol v0.0.72-alpha.55 github.com/openimsdk/tools v0.0.50-alpha.32 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index df5a34516..36f3d1615 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s= -github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.55 h1:9PPWPHvkFk3neBSbNr+IoOdKIFjxTvEqUfMK/TEq1+8= +github.com/openimsdk/protocol v0.0.72-alpha.55/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw= github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 09831cabf..0bccaf4a4 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -29,5 +29,6 @@ type Dummy struct { func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { log.ZDebug(ctx, "dummy push") + log.ZWarn(ctx, "Dummy push", nil, "ps", "The offline push is not configured. To configure it, please go to config/openim-push.yml.") return nil } diff --git a/internal/push/offlinepush/offlinepusher.go b/internal/push/offlinepush/offlinepusher.go index d655a924a..17d5d7071 100644 --- a/internal/push/offlinepush/offlinepusher.go +++ b/internal/push/offlinepush/offlinepusher.go @@ -23,8 +23,6 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" "strings" ) @@ -51,7 +49,6 @@ func NewOfflinePusher(pushConf *config.Push, cache cache.ThirdCache, fcmConfigPa offlinePusher = jpush.NewClient(pushConf) default: offlinePusher = dummy.NewClient() - log.ZWarn(mcontext.WithMustInfoCtx([]string{"push start", "admin", "admin", ""}), "Unknown push config", nil) } return offlinePusher, nil } diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index 03f35b42d..312c4d556 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -16,6 +16,7 @@ package msg import ( "context" + "errors" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/protocol/constant" @@ -108,7 +109,7 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR return nil, err } currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } if hasReadSeq > currentHasReadSeq { @@ -136,7 +137,7 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon return nil, err } hasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } var seqs []int64 diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 5d40160de..ddf84a267 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -16,15 +16,15 @@ package msg import ( "context" + "errors" pbmsg "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" "sort" ) func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c40018cbc..468a150e8 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -369,20 +369,8 @@ type Share struct { } type MultiLogin struct { - Policy int `mapstructure:"policy"` - MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` - CustomizeLoginNum struct { - IOS int `mapstructure:"ios"` - Android int `mapstructure:"android"` - Windows int `mapstructure:"windows"` - OSX int `mapstructure:"osx"` - Web int `mapstructure:"web"` - MiniWeb int `mapstructure:"miniWeb"` - Linux int `mapstructure:"linux"` - APad int `mapstructure:"aPad"` - IPad int `mapstructure:"iPad"` - Admin int `mapstructure:"admin"` - } `mapstructure:"customizeLoginNum"` + Policy int `mapstructure:"policy"` + MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` } type RpcRegisterName struct { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index e7b5bc297..0a7029662 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -25,9 +25,8 @@ type AuthDatabase interface { } type multiLoginConfig struct { - Policy int - MaxNumOneEnd int - CustomizeLoginNum map[int]int + Policy int + MaxNumOneEnd int } type authDatabase struct { @@ -42,19 +41,8 @@ func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire i return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{ Policy: multiLogin.Policy, MaxNumOneEnd: multiLogin.MaxNumOneEnd, - CustomizeLoginNum: map[int]int{ - constant.IOSPlatformID: multiLogin.CustomizeLoginNum.IOS, - constant.AndroidPlatformID: multiLogin.CustomizeLoginNum.Android, - constant.WindowsPlatformID: multiLogin.CustomizeLoginNum.Windows, - constant.OSXPlatformID: multiLogin.CustomizeLoginNum.OSX, - constant.WebPlatformID: multiLogin.CustomizeLoginNum.Web, - constant.MiniWebPlatformID: multiLogin.CustomizeLoginNum.MiniWeb, - constant.LinuxPlatformID: multiLogin.CustomizeLoginNum.Linux, - constant.AndroidPadPlatformID: multiLogin.CustomizeLoginNum.APad, - constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad, - constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin, - }, - }, adminUserIDs: adminUserIDs, + }, + adminUserIDs: adminUserIDs, } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 9636f7a15..789adb1f6 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -372,7 +372,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) From bfc4684a36bd80f8faeceb33d3d01525a394e38c Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:46:02 +0800 Subject: [PATCH 09/12] fix: err (#2876) --- internal/rpc/msg/as_read.go | 2 +- internal/rpc/msg/seq.go | 2 +- pkg/common/storage/controller/msg.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index 312c4d556..e13e017e5 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -109,7 +109,7 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR return nil, err } currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } if hasReadSeq > currentHasReadSeq { diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index ddf84a267..7f5fa1adb 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -24,7 +24,7 @@ import ( func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 789adb1f6..cdc8fd888 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -372,7 +372,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) From cbade46ae0c40ecd47e9177c20862d68df46ef71 Mon Sep 17 00:00:00 2001 From: Morya Date: Mon, 25 Nov 2024 10:49:31 +0800 Subject: [PATCH 10/12] fix: minor log typo (#2881) --- pkg/common/storage/cache/redis/conversation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 326f60b96..91d8ed69d 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -38,7 +38,7 @@ const ( func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCache, opts *rockscache.Options, db database.Conversation) cache.ConversationCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Conversation.Topic}) c := localCache.Conversation - log.ZDebug(context.Background(), "black local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable()) + log.ZDebug(context.Background(), "conversation local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable()) return &ConversationRedisCache{ BatchDeleter: batchHandler, rcClient: rockscache.NewClient(rdb, *opts), From 38cfa4a9c41c0d52aef9c63a914dcbf9a77bda3d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:11:34 +0800 Subject: [PATCH 11/12] fix: webhookAfterSingleMsgRead (#2884) --- internal/rpc/msg/as_read.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index e13e017e5..c9610969d 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -181,14 +181,23 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon req.UserID, seqs, hasReadSeq) } - reqCall := &cbapi.CallbackGroupMsgReadReq{ - SendID: conversation.OwnerUserID, - ReceiveID: req.UserID, - UnreadMsgNum: req.HasReadSeq, - ContentType: int64(conversation.ConversationType), + if conversation.ConversationType == constant.SingleChatType { + reqCall := &cbapi.CallbackSingleMsgReadReq{ + ConversationID: conversation.ConversationID, + UserID: conversation.OwnerUserID, + Seqs: req.Seqs, + ContentType: conversation.ConversationType, + } + m.webhookAfterSingleMsgRead(ctx, &m.config.WebhooksConfig.AfterSingleMsgRead, reqCall) + } else if conversation.ConversationType == constant.ReadGroupChatType { + reqCall := &cbapi.CallbackGroupMsgReadReq{ + SendID: conversation.OwnerUserID, + ReceiveID: req.UserID, + UnreadMsgNum: req.HasReadSeq, + ContentType: int64(conversation.ConversationType), + } + m.webhookAfterGroupMsgRead(ctx, &m.config.WebhooksConfig.AfterGroupMsgRead, reqCall) } - - m.webhookAfterGroupMsgRead(ctx, &m.config.WebhooksConfig.AfterGroupMsgRead, reqCall) return &msg.MarkConversationAsReadResp{}, nil } From f0a60a71aea9ec1627704731a67bfb6a2f70c7e3 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:12:03 +0800 Subject: [PATCH 12/12] fix: webhookBeforeSendSingleMsg will call before black and friend check (#2885) --- internal/rpc/msg/send.go | 3 --- internal/rpc/msg/verify.go | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 5c5d081c3..2cbbcd1fc 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -179,9 +179,6 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq prommetrics.SingleChatMsgProcessFailedCounter.Inc() return nil, nil } else { - if err = m.webhookBeforeSendSingleMsg(ctx, &m.config.WebhooksConfig.BeforeSendSingleMsg, req); err != nil { - return nil, err - } if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req); err != nil { return nil, err } diff --git a/internal/rpc/msg/verify.go b/internal/rpc/msg/verify.go index 33879bfe7..f6c3147ba 100644 --- a/internal/rpc/msg/verify.go +++ b/internal/rpc/msg/verify.go @@ -59,6 +59,9 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe data.MsgData.ContentType >= constant.NotificationBegin { return nil } + if err := m.webhookBeforeSendSingleMsg(ctx, &m.config.WebhooksConfig.BeforeSendSingleMsg, data); err != nil { + return err + } black, err := m.FriendLocalCache.IsBlack(ctx, data.MsgData.SendID, data.MsgData.RecvID) if err != nil { return err