From bba662f4046653e059dfc24d4250908845eb673e Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:08:05 +0800 Subject: [PATCH 1/3] Feature: add log support for developer (#1101) * fix: to start im or chat, ZooKeeper must be started first. * fix: msg gateway start output err info Signed-off-by: Gordon <1432970085@qq.com> * fix: msg gateway start output err info Signed-off-by: Gordon <1432970085@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * fix: go mod update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: get all userID Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: msggateway add online status call Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: log change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: log change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * chore: network mode change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * feat: add api of get server time Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * feat: remove go work sum Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: pull message add isRead field Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: check msg-transfer script Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: start don't kill old process Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: check component Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: pull message set isRead only message come from single. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: add ex field to update group info. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * cicd: robot automated Change * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: to start im or chat, ZooKeeper must be started first. --------- Signed-off-by: Gordon <1432970085@qq.com> Signed-off-by: withchao <993506633@qq.com> Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: withchao <993506633@qq.com> Co-authored-by: Xinwei Xiong <3293172751NSS@gmail.com> Co-authored-by: FGadvancer --- go.mod | 2 +- internal/api/route.go | 5 + internal/api/third.go | 13 +++ internal/msggateway/client.go | 11 ++- internal/msggateway/long_conn.go | 8 +- internal/rpc/third/log.go | 145 ++++++++++++++++++++++++++++ internal/rpc/third/third.go | 3 +- pkg/common/db/cache/conversation.go | 19 ++-- pkg/common/db/controller/third.go | 44 ++++++++- pkg/common/db/relation/log_model.go | 46 +++++++++ pkg/common/db/table/relation/log.go | 25 +++++ tools/imctl/go.mod | 5 +- tools/imctl/go.sum | 3 + 13 files changed, 304 insertions(+), 25 deletions(-) create mode 100644 internal/rpc/third/log.go create mode 100644 pkg/common/db/relation/log_model.go create mode 100644 pkg/common/db/table/relation/log.go diff --git a/go.mod b/go.mod index 7e3af7de2..0bfc858e8 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require github.com/google/uuid v1.3.1 require ( github.com/IBM/sarama v1.41.1 - github.com/OpenIMSDK/protocol v0.0.21 + github.com/OpenIMSDK/protocol v0.0.23 github.com/OpenIMSDK/tools v0.0.14 github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/go-redis/redis v6.15.9+incompatible diff --git a/internal/api/route.go b/internal/api/route.go index 221e180a1..9a639a2de 100644 --- a/internal/api/route.go +++ b/internal/api/route.go @@ -156,6 +156,11 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken) thirdGroup.POST("/set_app_badge", t.SetAppBadge) + logs := thirdGroup.Group("/logs") + logs.POST("/upload", t.UploadLogs) + logs.POST("/delete", t.DeleteLogs) + logs.POST("/search", t.SearchLogs) + objectGroup := r.Group("/object", ParseToken) objectGroup.POST("/part_limit", t.PartLimit) diff --git a/internal/api/third.go b/internal/api/third.go index dfc82d316..8c4fddb67 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -105,3 +105,16 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { } c.Redirect(http.StatusFound, resp.Url) } + +// #################### logs #################### +func (o *ThirdApi) UploadLogs(c *gin.Context) { + a2r.Call(third.ThirdClient.UploadLogs, o.Client, c) +} + +func (o *ThirdApi) DeleteLogs(c *gin.Context) { + a2r.Call(third.ThirdClient.DeleteLogs, o.Client, c) +} + +func (o *ThirdApi) SearchLogs(c *gin.Context) { + a2r.Call(third.ThirdClient.SearchLogs, o.Client, c) +} diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 8121535ad..739fa9688 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -59,7 +59,7 @@ const ( PongMessage = 10 ) -type PongHandler func(string) error +type PingPongHandler func(string) error type Client struct { w *sync.Mutex @@ -107,8 +107,12 @@ func (c *Client) ResetClient( c.token = token } -func (c *Client) pongHandler(_ string) error { +func (c *Client) pingHandler(_ string) error { c.conn.SetReadDeadline(pongWait) + err := c.writePongMsg() + if err != nil { + return err + } return nil } @@ -122,10 +126,11 @@ func (c *Client) readMessage() { }() c.conn.SetReadLimit(maxMessageSize) _ = c.conn.SetReadDeadline(pongWait) - c.conn.SetPongHandler(c.pongHandler) + c.conn.SetPingHandler(c.pingHandler) for { messageType, message, returnErr := c.conn.ReadMessage() if returnErr != nil { + log.ZWarn(c.ctx, "readMessage", returnErr, "messageType", messageType) c.closedErr = returnErr return } diff --git a/internal/msggateway/long_conn.go b/internal/msggateway/long_conn.go index 59e4d5b45..f7676aec1 100644 --- a/internal/msggateway/long_conn.go +++ b/internal/msggateway/long_conn.go @@ -41,7 +41,8 @@ type LongConn interface { SetConnNil() // SetReadLimit sets the maximum size for a message read from the peer.bytes SetReadLimit(limit int64) - SetPongHandler(handler PongHandler) + SetPongHandler(handler PingPongHandler) + SetPingHandler(handler PingPongHandler) // GenerateLongConn Check the connection of the current and when it was sent are the same GenerateLongConn(w http.ResponseWriter, r *http.Request) error } @@ -116,9 +117,12 @@ func (d *GWebSocket) SetReadLimit(limit int64) { d.conn.SetReadLimit(limit) } -func (d *GWebSocket) SetPongHandler(handler PongHandler) { +func (d *GWebSocket) SetPongHandler(handler PingPongHandler) { d.conn.SetPongHandler(handler) } +func (d *GWebSocket) SetPingHandler(handler PingPongHandler) { + d.conn.SetPingHandler(handler) +} //func (d *GWebSocket) CheckSendConnDiffNow() bool { // return d.conn == d.sendConn diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go new file mode 100644 index 000000000..b56d82f0f --- /dev/null +++ b/internal/rpc/third/log.go @@ -0,0 +1,145 @@ +package third + +import ( + "context" + "crypto/rand" + "fmt" + "time" + + "github.com/OpenIMSDK/protocol/constant" + "github.com/OpenIMSDK/protocol/third" + "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/utils" + utils2 "github.com/OpenIMSDK/tools/utils" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" +) + +func genLogID() string { + const dataLen = 10 + data := make([]byte, dataLen) + rand.Read(data) + chars := []byte("0123456789") + for i := 0; i < len(data); i++ { + if i == 0 { + data[i] = chars[1:][data[i]%9] + } else { + data[i] = chars[data[i]%10] + } + } + return string(data) +} + +func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) (*third.UploadLogsResp, error) { + var DBlogs []*relationtb.Log + userID := ctx.Value(constant.OpUserID).(string) + platform := constant.PlatformID2Name[int(req.Platform)] + for _, fileURL := range req.FileURLs { + log := relationtb.Log{ + Version: req.Version, + SystemType: req.SystemType, + Platform: platform, + UserID: userID, + CreateTime: time.Now(), + Url: fileURL.URL, + FileName: fileURL.Filename, + } + for i := 0; i < 20; i++ { + id := genLogID() + logs, err := t.thirdDatabase.GetLogs(ctx, []string{id}, "") + if err != nil { + return nil, err + } + if len(logs) == 0 { + log.LogID = id + break + } + } + if log.LogID == "" { + return nil, errs.ErrData.Wrap("Log id gen error") + } + DBlogs = append(DBlogs, &log) + } + err := t.thirdDatabase.UploadLogs(ctx, DBlogs) + if err != nil { + return nil, err + } + return &third.UploadLogsResp{}, nil +} + +func (t *thirdServer) DeleteLogs(ctx context.Context, req *third.DeleteLogsReq) (*third.DeleteLogsResp, error) { + + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } + userID := "" + logs, err := t.thirdDatabase.GetLogs(ctx, req.LogIDs, userID) + if err != nil { + return nil, err + } + var logIDs []string + for _, log := range logs { + logIDs = append(logIDs, log.LogID) + } + if ids := utils2.Single(req.LogIDs, logIDs); len(ids) > 0 { + return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("logIDs not found%#v", ids)) + } + err = t.thirdDatabase.DeleteLogs(ctx, req.LogIDs, userID) + if err != nil { + return nil, err + } + + return &third.DeleteLogsResp{}, nil +} + +func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo { + db2pbForLogInfo := func(log *relationtb.Log) *third.LogInfo { + return &third.LogInfo{ + Filename: log.FileName, + UserID: log.UserID, + Platform: utils.StringToInt32(log.Platform), + Url: log.Url, + CreateTime: log.CreateTime.UnixMilli(), + LogID: log.LogID, + SystemType: log.SystemType, + Version: log.Version, + Ex: log.Ex, + } + } + return utils.Slice(logs, db2pbForLogInfo) +} + +func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq) (*third.SearchLogsResp, error) { + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } + var ( + resp third.SearchLogsResp + userIDs []string + ) + if req.StartTime > req.EndTime { + return nil, errs.ErrArgs.Wrap("startTime>endTime") + } + total, logs, err := t.thirdDatabase.SearchLogs(ctx, req.Keyword, time.UnixMilli(req.StartTime), time.UnixMilli(req.EndTime), req.Pagination.PageNumber, req.Pagination.ShowNumber) + if err != nil { + return nil, err + } + pbLogs := dbToPbLogInfos(logs) + for _, log := range logs { + userIDs = append(userIDs, log.UserID) + } + users, err := t.thirdDatabase.FindUsers(ctx, userIDs) + if err != nil { + return nil, err + } + IDtoName := make(map[string]string) + for _, user := range users { + IDtoName[user.UserID] = user.Nickname + } + for _, pbLog := range pbLogs { + pbLog.Nickname = IDtoName[pbLog.UserID] + } + resp.LogsInfos = pbLogs + resp.Total = total + return &resp, nil +} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 221004bd5..ae32a1f40 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -35,6 +35,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/db/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/db/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" ) @@ -79,7 +80,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e } third.RegisterThirdServer(server, &thirdServer{ apiURL: apiURL, - thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb)), + thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb), db), userRpcClient: rpcclient.NewUserRpcClient(client), s3dataBase: controller.NewS3Database(o, relation.NewObjectInfo(db)), defaultExpire: time.Hour * 24 * 7, diff --git a/pkg/common/db/cache/conversation.go b/pkg/common/db/cache/conversation.go index 083cf6d0b..a21168b55 100644 --- a/pkg/common/db/cache/conversation.go +++ b/pkg/common/db/cache/conversation.go @@ -59,11 +59,8 @@ type ConversationCache interface { DelConversations(ownerUserID string, conversationIDs ...string) ConversationCache DelUsersConversation(conversationID string, ownerUserIDs ...string) ConversationCache // get one conversation from msgCache - GetConversations( - ctx context.Context, - ownerUserID string, - conversationIDs []string, - ) ([]*relationtb.ConversationModel, error) + GetConversations(ctx context.Context, ownerUserID string, + conversationIDs []string) ([]*relationtb.ConversationModel, error) // get one user's all conversations from msgCache GetUserAllConversations(ctx context.Context, ownerUserID string) ([]*relationtb.ConversationModel, error) // get user conversation recv msg from msgCache @@ -79,10 +76,8 @@ type ConversationCache interface { GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error) DelUserAllHasReadSeqs(ownerUserID string, conversationIDs ...string) ConversationCache - GetConversationsByConversationID( - ctx context.Context, - conversationIDs []string, - ) ([]*relationtb.ConversationModel, error) + GetConversationsByConversationID(ctx context.Context, + conversationIDs []string) ([]*relationtb.ConversationModel, error) DelConversationByConversationID(conversationIDs ...string) ConversationCache GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache @@ -418,10 +413,8 @@ func (c *ConversationRedisCache) GetUserAllHasReadSeqs( ) } -func (c *ConversationRedisCache) DelUserAllHasReadSeqs( - ownerUserID string, - conversationIDs ...string, -) ConversationCache { +func (c *ConversationRedisCache) DelUserAllHasReadSeqs(ownerUserID string, + conversationIDs ...string) ConversationCache { cache := c.NewCache() for _, conversationID := range conversationIDs { cache.AddKeys(c.getConversationHasReadSeqKey(ownerUserID, conversationID)) diff --git a/pkg/common/db/controller/third.go b/pkg/common/db/controller/third.go index c5476e490..247dfb408 100644 --- a/pkg/common/db/controller/third.go +++ b/pkg/common/db/controller/third.go @@ -16,21 +16,59 @@ package controller import ( "context" + "time" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" + dbimpl "github.com/openimsdk/open-im-server/v3/pkg/common/db/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "gorm.io/gorm" ) type ThirdDatabase interface { FcmUpdateToken(ctx context.Context, account string, platformID int, fcmToken string, expireTime int64) error SetAppBadge(ctx context.Context, userID string, value int) error + //about log for debug + UploadLogs(ctx context.Context, logs []*relation.Log) error + DeleteLogs(ctx context.Context, logID []string, userID string) error + SearchLogs(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relation.Log, error) + GetLogs(ctx context.Context, LogIDs []string, userID string) ([]*relation.Log, error) + FindUsers(ctx context.Context, userIDs []string) ([]*relation.UserModel, error) } type thirdDatabase struct { - cache cache.MsgModel + cache cache.MsgModel + logdb relation.LogInterface + userdb relation.UserModelInterface } -func NewThirdDatabase(cache cache.MsgModel) ThirdDatabase { - return &thirdDatabase{cache: cache} +// FindUsers implements ThirdDatabase. +func (t *thirdDatabase) FindUsers(ctx context.Context, userIDs []string) ([]*relation.UserModel, error) { + return t.userdb.Find(ctx, userIDs) +} + +// DeleteLogs implements ThirdDatabase. +func (t *thirdDatabase) DeleteLogs(ctx context.Context, logID []string, userID string) error { + return t.logdb.Delete(ctx, logID, userID) +} + +// GetLogs implements ThirdDatabase. +func (t *thirdDatabase) GetLogs(ctx context.Context, LogIDs []string, userID string) ([]*relation.Log, error) { + return t.logdb.Get(ctx, LogIDs, userID) +} + +// SearchLogs implements ThirdDatabase. +func (t *thirdDatabase) SearchLogs(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relation.Log, error) { + return t.logdb.Search(ctx, keyword, start, end, pageNumber, showNumber) + +} + +// UploadLogs implements ThirdDatabase. +func (t *thirdDatabase) UploadLogs(ctx context.Context, logs []*relation.Log) error { + return t.logdb.Create(ctx, logs) +} + +func NewThirdDatabase(cache cache.MsgModel, db *gorm.DB) ThirdDatabase { + return &thirdDatabase{cache: cache, logdb: dbimpl.NewLogGorm(db), userdb: dbimpl.NewUserGorm(db)} } func (t *thirdDatabase) FcmUpdateToken( diff --git a/pkg/common/db/relation/log_model.go b/pkg/common/db/relation/log_model.go new file mode 100644 index 000000000..4508297c2 --- /dev/null +++ b/pkg/common/db/relation/log_model.go @@ -0,0 +1,46 @@ +package relation + +import ( + "context" + "time" + + "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/ormutil" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "gorm.io/gorm" +) + +type LogGorm struct { + db *gorm.DB +} + +func (l *LogGorm) Create(ctx context.Context, log []*relationtb.Log) error { + return errs.Wrap(l.db.WithContext(ctx).Create(log).Error) +} + +func (l *LogGorm) Search(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relationtb.Log, error) { + db := l.db.WithContext(ctx).Where("create_time >= ?", start) + if end.UnixMilli() != 0 { + db = l.db.WithContext(ctx).Where("create_time <= ?", end) + } + return ormutil.GormSearch[relationtb.Log](db, []string{"user_id"}, keyword, pageNumber, showNumber) +} + +func (l *LogGorm) Delete(ctx context.Context, logIDs []string, userID string) error { + if userID == "" { + return errs.Wrap(l.db.WithContext(ctx).Where("log_id in ?", logIDs).Delete(&relationtb.Log{}).Error) + } + return errs.Wrap(l.db.WithContext(ctx).Where("log_id in ? and user_id=?", logIDs, userID).Delete(&relationtb.Log{}).Error) +} + +func (l *LogGorm) Get(ctx context.Context, logIDs []string, userID string) ([]*relationtb.Log, error) { + var logs []*relationtb.Log + if userID == "" { + return logs, errs.Wrap(l.db.WithContext(ctx).Where("log_id in ?", logIDs).Find(&logs).Error) + } + return logs, errs.Wrap(l.db.WithContext(ctx).Where("log_id in ? and user_id=?", logIDs, userID).Find(&logs).Error) +} +func NewLogGorm(db *gorm.DB) relationtb.LogInterface { + db.AutoMigrate(&relationtb.Log{}) + return &LogGorm{db: db} +} diff --git a/pkg/common/db/table/relation/log.go b/pkg/common/db/table/relation/log.go new file mode 100644 index 000000000..72d0fa64e --- /dev/null +++ b/pkg/common/db/table/relation/log.go @@ -0,0 +1,25 @@ +package relation + +import ( + "context" + "time" +) + +type Log struct { + LogID string `gorm:"column:log_id;primary_key;type:char(64)"` + Platform string `gorm:"column:platform;type:varchar(32)"` + UserID string `gorm:"column:user_id;type:char(64)"` + CreateTime time.Time `gorm:"index:,sort:desc"` + Url string `gorm:"column:url;type varchar(255)"` + FileName string `gorm:"column:filename;type varchar(255)"` + SystemType string `gorm:"column:system_type;type varchar(255)"` + Version string `gorm:"column:version;type varchar(255)"` + Ex string `gorm:"column:ex;type varchar(255)"` +} + +type LogInterface interface { + Create(ctx context.Context, log []*Log) error + Search(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*Log, error) + Delete(ctx context.Context, logID []string, userID string) error + Get(ctx context.Context, logIDs []string, userID string) ([]*Log, error) +} diff --git a/tools/imctl/go.mod b/tools/imctl/go.mod index 60b06b45b..028520586 100644 --- a/tools/imctl/go.mod +++ b/tools/imctl/go.mod @@ -1,4 +1,4 @@ -module github.com/openimsdk/open-im-server/v3/tools/imctl +module github.com/openimsdk/open-im-server/v3/tools/imctl go 1.18 @@ -14,5 +14,6 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.10.0 // indirect + k8s.io/kubernetes v1.28.2 ) diff --git a/tools/imctl/go.sum b/tools/imctl/go.sum index 3d4c61343..762b21142 100644 --- a/tools/imctl/go.sum +++ b/tools/imctl/go.sum @@ -20,5 +20,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/kubernetes v1.28.2 h1:GhcnYeNTukeaC0dD5BC+UWBvzQsFEpWj7XBVMQptfYc= +k8s.io/kubernetes v1.28.2/go.mod h1:FmB1Mlp9ua0ezuwQCTGs/y6wj/fVisN2sVxhzjj0WDk= From 5c31d12253f9e2459fbccf3bf4dc878b80e92ef0 Mon Sep 17 00:00:00 2001 From: withchao <48119764+withchao@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:23:46 +0800 Subject: [PATCH 2/3] feat: s3 public read (#1080) * fix: repeated modification session notification * fix: repeated modification session notification * fix: jpush return a nil pointer panic * fix: push redis pkg * fix: OANotification * feat: add rpc GetConversationNeedOfflinePushUserIDs * update pkg * cicd: robot automated Change * offlinePushMsg * conversation * conversation * cicd: robot automated Change * conversation * cicd: robot automated Change * conversation * url 2 im s3 * url 2 im s3 * cicd: robot automated Change * url 2 im s3 * s3 public read * cicd: robot automated Change * s3 public read * cicd: robot automated Change * s3 public read * s3 public read * s3 public read * s3 public read * s3 public read * cicd: robot automated Change * s3 public read * s3 public read * fix: SendMsg api * config scripts * config scripts --------- Co-authored-by: withchao --- config/config.yaml | 3 ++ deployments/templates/openim.yaml | 4 ++- internal/api/msg.go | 2 +- pkg/common/config/config.go | 3 ++ pkg/common/db/s3/cos/cos.go | 20 +++++++++---- pkg/common/db/s3/cos/doc.go | 15 ---------- pkg/common/db/s3/cos/internal.go | 13 +++++++++ pkg/common/db/s3/minio/doc.go | 15 ---------- pkg/common/db/s3/minio/internal.go | 11 ++++++++ pkg/common/db/s3/minio/minio.go | 19 ++++++++++++- pkg/common/db/s3/oss/doc.go | 15 ---------- pkg/common/db/s3/oss/{sign.go => internal.go} | 18 ++++++++++-- pkg/common/db/s3/oss/oss.go | 28 ++++++++++++++----- scripts/install/environment.sh | 4 +++ 14 files changed, 108 insertions(+), 62 deletions(-) delete mode 100644 pkg/common/db/s3/cos/doc.go create mode 100644 pkg/common/db/s3/cos/internal.go delete mode 100644 pkg/common/db/s3/minio/doc.go create mode 100644 pkg/common/db/s3/minio/internal.go delete mode 100644 pkg/common/db/s3/oss/doc.go rename pkg/common/db/s3/oss/{sign.go => internal.go} (56%) diff --git a/config/config.yaml b/config/config.yaml index 9d6b3c335..0d2493604 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -140,11 +140,13 @@ object: secretAccessKey: "openIM123" sessionToken: '' signEndpoint: "http://127.0.0.1:10005" + publicRead: false cos: bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com secretID: '' secretKey: '' sessionToken: '' + publicRead: false oss: endpoint: "https://oss-cn-chengdu.aliyuncs.com" bucket: "demo-9999999" @@ -152,6 +154,7 @@ object: accessKeyID: '' accessKeySecret: '' sessionToken: '' + publicRead: false ###################### RPC Port Configuration ###################### diff --git a/deployments/templates/openim.yaml b/deployments/templates/openim.yaml index 9465ea872..5c6be94f5 100644 --- a/deployments/templates/openim.yaml +++ b/deployments/templates/openim.yaml @@ -140,11 +140,13 @@ object: secretAccessKey: "${MINIO_SECRET_KEY}" sessionToken: ${MINIO_SESSION_TOKEN} signEndpoint: "${MINIO_SIGN_ENDPOINT}" + publicRead: ${MINIO_PUBLIC_READ} cos: bucketURL: ${COS_BUCKET_URL} secretID: ${COS_SECRET_ID} secretKey: ${COS_SECRET_KEY} sessionToken: ${COS_SESSION_TOKEN} + publicRead: ${COS_PUBLIC_READ} oss: endpoint: "${OSS_ENDPOINT}" bucket: "${OSS_BUCKET}" @@ -152,7 +154,7 @@ object: accessKeyID: ${OSS_ACCESS_KEY_ID} accessKeySecret: ${OSS_ACCESS_KEY_SECRET} sessionToken: ${OSS_SESSION_TOKEN} - + publicRead: ${OSS_PUBLIC_READ} ###################### RPC Port Configuration ###################### # RPC service ports diff --git a/internal/api/msg.go b/internal/api/msg.go index 2777d8855..83949903e 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -58,7 +58,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg) options := make(map[string]bool, 5) switch params.ContentType { case constant.Text: - newContent = params.Content["text"].(string) + fallthrough case constant.Picture: fallthrough case constant.Custom: diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c6dd41419..c31814743 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -128,12 +128,14 @@ type configStruct struct { SecretAccessKey string `yaml:"secretAccessKey"` SessionToken string `yaml:"sessionToken"` SignEndpoint string `yaml:"signEndpoint"` + PublicRead bool `yaml:"publicRead"` } `yaml:"minio"` Cos struct { BucketURL string `yaml:"bucketURL"` SecretID string `yaml:"secretID"` SecretKey string `yaml:"secretKey"` SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } `yaml:"cos"` Oss struct { Endpoint string `yaml:"endpoint"` @@ -142,6 +144,7 @@ type configStruct struct { AccessKeyID string `yaml:"accessKeyID"` AccessKeySecret string `yaml:"accessKeySecret"` SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } `yaml:"oss"` } `yaml:"object"` diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 57a205cbe..7add88487 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -288,7 +288,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, style = append(style, "format/"+opt.Image.Format) } if len(style) > 0 { - imageMogr = "&imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1" + imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1" } } if opt.ContentType != "" { @@ -306,13 +306,23 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, &option) + rawURL, err := c.getPresignedURL(ctx, name, expire, &option) if err != nil { return "", err } - urlStr := rawURL.String() if imageMogr != "" { - urlStr += imageMogr + if rawURL.RawQuery == "" { + rawURL.RawQuery = imageMogr + } else { + rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr + } + } + return rawURL.String(), nil +} + +func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) { + if !config.Config.Object.Cos.PublicRead { + return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt) } - return urlStr, nil + return c.client.Object.GetObjectURL(name), nil } diff --git a/pkg/common/db/s3/cos/doc.go b/pkg/common/db/s3/cos/doc.go deleted file mode 100644 index 592f90ee2..000000000 --- a/pkg/common/db/s3/cos/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" diff --git a/pkg/common/db/s3/cos/internal.go b/pkg/common/db/s3/cos/internal.go new file mode 100644 index 000000000..0e58a851c --- /dev/null +++ b/pkg/common/db/s3/cos/internal.go @@ -0,0 +1,13 @@ +package cos + +import ( + "context" + "net/http" + "net/url" + _ "unsafe" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest +func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) diff --git a/pkg/common/db/s3/minio/doc.go b/pkg/common/db/s3/minio/doc.go deleted file mode 100644 index 42e898c15..000000000 --- a/pkg/common/db/s3/minio/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" diff --git a/pkg/common/db/s3/minio/internal.go b/pkg/common/db/s3/minio/internal.go new file mode 100644 index 000000000..41129ce31 --- /dev/null +++ b/pkg/common/db/s3/minio/internal.go @@ -0,0 +1,11 @@ +package minio + +import ( + "net/url" + _ "unsafe" + + "github.com/minio/minio-go/v7" +) + +//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL +func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index 937b9f78a..037c933bb 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -139,6 +139,15 @@ func (m *Minio) initMinio(ctx context.Context) error { return fmt.Errorf("make bucket error: %w", err) } } + if conf.PublicRead { + policy := fmt.Sprintf( + `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, + conf.Bucket, + ) + if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil { + return err + } + } m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket) if err != nil { return err @@ -375,7 +384,15 @@ func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time } else if expire < time.Second { expire = time.Second } - rawURL, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query) + var ( + rawURL *url.URL + err error + ) + if config.Config.Object.Minio.PublicRead { + rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query) + } else { + rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query) + } if err != nil { return "", err } diff --git a/pkg/common/db/s3/oss/doc.go b/pkg/common/db/s3/oss/doc.go deleted file mode 100644 index d2e2c91af..000000000 --- a/pkg/common/db/s3/oss/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss" diff --git a/pkg/common/db/s3/oss/sign.go b/pkg/common/db/s3/oss/internal.go similarity index 56% rename from pkg/common/db/s3/oss/sign.go rename to pkg/common/db/s3/oss/internal.go index 1bff18f4d..4ca1acc47 100644 --- a/pkg/common/db/s3/oss/sign.go +++ b/pkg/common/db/s3/oss/internal.go @@ -16,10 +16,24 @@ package oss import ( "net/http" + "net/url" _ "unsafe" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) -//go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.(*Conn).signHeader -func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string) +//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader +func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string) + +//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams +func getURLParams(c oss.Conn, params map[string]interface{}) string + +//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL +func getURL(um urlMaker, bucket, object, params string) *url.URL + +type urlMaker struct { + Scheme string + NetLoc string + Type int + IsProxy bool +} diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 384ce8093..6a728127b 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -20,6 +20,7 @@ import ( "fmt" "net/http" "net/url" + "reflect" "strconv" "strings" "time" @@ -69,6 +70,7 @@ func NewOSS() (s3.Interface, error) { bucketURL: conf.BucketURL, bucket: bucket, credentials: client.Config.GetCredentials(), + um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()), }, nil } @@ -76,6 +78,7 @@ type OSS struct { bucketURL string bucket *oss.Bucket credentials oss.Credentials + um urlMaker } func (o *OSS) Engine() string { @@ -163,7 +166,7 @@ func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire request.Header.Set(oss.HTTPHeaderHost, request.Host) request.Header.Set(oss.HTTPHeaderDate, now) request.Header.Set(oss.HttpHeaderOssDate, now) - ossSignHeader(o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID)) + signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID)) delete(request.Header, oss.HTTPHeaderDate) result.Parts[i] = s3.SignPart{ PartNumber: partNumber, @@ -272,6 +275,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin } func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { + publicRead := config.Config.Object.Oss.PublicRead var opts []oss.Option if opt != nil { if opt.Image != nil { @@ -299,11 +303,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, process += ",format," + format opts = append(opts, oss.Process(process)) } - if opt.ContentType != "" { - opts = append(opts, oss.ResponseContentType(opt.ContentType)) - } - if opt.Filename != "" { - opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename))) + if !publicRead { + if opt.ContentType != "" { + opts = append(opts, oss.ResponseContentType(opt.ContentType)) + } + if opt.Filename != "" { + opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename))) + } } } if expire <= 0 { @@ -311,5 +317,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, } else if expire < time.Second { expire = time.Second } - return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + if !publicRead { + return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + } + rawParams, err := oss.GetRawParams(opts) + if err != nil { + return "", err + } + params := getURLParams(*o.bucket.Client.Conn, rawParams) + return getURL(o.um, o.bucket.BucketName, name, params).String(), nil } diff --git a/scripts/install/environment.sh b/scripts/install/environment.sh index b17570be5..7984e23bd 100755 --- a/scripts/install/environment.sh +++ b/scripts/install/environment.sh @@ -186,17 +186,21 @@ def "MINIO_ACCESS_KEY" "${USER}" def "MINIO_SECRET_KEY" "${PASSWORD}" # MinIO的密钥 def "MINIO_SESSION_TOKEN" # MinIO的会话令牌 readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址 +def "MINIO_PUBLIC_READ" "false" # 公有读 + # 腾讯云COS的存储桶URL def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" def "COS_SECRET_ID" # 腾讯云COS的密钥ID def "COS_SECRET_KEY" # 腾讯云COS的密钥 def "COS_SESSION_TOKEN" # 腾讯云COS的会话令牌 +def "COS_PUBLIC_READ" "false" # 公有读 def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的端点URL def "OSS_BUCKET" "demo-9999999" # 阿里云OSS的存储桶名称 def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL def "OSS_ACCESS_KEY_ID" # 阿里云OSS的访问密钥ID def "OSS_ACCESS_KEY_SECRET" # 阿里云OSS的密钥 def "OSS_SESSION_TOKEN" # 阿里云OSS的会话令牌 +def "OSS_PUBLIC_READ" "false" # 公有读 ###################### Redis 配置信息 ###################### def "REDIS_PORT" "16379" # Redis的端口 From 9673f505189f2e6f6737b91d88e26f2d2e1811af Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 20 Sep 2023 12:04:25 +0800 Subject: [PATCH 3/3] feat: send message add sendTime field for import messages. (#1104) * fix: to start im or chat, ZooKeeper must be started first. * fix: msg gateway start output err info Signed-off-by: Gordon <1432970085@qq.com> * fix: msg gateway start output err info Signed-off-by: Gordon <1432970085@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * fix: go mod update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * chore: package path changes Signed-off-by: withchao <993506633@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: token update Signed-off-by: Gordon <1432970085@qq.com> * fix: get all userID Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: msggateway add online status call Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: log change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: log change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * chore: network mode change Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * feat: add api of get server time Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * feat: remove go work sum Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: pull message add isRead field Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: check msg-transfer script Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: script update Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: start don't kill old process Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: check component Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: pull message set isRead only message come from single. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: multiple gateway kick user each other. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fix: add ex field to update group info. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * cicd: robot automated Change * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * refactor: change project module name. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: for pressure test. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: message log. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * fxi: component check output valid info. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * fxi: component check output valid info. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * test: send message test log. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * cicd: robot automated Change * cicd: robot automated Change * test: remove info log. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> * feat: api of send message add sendTime field. Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> --------- Signed-off-by: Gordon <1432970085@qq.com> Signed-off-by: withchao <993506633@qq.com> Signed-off-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: withchao <993506633@qq.com> Co-authored-by: Xinwei Xiong <3293172751NSS@gmail.com> Co-authored-by: FGadvancer --- config/config.yaml | 2 +- go.sum | 4 ++-- internal/api/msg.go | 3 ++- internal/rpc/third/log.go | 1 + pkg/apistruct/manage.go | 1 + pkg/common/db/controller/third.go | 3 ++- pkg/common/db/relation/log_model.go | 3 ++- tools/component/component.go | 31 ++++++++++++++++++----------- 8 files changed, 30 insertions(+), 18 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 0d2493604..ae607f9f6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -132,7 +132,7 @@ api: # minio.signEndpoint is minio public network address object: enable: "minio" - apiURL: "http://http://127.0.0.1:10002" + apiURL: "http://127.0.0.1:10002" minio: bucket: "openim" endpoint: "http://172.28.0.1:10005" diff --git a/go.sum b/go.sum index 5b180156e..23ac482b8 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/IBM/sarama v1.41.1 h1:B4/TdHce/8Ipza+qrLIeNJ9D1AOxZVp/3uDv6H/dp2M= github.com/IBM/sarama v1.41.1/go.mod h1:JFCPURVskaipJdKRFkiE/OZqQHw7jqliaJmRwXCmSSw= -github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY= -github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= +github.com/OpenIMSDK/protocol v0.0.23 h1:L545aRQez6Ro+AaJB1Z6Mz7ojnDtp41WqASxYveCkcE= +github.com/OpenIMSDK/protocol v0.0.23/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/tools v0.0.14 h1:WLof/+WxyPyRST+QkoTKubYCiV73uCLiL8pgnpH/yKQ= github.com/OpenIMSDK/tools v0.0.14/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= diff --git a/internal/api/msg.go b/internal/api/msg.go index 83949903e..b9b10e98e 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -100,6 +100,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg) ContentType: params.ContentType, Content: []byte(newContent), CreateTime: utils.GetCurrentTimestampByMill(), + SendTime: params.SendTime, Options: options, OfflinePushInfo: params.OfflinePushInfo, }, @@ -207,7 +208,6 @@ func (m *MessageApi) SendMessage(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - log.ZInfo(c, "SendMessage", "req", req) if !authverify.IsAppManagerUid(c) { apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message")) return @@ -224,6 +224,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { respPb, err := m.Client.SendMsg(c, sendMsgReq) if err != nil { status = constant.MsgSendFailed + log.ZError(c, "send message err", err) apiresp.GinError(c, err) return } diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index b56d82f0f..1bbf168a9 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -11,6 +11,7 @@ import ( "github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/utils" utils2 "github.com/OpenIMSDK/tools/utils" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index eef38e403..7d30d2151 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -29,6 +29,7 @@ type SendMsg struct { SessionType int32 `json:"sessionType" binding:"required"` IsOnlineOnly bool `json:"isOnlineOnly"` NotOfflinePush bool `json:"notOfflinePush"` + SendTime int64 `json:"sendTime"` OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` } diff --git a/pkg/common/db/controller/third.go b/pkg/common/db/controller/third.go index 247dfb408..b97edd7aa 100644 --- a/pkg/common/db/controller/third.go +++ b/pkg/common/db/controller/third.go @@ -18,10 +18,11 @@ import ( "context" "time" + "gorm.io/gorm" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" dbimpl "github.com/openimsdk/open-im-server/v3/pkg/common/db/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" - "gorm.io/gorm" ) type ThirdDatabase interface { diff --git a/pkg/common/db/relation/log_model.go b/pkg/common/db/relation/log_model.go index 4508297c2..8ca9579f7 100644 --- a/pkg/common/db/relation/log_model.go +++ b/pkg/common/db/relation/log_model.go @@ -6,8 +6,9 @@ import ( "github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/ormutil" - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "gorm.io/gorm" + + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) type LogGorm struct { diff --git a/tools/component/component.go b/tools/component/component.go index 04c6ba9a5..df5849b51 100644 --- a/tools/component/component.go +++ b/tools/component/component.go @@ -22,7 +22,6 @@ import ( "net" "net/url" "os" - "strings" "time" "github.com/minio/minio-go/v7" @@ -117,16 +116,24 @@ func main() { os.Exit(1) } -func exactIP(urll string) string { - u, _ := url.Parse(urll) - host, _, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Host - } - if strings.HasSuffix(host, ":") { - host = host[0 : len(host)-1] +func checkMinioIP() error { + for _, i := range []string{config.Config.Object.ApiURL, config.Config.Object.Minio.SignEndpoint} { + u, err := url.Parse(i) + if err != nil { + return utils.Wrap(err, "api format error,please check config file apiURL or Minio SignEndpoint") + } + if u.Scheme == "https" { + continue + } + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + host = u.Host + } + if host == "127.0.0.1" { + return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1,please modify it") + } } - return host + return nil } func checkMysql() error { @@ -205,8 +212,8 @@ func checkMinio() error { return ErrComponentStart.Wrap("Minio server is offline") } } - if exactIP(config.Config.Object.ApiURL) == "127.0.0.1" || exactIP(config.Config.Object.Minio.SignEndpoint) == "127.0.0.1" { - return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1") + if checkMinioIP() != nil { + return checkMinioIP() } } return nil