From 7389639f174441ff66414e2e541e52aa138ba67f Mon Sep 17 00:00:00 2001 From: Brabem <69128477+luhaoling@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:15:15 +0800 Subject: [PATCH 1/4] feat: add the notificationAccount (#1602) * feat: add notification API * fix: fix the script * fix: fix the error --- deployments/templates/openim.yaml | 8 ++ docs/contrib/environment.md | 74 ++++++++--------- go.mod | 2 + go.sum | 4 +- internal/api/friend.go | 3 - internal/api/msg.go | 5 +- internal/api/route.go | 6 +- internal/api/user.go | 15 +++- internal/rpc/friend/friend.go | 11 ++- internal/rpc/user/user.go | 128 ++++++++++++++++++++++++++++++ pkg/apistruct/msg.go | 2 +- pkg/authverify/token.go | 9 +++ pkg/common/config/config.go | 5 ++ pkg/common/db/controller/user.go | 6 ++ pkg/common/http/http_client.go | 2 +- pkg/rpcclient/user.go | 7 ++ scripts/check-all.sh | 2 +- scripts/install/environment.sh | 2 + 18 files changed, 238 insertions(+), 53 deletions(-) diff --git a/deployments/templates/openim.yaml b/deployments/templates/openim.yaml index ed5dc4fe1..cbb4a8c6e 100644 --- a/deployments/templates/openim.yaml +++ b/deployments/templates/openim.yaml @@ -247,6 +247,14 @@ manager: userID: [ "${MANAGER_USERID_1}", "${MANAGER_USERID_2}", "${MANAGER_USERID_3}" ] nickname: [ "${NICKNAME_1}", "${NICKNAME_2}", "${NICKNAME_3}" ] +# chatAdmin, use for send notification +# +# Built-in app system notification account ID +# Built-in app system notification account nickname +im-admin: + userID: [ "${IM_ADMIN_USERID}" ] + nickname: [ "${IM_ADMIN_NAME}" ] + # Multi-platform login policy # For each platform(Android, iOS, Windows, Mac, web), only one can be online at a time multiLoginPolicy: ${MULTILOGIN_POLICY} diff --git a/docs/contrib/environment.md b/docs/contrib/environment.md index 310c2df6a..e52d57235 100644 --- a/docs/contrib/environment.md +++ b/docs/contrib/environment.md @@ -453,43 +453,45 @@ This section involves configuring the log settings, including storage location, This section involves setting up additional configuration variables for Websocket, Push Notifications, and Chat. -| Parameter | Example Value | Description | -|-------------------------|-------------------|------------------------------------| -| WEBSOCKET_MAX_CONN_NUM | "100000" | Maximum Websocket connections | -| WEBSOCKET_MAX_MSG_LEN | "4096" | Maximum Websocket message length | -| WEBSOCKET_TIMEOUT | "10" | Websocket timeout | -| PUSH_ENABLE | "getui" | Push notification enable status | -| GETUI_PUSH_URL | [Generated URL] | GeTui Push Notification URL | -| GETUI_MASTER_SECRET | [User Defined] | GeTui Master Secret | -| GETUI_APP_KEY | [User Defined] | GeTui Application Key | -| GETUI_INTENT | [User Defined] | GeTui Push Intent | -| GETUI_CHANNEL_ID | [User Defined] | GeTui Channel ID | -| GETUI_CHANNEL_NAME | [User Defined] | GeTui Channel Name | -| FCM_SERVICE_ACCOUNT | "x.json" | FCM Service Account | -| JPNS_APP_KEY | [User Defined] | JPNS Application Key | -| JPNS_MASTER_SECRET | [User Defined] | JPNS Master Secret | -| JPNS_PUSH_URL | [User Defined] | JPNS Push Notification URL | -| JPNS_PUSH_INTENT | [User Defined] | JPNS Push Intent | -| MANAGER_USERID_1 | "openIM123456" | Administrator ID 1 | -| MANAGER_USERID_2 | "openIM654321" | Administrator ID 2 | -| MANAGER_USERID_3 | "openIMAdmin" | Administrator ID 3 | -| NICKNAME_1 | "system1" | Nickname 1 | -| NICKNAME_2 | "system2" | Nickname 2 | -| NICKNAME_3 | "system3" | Nickname 3 | -| MULTILOGIN_POLICY | "1" | Multi-login Policy | -| CHAT_PERSISTENCE_MYSQL | "true" | Chat Persistence in MySQL | -| MSG_CACHE_TIMEOUT | "86400" | Message Cache Timeout | -| GROUP_MSG_READ_RECEIPT | "true" | Group Message Read Receipt Enable | +| Parameter | Example Value | Description | +|-------------------------|-------------------|----------------------------------| +| WEBSOCKET_MAX_CONN_NUM | "100000" | Maximum Websocket connections | +| WEBSOCKET_MAX_MSG_LEN | "4096" | Maximum Websocket message length | +| WEBSOCKET_TIMEOUT | "10" | Websocket timeout | +| PUSH_ENABLE | "getui" | Push notification enable status | +| GETUI_PUSH_URL | [Generated URL] | GeTui Push Notification URL | +| GETUI_MASTER_SECRET | [User Defined] | GeTui Master Secret | +| GETUI_APP_KEY | [User Defined] | GeTui Application Key | +| GETUI_INTENT | [User Defined] | GeTui Push Intent | +| GETUI_CHANNEL_ID | [User Defined] | GeTui Channel ID | +| GETUI_CHANNEL_NAME | [User Defined] | GeTui Channel Name | +| FCM_SERVICE_ACCOUNT | "x.json" | FCM Service Account | +| JPNS_APP_KEY | [User Defined] | JPNS Application Key | +| JPNS_MASTER_SECRET | [User Defined] | JPNS Master Secret | +| JPNS_PUSH_URL | [User Defined] | JPNS Push Notification URL | +| JPNS_PUSH_INTENT | [User Defined] | JPNS Push Intent | +| MANAGER_USERID_1 | "openIM123456" | Administrator ID 1 | +| MANAGER_USERID_2 | "openIM654321" | Administrator ID 2 | +| MANAGER_USERID_3 | "openIMAdmin" | Administrator ID 3 | +| NICKNAME_1 | "system1" | Nickname 1 | +| NICKNAME_2 | "system2" | Nickname 2 | +| NICKNAME_3 | "system3" | Nickname 3 | +| IM_ADMIN_USERID | "imAdmin" | IM Administrator ID | +| IM_ADMIN_NAME | "imAdmin" | IM Administrator Nickname | +| MULTILOGIN_POLICY | "1" | Multi-login Policy | +| CHAT_PERSISTENCE_MYSQL | "true" | Chat Persistence in MySQL | +| MSG_CACHE_TIMEOUT | "86400" | Message Cache Timeout | +| GROUP_MSG_READ_RECEIPT | "true" | Group Message Read Receipt Enable | | SINGLE_MSG_READ_RECEIPT | "true" | Single Message Read Receipt Enable | -| RETAIN_CHAT_RECORDS | "365" | Retain Chat Records (in days) | -| CHAT_RECORDS_CLEAR_TIME | [Cron Expression] | Chat Records Clear Time | -| MSG_DESTRUCT_TIME | [Cron Expression] | Message Destruct Time | -| SECRET | "${PASSWORD}" | Secret Key | -| TOKEN_EXPIRE | "90" | Token Expiry Time | -| FRIEND_VERIFY | "false" | Friend Verification Enable | -| IOS_PUSH_SOUND | "xxx" | iOS | -| CALLBACK_ENABLE | "false" | Enable callback | -| CALLBACK_TIMEOUT | "5" | Maximum timeout for callback call | +| RETAIN_CHAT_RECORDS | "365" | Retain Chat Records (in days) | +| CHAT_RECORDS_CLEAR_TIME | [Cron Expression] | Chat Records Clear Time | +| MSG_DESTRUCT_TIME | [Cron Expression] | Message Destruct Time | +| SECRET | "${PASSWORD}" | Secret Key | +| TOKEN_EXPIRE | "90" | Token Expiry Time | +| FRIEND_VERIFY | "false" | Friend Verification Enable | +| IOS_PUSH_SOUND | "xxx" | iOS | +| CALLBACK_ENABLE | "false" | Enable callback | +| CALLBACK_TIMEOUT | "5" | Maximum timeout for callback call | | CALLBACK_FAILED_CONTINUE| "true" | fails to continue to the next step | ### 2.20. Prometheus Configuration diff --git a/go.mod b/go.mod index 9a9cde30e..fb5be5b90 100644 --- a/go.mod +++ b/go.mod @@ -156,3 +156,5 @@ require ( golang.org/x/crypto v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/OpenIMSDK/protocol v0.0.36 => github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5 diff --git a/go.sum b/go.sum index 61323c57b..cde33c7cd 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/IBM/sarama v1.41.3 h1:MWBEJ12vHC8coMjdEXFq/6ftO6DUZnQlFYcxtOJFa7c= github.com/IBM/sarama v1.41.3/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= -github.com/OpenIMSDK/protocol v0.0.36 h1:UJnFsr1A4RrNeHMNDVS/1nvXWFoGM43dcXpZeJiIZ+0= -github.com/OpenIMSDK/protocol v0.0.36/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/tools v0.0.21 h1:iTapc2mIEVH/xl5Nd6jfwPub11Pgp44tVcE1rjB3a48= github.com/OpenIMSDK/tools v0.0.21/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= @@ -227,6 +225,8 @@ github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205Ah github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w= +github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5 h1:nmrJmAgQsCAxKgw109kaTcBV4rMWDRvqOson0ehw708= +github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= diff --git a/internal/api/friend.go b/internal/api/friend.go index 58e7398dd..23f337a9f 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -92,6 +92,3 @@ func (o *FriendApi) GetFriendIDs(c *gin.Context) { func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } -func (o *FriendApi) SetPinFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.PinFriends, o.Client, c) -} diff --git a/internal/api/msg.go b/internal/api/msg.go index 664ee505a..b6bfcebb6 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -169,9 +169,8 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM case constant.OANotification: data = apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType - if !authverify.IsManagerUserID(req.SendID) { - return nil, errs.ErrNoPermission. - Wrap("only app manager can as sender send OANotificationElem") + if err = m.userRpcClient.GetNotificationByID(c, req.SendID); err != nil { + return nil, err } default: return nil, errs.ErrArgs.WithDetail("not support err contentType") diff --git a/internal/api/route.go b/internal/api/route.go index 8bfea5cca..05aa6ac92 100644 --- a/internal/api/route.go +++ b/internal/api/route.go @@ -82,6 +82,10 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive userRouterGroup.POST("/process_user_command_delete", ParseToken, u.ProcessUserCommandDelete) userRouterGroup.POST("/process_user_command_update", ParseToken, u.ProcessUserCommandUpdate) userRouterGroup.POST("/process_user_command_get", ParseToken, u.ProcessUserCommandGet) + + userRouterGroup.POST("/add_notification_account", ParseToken, u.AddNotificationAccount) + userRouterGroup.POST("/update_notification_account", ParseToken, u.UpdateNotificationAccountInfo) + userRouterGroup.POST("/search_notification_account", ParseToken, u.SearchNotificationAccount) } // friend routing group friendRouterGroup := r.Group("/friend", ParseToken) @@ -103,7 +107,7 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive friendRouterGroup.POST("/is_friend", f.IsFriend) friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) - friendRouterGroup.POST("/set_pin_friend", f.SetPinFriends) + //friendRouterGroup.POST("/set_pin_friend", f.SetPinFriends) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group", ParseToken) diff --git a/internal/api/user.go b/internal/api/user.go index 8350d1711..cf68fb422 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -15,8 +15,6 @@ package api import ( - "github.com/gin-gonic/gin" - "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/msggateway" "github.com/OpenIMSDK/protocol/user" @@ -24,6 +22,7 @@ import ( "github.com/OpenIMSDK/tools/apiresp" "github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/log" + "github.com/gin-gonic/gin" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" @@ -219,3 +218,15 @@ func (u *UserApi) ProcessUserCommandUpdate(c *gin.Context) { func (u *UserApi) ProcessUserCommandGet(c *gin.Context) { a2r.Call(user.UserClient.ProcessUserCommandGet, u.Client, c) } + +func (u *UserApi) AddNotificationAccount(c *gin.Context) { + a2r.Call(user.UserClient.AddNotificationAccount, u.Client, c) +} + +func (u *UserApi) UpdateNotificationAccountInfo(c *gin.Context) { + a2r.Call(user.UserClient.UpdateNotificationAccountInfo, u.Client, c) +} + +func (u *UserApi) SearchNotificationAccount(c *gin.Context) { + a2r.Call(user.UserClient.SearchNotificationAccount, u.Client, c) +} diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 12203aa7e..97e53d197 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -53,6 +53,11 @@ type friendServer struct { RegisterCenter registry.SvcDiscoveryRegistry } +func (s *friendServer) UpdateFriends(ctx context.Context, req *pbfriend.UpdateFriendsReq) (*pbfriend.UpdateFriendsResp, error) { + //TODO implement me + panic("implement me") +} + func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error { // Initialize MongoDB mongo, err := unrelation.NewMongo() @@ -438,8 +443,8 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } func (s *friendServer) PinFriends( ctx context.Context, - req *pbfriend.PinFriendsReq, -) (*pbfriend.PinFriendsResp, error) { + req *pbfriend.UpdateFriendsReq, +) (*pbfriend.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.Wrap("friendIDList is empty") } @@ -465,6 +470,6 @@ func (s *friendServer) PinFriends( } } - resp := &pbfriend.PinFriendsResp{} + resp := &pbfriend.UpdateFriendsResp{} return resp, nil } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 70e3aebe6..9bc56298f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,6 +17,7 @@ package user import ( "context" "errors" + "math/rand" "strings" "time" @@ -72,6 +73,12 @@ func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error { for k, v := range config.Config.Manager.UserID { users = append(users, &tablerelation.UserModel{UserID: v, Nickname: config.Config.Manager.Nickname[k], AppMangerLevel: constant.AppAdmin}) } + if len(config.Config.IMAdmin.UserID) != len(config.Config.IMAdmin.Nickname) { + return errors.New("len(config.Config.AppNotificationAdmin.AppManagerUid) != len(config.Config.AppNotificationAdmin.Nickname)") + } + for k, v := range config.Config.IMAdmin.UserID { + users = append(users, &tablerelation.UserModel{UserID: v, Nickname: config.Config.IMAdmin.Nickname[k], AppMangerLevel: constant.AppNotificationAdmin}) + } userDB, err := mgo.NewUserMongo(mongo.GetDatabase()) if err != nil { return err @@ -390,3 +397,124 @@ func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.Proc // Return the response with the slice return &pbuser.ProcessUserCommandGetResp{KVArray: commandInfoSlice}, nil } + +func (s *userServer) AddNotificationAccount(ctx context.Context, req *pbuser.AddNotificationAccountReq) (*pbuser.AddNotificationAccountResp, error) { + if err := authverify.CheckIMAdmin(ctx); err != nil { + return nil, err + } + + var userID string + for i := 0; i < 20; i++ { + userId := s.genUserID() + _, err := s.UserDatabase.FindWithError(ctx, []string{userId}) + if err == nil { + continue + } + userID = userId + break + } + if userID == "" { + return nil, errs.ErrInternalServer.Wrap("gen user id failed") + } + + user := &tablerelation.UserModel{ + UserID: userID, + Nickname: req.NickName, + FaceURL: req.FaceURL, + CreateTime: time.Now(), + AppMangerLevel: constant.AppNotificationAdmin, + } + if err := s.UserDatabase.Create(ctx, []*tablerelation.UserModel{user}); err != nil { + return nil, err + } + + return &pbuser.AddNotificationAccountResp{}, nil +} + +func (s *userServer) UpdateNotificationAccountInfo(ctx context.Context, req *pbuser.UpdateNotificationAccountInfoReq) (*pbuser.UpdateNotificationAccountInfoResp, error) { + if err := authverify.CheckIMAdmin(ctx); err != nil { + return nil, err + } + + if _, err := s.UserDatabase.FindWithError(ctx, []string{req.UserID}); err != nil { + return nil, errs.ErrArgs.Wrap() + } + + user := map[string]interface{}{} + + if req.NickName != "" { + user["nickname"] = req.NickName + } + + if req.FaceURL != "" { + user["face_url"] = req.FaceURL + } + + if err := s.UserDatabase.UpdateByMap(ctx, req.UserID, user); err != nil { + return nil, err + } + + return &pbuser.UpdateNotificationAccountInfoResp{}, nil +} + +func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser.SearchNotificationAccountReq) (*pbuser.SearchNotificationAccountResp, error) { + if err := authverify.CheckIMAdmin(ctx); err != nil { + return nil, err + } + + _, users, err := s.UserDatabase.Page(ctx, req.Pagination) + if err != nil { + return nil, err + } + + var total int64 + accounts := make([]*pbuser.NotificationAccountInfo, 0, len(users)) + for _, v := range users { + if v.AppMangerLevel != constant.AppNotificationAdmin { + continue + } + temp := &pbuser.NotificationAccountInfo{ + UserID: v.UserID, + FaceURL: v.FaceURL, + NickName: v.Nickname, + } + accounts = append(accounts, temp) + total += 1 + } + return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: accounts}, nil +} + +func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (*pbuser.UpdateUserInfoExResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *userServer) GetNotificationAccount(ctx context.Context, req *pbuser.GetNotificationAccountReq) (*pbuser.GetNotificationAccountResp, error) { + if req.UserID == "" { + return nil, errs.ErrArgs.Wrap("userID is empty") + } + user, err := s.UserDatabase.GetUserByID(ctx, req.UserID) + if err != nil { + return nil, errs.ErrUserIDNotFound.Wrap() + } + if user.AppMangerLevel == constant.AppAdmin || user.AppMangerLevel == constant.AppNotificationAdmin { + return &pbuser.GetNotificationAccountResp{}, nil + } + + return nil, errs.ErrNoPermission.Wrap("notification messages cannot be sent for this ID") +} + +func (s *userServer) genUserID() string { + const l = 10 + data := make([]byte, l) + rand.Read(data) + chars := []byte("0123456789") + for i := 0; i < len(data); i++ { + if i == 0 { + data[i] = chars[1:][data[i]%9] + } else { + data[i] = chars[data[i]%10] + } + } + return string(data) +} diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index 4b7bd1e6f..d23db9bf5 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -87,7 +87,7 @@ type OANotificationElem struct { NotificationType int32 `mapstructure:"notificationType" json:"notificationType" validate:"required"` Text string `mapstructure:"text" json:"text" validate:"required"` Url string `mapstructure:"url" json:"url"` - MixType int32 `mapstructure:"mixType" json:"mixType" validate:"required"` + MixType int32 `mapstructure:"mixType" json:"mixType"` PictureElem *PictureElem `mapstructure:"pictureElem" json:"pictureElem"` SoundElem *SoundElem `mapstructure:"soundElem" json:"soundElem"` VideoElem *VideoElem `mapstructure:"videoElem" json:"videoElem"` diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index fd01e8c5a..d9aa0dbb1 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -54,6 +54,15 @@ func CheckAdmin(ctx context.Context) error { } return errs.ErrNoPermission.Wrap(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx))) } +func CheckIMAdmin(ctx context.Context) error { + if utils.IsContain(mcontext.GetOpUserID(ctx), config.Config.IMAdmin.UserID) { + return nil + } + if utils.IsContain(mcontext.GetOpUserID(ctx), config.Config.Manager.UserID) { + return nil + } + return errs.ErrNoPermission.Wrap(fmt.Sprintf("user %s is not CheckIMAdmin userID", mcontext.GetOpUserID(ctx))) +} func ParseRedisInterfaceToken(redisToken any) (*tokenverify.Claims, error) { return tokenverify.GetClaimFromToken(string(redisToken.([]uint8)), Secret()) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6da89fc8f..707601234 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -236,6 +236,11 @@ type configStruct struct { Nickname []string `yaml:"nickname"` } `yaml:"manager"` + IMAdmin struct { + UserID []string `yaml:"userID"` + Nickname []string `yaml:"nickname"` + } `yaml:"im-admin"` + MultiLoginPolicy int `yaml:"multiLoginPolicy"` ChatPersistenceMysql bool `yaml:"chatPersistenceMysql"` MsgCacheTimeout int `yaml:"msgCacheTimeout"` diff --git a/pkg/common/db/controller/user.go b/pkg/common/db/controller/user.go index 433fb7ad3..2fafcf266 100644 --- a/pkg/common/db/controller/user.go +++ b/pkg/common/db/controller/user.go @@ -50,6 +50,8 @@ type UserDatabase interface { IsExist(ctx context.Context, userIDs []string) (exist bool, err error) // GetAllUserID Get all user IDs GetAllUserID(ctx context.Context, pagination pagination.Pagination) (int64, []string, error) + // Get user by userID + GetUserByID(ctx context.Context, userID string) (user *relation.UserModel, err error) // InitOnce Inside the function, first query whether it exists in the db, if it exists, do nothing; if it does not exist, insert it InitOnce(ctx context.Context, users []*relation.UserModel) (err error) // CountTotal Get the total number of users @@ -183,6 +185,10 @@ func (u *userDatabase) GetAllUserID(ctx context.Context, pagination pagination.P return u.userDB.GetAllUserID(ctx, pagination) } +func (u *userDatabase) GetUserByID(ctx context.Context, userID string) (user *relation.UserModel, err error) { + return u.userDB.Take(ctx, userID) +} + // CountTotal Get the total number of users. func (u *userDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) { return u.userDB.CountTotal(ctx, before) diff --git a/pkg/common/http/http_client.go b/pkg/common/http/http_client.go index baad60459..a80d1c9a4 100644 --- a/pkg/common/http/http_client.go +++ b/pkg/common/http/http_client.go @@ -112,7 +112,6 @@ func callBackPostReturn(ctx context.Context, url, command string, input interfac //v.Set(constant.CallbackCommand, command) //url = url + "/" + v.Encode() url = url + "/" + command - b, err := Post(ctx, url, nil, input, callbackConfig.CallbackTimeOut) if err != nil { if callbackConfig.CallbackFailedContinue != nil && *callbackConfig.CallbackFailedContinue { @@ -121,6 +120,7 @@ func callBackPostReturn(ctx context.Context, url, command string, input interfac } return errs.ErrNetwork.Wrap(err.Error()) } + defer log.ZDebug(ctx, "callback", "data", string(b)) if err = json.Unmarshal(b, output); err != nil { if callbackConfig.CallbackFailedContinue != nil && *callbackConfig.CallbackFailedContinue { diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index c40d95727..de633ee30 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -179,3 +179,10 @@ func (u *UserRpcClient) SetUserStatus(ctx context.Context, userID string, status }) return err } + +func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) error { + _, err := u.Client.GetNotificationAccount(ctx, &user.GetNotificationAccountReq{ + UserID: userID, + }) + return err +} diff --git a/scripts/check-all.sh b/scripts/check-all.sh index efaa2ccb7..e1e07bd65 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -89,4 +89,4 @@ else echo "++++ Check all openim service ports successfully !" fi -set -e +set -e \ No newline at end of file diff --git a/scripts/install/environment.sh b/scripts/install/environment.sh index 3dd062af6..feb36c3c1 100755 --- a/scripts/install/environment.sh +++ b/scripts/install/environment.sh @@ -353,6 +353,8 @@ def "MANAGER_USERID_3" "openIMAdmin" # 管理员ID 3 def "NICKNAME_1" "system1" # 昵称1 def "NICKNAME_2" "system2" # 昵称2 def "NICKNAME_3" "system3" # 昵称3 +def "IM_ADMIN_USERID" "imAdmin" # IM管理员ID +def "IM_ADMIN_NAME" "imAdmin" # IM管理员昵称 def "MULTILOGIN_POLICY" "1" # 多登录策略 def "CHAT_PERSISTENCE_MYSQL" "true" # 聊天持久化MySQL def "MSG_CACHE_TIMEOUT" "86400" # 消息缓存超时 From 47dd6b17f61736686dabf3798888aaa7060d75a8 Mon Sep 17 00:00:00 2001 From: Xinwei Xiong <3293172751@qq.com> Date: Tue, 26 Dec 2023 12:16:55 +0800 Subject: [PATCH 2/4] Update openimci.yml (#1610) * Update openimci.yml * Update Makefile --- .github/workflows/openimci.yml | 8 +++++++- Makefile | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index d65c0dc89..d8e988f0b 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -207,6 +207,12 @@ jobs: sudo make check || \ (echo "An error occurred, printing logs:" && sudo cat ./_output/logs/* 2>/dev/null) + - name: Restart Services and Print Logs for Ubuntu + if: runner.os == 'Linux' + run: | + sudo make restart + sudo make check + # - name: Build, Start, Check Services and Print Logs for macOS # if: runner.os == 'macOS' # run: | @@ -239,4 +245,4 @@ jobs: - name: Test Docker Build run: | sudo make init - sudo make image \ No newline at end of file + sudo make image diff --git a/Makefile b/Makefile index 1941bde6c..4faf1c21d 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ stop: ## restart: Restart openim (make init configuration file is initialized) ✨ .PHONY: restart -restart: clean stop build init start check +restart: clean stop build start check ## multiarch: Build binaries for multiple platforms. See option PLATFORMS. ✨ .PHONY: multiarch From cd1235fb32072af665808d3bffd6f782a6643dd3 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 26 Dec 2023 17:29:42 +0800 Subject: [PATCH 3/4] feat: s3 FormData upload (#1614) * upgrade package and rtc convert * upgrade package and rtc convert * upgrade package and rtc convert * upgrade package and rtc convert * friend user * s3 form data * s3 form data * s3 form data * s3 form data * s3 form data * s3 form data * s3 form data * s3 form data * s3 form data --- go.mod | 8 +- go.sum | 8 +- internal/api/route.go | 2 + internal/api/third.go | 8 ++ internal/rpc/friend/friend.go | 3 +- internal/rpc/third/s3.go | 113 ++++++++++++++++++++++ internal/rpc/third/third.go | 10 -- internal/rpc/third/tool.go | 3 + internal/rpc/user/user.go | 9 +- pkg/common/cmd/root.go | 2 +- pkg/common/db/controller/s3.go | 10 ++ pkg/common/db/s3/cont/consts.go | 1 + pkg/common/db/s3/cont/controller.go | 4 + pkg/common/db/s3/cos/cos.go | 69 +++++++++++++ pkg/common/db/s3/minio/minio.go | 50 ++++++++++ pkg/common/db/s3/oss/oss.go | 49 ++++++++++ pkg/common/db/s3/s3.go | 11 +++ pkg/common/ginprometheus/ginprometheus.go | 2 +- 18 files changed, 334 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index fb5be5b90..a753cae51 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.19 require ( firebase.google.com/go v3.13.0+incompatible + github.com/OpenIMSDK/protocol v0.0.39 + github.com/OpenIMSDK/tools v0.0.21 github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/dtm-labs/rockscache v0.1.1 github.com/gin-gonic/gin v1.9.1 @@ -33,8 +35,6 @@ require github.com/google/uuid v1.3.1 require ( github.com/IBM/sarama v1.41.3 - github.com/OpenIMSDK/protocol v0.0.36 - github.com/OpenIMSDK/tools v0.0.21 github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/go-redis/redis v6.15.9+incompatible github.com/redis/go-redis/v9 v9.2.1 @@ -133,7 +133,7 @@ require ( golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect @@ -156,5 +156,3 @@ require ( golang.org/x/crypto v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -replace github.com/OpenIMSDK/protocol v0.0.36 => github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5 diff --git a/go.sum b/go.sum index cde33c7cd..a4609f6f2 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/IBM/sarama v1.41.3 h1:MWBEJ12vHC8coMjdEXFq/6ftO6DUZnQlFYcxtOJFa7c= github.com/IBM/sarama v1.41.3/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/OpenIMSDK/protocol v0.0.39 h1:DfvFcNGBcfj2vtT7W3uw4U/ipnI7NecTzQdlSYGuQz8= +github.com/OpenIMSDK/protocol v0.0.39/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/OpenIMSDK/tools v0.0.21 h1:iTapc2mIEVH/xl5Nd6jfwPub11Pgp44tVcE1rjB3a48= github.com/OpenIMSDK/tools v0.0.21/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= @@ -225,8 +227,6 @@ github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205Ah github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w= -github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5 h1:nmrJmAgQsCAxKgw109kaTcBV4rMWDRvqOson0ehw708= -github.com/luhaoling/protocol v0.0.0-20231222100538-d625562d53d5/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -453,8 +453,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/internal/api/route.go b/internal/api/route.go index 05aa6ac92..8fb372587 100644 --- a/internal/api/route.go +++ b/internal/api/route.go @@ -171,6 +171,8 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive objectGroup.POST("/auth_sign", t.AuthSign) objectGroup.POST("/complete_multipart_upload", t.CompleteMultipartUpload) objectGroup.POST("/access_url", t.AccessURL) + objectGroup.POST("/initiate_form_data", t.InitiateFormData) + objectGroup.POST("/complete_form_data", t.CompleteFormData) objectGroup.GET("/*name", t.ObjectRedirect) } // Message diff --git a/internal/api/third.go b/internal/api/third.go index 5191903da..37ec55098 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -71,6 +71,14 @@ func (o *ThirdApi) AccessURL(c *gin.Context) { a2r.Call(third.ThirdClient.AccessURL, o.Client, c) } +func (o *ThirdApi) InitiateFormData(c *gin.Context) { + a2r.Call(third.ThirdClient.InitiateFormData, o.Client, c) +} + +func (o *ThirdApi) CompleteFormData(c *gin.Context) { + a2r.Call(third.ThirdClient.CompleteFormData, o.Client, c) +} + func (o *ThirdApi) ObjectRedirect(c *gin.Context) { name := c.Param("name") if name == "" { diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 97e53d197..c40b566f3 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -54,8 +54,7 @@ type friendServer struct { } func (s *friendServer) UpdateFriends(ctx context.Context, req *pbfriend.UpdateFriendsReq) (*pbfriend.UpdateFriendsResp, error) { - //TODO implement me - panic("implement me") + return nil, errs.ErrInternalServer.Wrap("not implemented") } func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index ca826e805..2c230f258 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -16,6 +16,12 @@ package third import ( "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "github.com/google/uuid" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "path" "strconv" "time" @@ -179,6 +185,113 @@ func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (* }, nil } +func (t *thirdServer) InitiateFormData(ctx context.Context, req *third.InitiateFormDataReq) (*third.InitiateFormDataResp, error) { + if req.Name == "" { + return nil, errs.ErrArgs.Wrap("name is empty") + } + if req.Size <= 0 { + return nil, errs.ErrArgs.Wrap("size must be greater than 0") + } + if err := checkUploadName(ctx, req.Name); err != nil { + return nil, err + } + var duration time.Duration + opUserID := mcontext.GetOpUserID(ctx) + var key string + if authverify.IsManagerUserID(opUserID) { + if req.Millisecond <= 0 { + duration = time.Minute * 10 + } else { + duration = time.Millisecond * time.Duration(req.Millisecond) + } + if req.Absolute { + key = req.Name + } + } else { + duration = time.Minute * 10 + } + uid, err := uuid.NewRandom() + if err != nil { + return nil, err + } + if key == "" { + date := time.Now().Format("20060102") + key = path.Join(cont.DirectPath, date, opUserID, hex.EncodeToString(uid[:])+path.Ext(req.Name)) + } + mate := FormDataMate{ + Name: req.Name, + Size: req.Size, + ContentType: req.ContentType, + Group: req.Group, + Key: key, + } + mateData, err := json.Marshal(&mate) + if err != nil { + return nil, err + } + resp, err := t.s3dataBase.FormData(ctx, key, req.Size, req.ContentType, duration) + if err != nil { + return nil, err + } + return &third.InitiateFormDataResp{ + Id: base64.RawStdEncoding.EncodeToString(mateData), + Url: resp.URL, + File: resp.File, + Header: toPbMapArray(resp.Header), + FormData: resp.FormData, + Expires: resp.Expires.UnixMilli(), + SuccessCodes: utils.Slice(resp.SuccessCodes, func(code int) int32 { + return int32(code) + }), + }, nil +} + +func (t *thirdServer) CompleteFormData(ctx context.Context, req *third.CompleteFormDataReq) (*third.CompleteFormDataResp, error) { + if req.Id == "" { + return nil, errs.ErrArgs.Wrap("id is empty") + } + data, err := base64.RawStdEncoding.DecodeString(req.Id) + if err != nil { + return nil, errs.ErrArgs.Wrap("invalid id " + err.Error()) + } + var mate FormDataMate + if err := json.Unmarshal(data, &mate); err != nil { + return nil, errs.ErrArgs.Wrap("invalid id " + err.Error()) + } + if err := checkUploadName(ctx, mate.Name); err != nil { + return nil, err + } + info, err := t.s3dataBase.StatObject(ctx, mate.Key) + if err != nil { + return nil, err + } + if info.Size > 0 && info.Size != mate.Size { + return nil, errs.ErrData.Wrap("file size mismatch") + } + obj := &relation.ObjectModel{ + Name: mate.Name, + UserID: mcontext.GetOpUserID(ctx), + Hash: "etag_" + info.ETag, + Key: info.Key, + Size: info.Size, + ContentType: mate.ContentType, + Group: mate.Group, + CreateTime: time.Now(), + } + if err := t.s3dataBase.SetObject(ctx, obj); err != nil { + return nil, err + } + return &third.CompleteFormDataResp{Url: t.apiAddress(mate.Name)}, nil +} + func (t *thirdServer) apiAddress(name string) string { return t.apiURL + name } + +type FormDataMate struct { + Name string `json:"name"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + Group string `json:"group"` + Key string `json:"key"` +} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 35df3f925..7a63d3526 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -101,16 +101,6 @@ type thirdServer struct { defaultExpire time.Duration } -func (t *thirdServer) InitiateFormData(ctx context.Context, req *third.InitiateFormDataReq) (*third.InitiateFormDataResp, error) { - //TODO implement me - panic("implement me") -} - -func (t *thirdServer) CompleteFormData(ctx context.Context, req *third.CompleteFormDataReq) (*third.CompleteFormDataResp, error) { - //TODO implement me - panic("implement me") -} - func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/rpc/third/tool.go b/internal/rpc/third/tool.go index a65d882dd..a6c16ff9d 100644 --- a/internal/rpc/third/tool.go +++ b/internal/rpc/third/tool.go @@ -29,6 +29,9 @@ import ( ) func toPbMapArray(m map[string][]string) []*third.KeyValues { + if len(m) == 0 { + return nil + } res := make([]*third.KeyValues, 0, len(m)) for key := range m { res = append(res, &third.KeyValues{ diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 9bc56298f..b5ad186a5 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -57,6 +57,10 @@ type userServer struct { RegisterCenter registry.SvcDiscoveryRegistry } +func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (*pbuser.UpdateUserInfoExResp, error) { + return nil, errs.ErrInternalServer.Wrap("not implemented") +} + func Start(client registry.SvcDiscoveryRegistry, server *grpc.Server) error { rdb, err := cache.NewRedis() if err != nil { @@ -484,11 +488,6 @@ func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser. return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: accounts}, nil } -func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (*pbuser.UpdateUserInfoExResp, error) { - //TODO implement me - panic("implement me") -} - func (s *userServer) GetNotificationAccount(ctx context.Context, req *pbuser.GetNotificationAccountReq) (*pbuser.GetNotificationAccountResp, error) { if req.UserID == "" { return nil, errs.ErrArgs.Wrap("userID is empty") diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 0bc308e07..66bec61a7 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -45,7 +45,7 @@ type CmdOpts struct { func WithCronTaskLogName() func(*CmdOpts) { return func(opts *CmdOpts) { - opts.loggerPrefixName = "OpenIM.CronTask.log.all" + opts.loggerPrefixName = "openim.crontask.log.all" } } diff --git a/pkg/common/db/controller/s3.go b/pkg/common/db/controller/s3.go index 6916a7d30..95505de41 100644 --- a/pkg/common/db/controller/s3.go +++ b/pkg/common/db/controller/s3.go @@ -35,6 +35,8 @@ type S3Database interface { CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error) SetObject(ctx context.Context, info *relation.ObjectModel) error + StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) + FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database { @@ -100,3 +102,11 @@ func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Dur } return expireTime, rawURL, nil } + +func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) { + return s.s3.StatObject(ctx, name) +} + +func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { + return s.s3.FormData(ctx, name, size, contentType, duration) +} diff --git a/pkg/common/db/s3/cont/consts.go b/pkg/common/db/s3/cont/consts.go index 1a0467ce5..a01a8312c 100644 --- a/pkg/common/db/s3/cont/consts.go +++ b/pkg/common/db/s3/cont/consts.go @@ -17,6 +17,7 @@ package cont const ( hashPath = "openim/data/hash/" tempPath = "openim/temp/" + DirectPath = "openim/direct" UploadTypeMultipart = 1 // 分片上传 UploadTypePresigned = 2 // 预签名上传 partSeparator = "," diff --git a/pkg/common/db/s3/cont/controller.go b/pkg/common/db/s3/cont/controller.go index 1bf1a4b12..82c27c1f2 100644 --- a/pkg/common/db/s3/cont/controller.go +++ b/pkg/common/db/s3/cont/controller.go @@ -279,3 +279,7 @@ func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Dur } return c.impl.AccessURL(ctx, name, expire, opt) } + +func (c *Controller) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { + return c.impl.FormData(ctx, name, size, contentType, duration) +} diff --git a/pkg/common/db/s3/cos/cos.go b/pkg/common/db/s3/cos/cos.go index 7add88487..7d2c0befe 100644 --- a/pkg/common/db/s3/cos/cos.go +++ b/pkg/common/db/s3/cos/cos.go @@ -16,6 +16,11 @@ package cos import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "encoding/json" "errors" "fmt" "net/http" @@ -44,6 +49,8 @@ const ( imageWebp = "webp" ) +const successCode = http.StatusOK + const ( videoSnapshotImagePng = "png" videoSnapshotImageJpg = "jpg" @@ -326,3 +333,65 @@ func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Dura } return c.client.Object.GetObjectURL(name), nil } + +func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { + // https://cloud.tencent.com/document/product/436/14690 + now := time.Now() + expiration := now.Add(duration) + keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix()) + conditions := []any{ + map[string]string{"q-sign-algorithm": "sha1"}, + map[string]string{"q-ak": c.credential.SecretID}, + map[string]string{"q-sign-time": keyTime}, + map[string]string{"key": name}, + } + if contentType != "" { + conditions = append(conditions, map[string]string{"Content-Type": contentType}) + } + policy := map[string]any{ + "expiration": expiration.Format("2006-01-02T15:04:05.000Z"), + "conditions": conditions, + } + policyJson, err := json.Marshal(policy) + if err != nil { + return nil, err + } + signKey := hmacSha1val(c.credential.SecretKey, keyTime) + strToSign := sha1val(string(policyJson)) + signature := hmacSha1val(signKey, strToSign) + + fd := &s3.FormData{ + URL: c.client.BaseURL.BucketURL.String(), + File: "file", + Expires: expiration, + FormData: map[string]string{ + "policy": base64.StdEncoding.EncodeToString(policyJson), + "q-sign-algorithm": "sha1", + "q-ak": c.credential.SecretID, + "q-key-time": keyTime, + "q-signature": signature, + "key": name, + "success_action_status": strconv.Itoa(successCode), + }, + SuccessCodes: []int{successCode}, + } + if contentType != "" { + fd.FormData["Content-Type"] = contentType + } + if c.credential.SessionToken != "" { + fd.FormData["x-cos-security-token"] = c.credential.SessionToken + } + return fd, nil +} + +func hmacSha1val(key, msg string) string { + v := hmac.New(sha1.New, []byte(key)) + v.Write([]byte(msg)) + return hex.EncodeToString(v.Sum(nil)) +} + +func sha1val(msg string) string { + sha1Hash := sha1.New() + sha1Hash.Write([]byte(msg)) + return hex.EncodeToString(sha1Hash.Sum(nil)) +} diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index be49e2faa..7dfe35b51 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -57,6 +57,8 @@ const ( imageThumbnailPath = "openim/thumbnail" ) +const successCode = http.StatusOK + func NewMinio(cache cache.MinioCache) (s3.Interface, error) { u, err := url.Parse(config.Config.Object.Minio.Endpoint) if err != nil { @@ -441,3 +443,51 @@ func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([] } return io.ReadAll(io.LimitReader(object, limit)) } + +func (m *Minio) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { + if err := m.initMinio(ctx); err != nil { + return nil, err + } + policy := minio.NewPostPolicy() + if err := policy.SetKey(name); err != nil { + return nil, err + } + expires := time.Now().Add(duration) + if err := policy.SetExpires(expires); err != nil { + return nil, err + } + if size > 0 { + if err := policy.SetContentLengthRange(0, size); err != nil { + return nil, err + } + } + if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil { + return nil, err + } + if contentType != "" { + if err := policy.SetContentType(contentType); err != nil { + return nil, err + } + } + if err := policy.SetBucket(m.bucket); err != nil { + return nil, err + } + u, fd, err := m.core.PresignedPostPolicy(ctx, policy) + if err != nil { + return nil, err + } + sign, err := url.Parse(m.signEndpoint) + if err != nil { + return nil, err + } + u.Scheme = sign.Scheme + u.Host = sign.Host + return &s3.FormData{ + URL: u.String(), + File: "file", + Header: nil, + FormData: fd, + Expires: expires, + SuccessCodes: []int{successCode}, + }, nil +} diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 6a728127b..8fa2a538e 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -16,8 +16,13 @@ package oss import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "reflect" @@ -45,6 +50,8 @@ const ( imageWebp = "webp" ) +const successCode = http.StatusOK + const ( videoSnapshotImagePng = "png" videoSnapshotImageJpg = "jpg" @@ -327,3 +334,45 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, params := getURLParams(*o.bucket.Client.Conn, rawParams) return getURL(o.um, o.bucket.BucketName, name, params).String(), nil } + +func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { + // https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn + expires := time.Now().Add(duration) + conditions := []any{ + map[string]string{"bucket": o.bucket.BucketName}, + map[string]string{"key": name}, + } + if size > 0 { + conditions = append(conditions, []any{"content-length-range", 0, size}) + } + policy := map[string]any{ + "expiration": expires.Format("2006-01-02T15:04:05.000Z"), + "conditions": conditions, + } + policyJson, err := json.Marshal(policy) + if err != nil { + return nil, err + } + policyStr := base64.StdEncoding.EncodeToString(policyJson) + h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret())) + if _, err := io.WriteString(h, policyStr); err != nil { + return nil, err + } + fd := &s3.FormData{ + URL: o.bucketURL, + File: "file", + Expires: expires, + FormData: map[string]string{ + "key": name, + "policy": policyStr, + "OSSAccessKeyId": o.credentials.GetAccessKeyID(), + "success_action_status": strconv.Itoa(successCode), + "signature": base64.StdEncoding.EncodeToString(h.Sum(nil)), + }, + SuccessCodes: []int{successCode}, + } + if contentType != "" { + fd.FormData["x-oss-content-type"] = contentType + } + return fd, nil +} diff --git a/pkg/common/db/s3/s3.go b/pkg/common/db/s3/s3.go index afbe91955..0352004b5 100644 --- a/pkg/common/db/s3/s3.go +++ b/pkg/common/db/s3/s3.go @@ -74,6 +74,15 @@ type CopyObjectInfo struct { ETag string `json:"etag"` } +type FormData struct { + URL string `json:"url"` + File string `json:"file"` + Header http.Header `json:"header"` + FormData map[string]string `json:"form"` + Expires time.Time `json:"expires"` + SuccessCodes []int `json:"successActionStatus"` +} + type SignPart struct { PartNumber int `json:"partNumber"` URL string `json:"url"` @@ -152,4 +161,6 @@ type Interface interface { ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error) AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error) + + FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*FormData, error) } diff --git a/pkg/common/ginprometheus/ginprometheus.go b/pkg/common/ginprometheus/ginprometheus.go index f116fc23a..1ee8f8e34 100644 --- a/pkg/common/ginprometheus/ginprometheus.go +++ b/pkg/common/ginprometheus/ginprometheus.go @@ -432,7 +432,7 @@ func computeApproximateRequestSize(r *http.Request) int { } s += len(r.Host) - // r.Form and r.MultipartForm are assumed to be included in r.URL. + // r.FormData and r.MultipartForm are assumed to be included in r.URL. if r.ContentLength != -1 { s += int(r.ContentLength) From 222064542927b643ed00d6edd7cd0ec9114a84e9 Mon Sep 17 00:00:00 2001 From: Xinwei Xiong <3293172751@qq.com> Date: Tue, 26 Dec 2023 18:03:07 +0800 Subject: [PATCH 4/4] Update README.md (#1615) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a2c5fc732..781db1217 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,10 @@ It's crafted in Golang and supports cross-platform deployment, ensuring a cohere ## :rocket: Quick Start +We support many platforms. Here are the addresses for quick experience on the web side： + +👉 **[OpenIM online web demo](https://web-enterprise.rentsoft.cn/)** + You can quickly learn OpenIM engineering solutions, all it takes is one simple command: ```bash