Merge branch 'openimsdk:main' into main

pull/1115/head
withchao 2 years ago committed by GitHub
commit 359ffd7b83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -132,7 +132,7 @@ api:
# minio.signEndpoint is minio public network address
object:
enable: "minio"
apiURL: "http://http://127.0.0.1:10002"
apiURL: "http://127.0.0.1:10002"
minio:
bucket: "openim"
endpoint: "http://172.28.0.1:10005"

@ -37,7 +37,7 @@ require github.com/google/uuid v1.3.1
require (
github.com/IBM/sarama v1.41.1
github.com/OpenIMSDK/protocol v0.0.21
github.com/OpenIMSDK/protocol v0.0.23
github.com/OpenIMSDK/tools v0.0.14
github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
github.com/go-redis/redis v6.15.9+incompatible

@ -19,8 +19,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/IBM/sarama v1.41.1 h1:B4/TdHce/8Ipza+qrLIeNJ9D1AOxZVp/3uDv6H/dp2M=
github.com/IBM/sarama v1.41.1/go.mod h1:JFCPURVskaipJdKRFkiE/OZqQHw7jqliaJmRwXCmSSw=
github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY=
github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/OpenIMSDK/protocol v0.0.23 h1:L545aRQez6Ro+AaJB1Z6Mz7ojnDtp41WqASxYveCkcE=
github.com/OpenIMSDK/protocol v0.0.23/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/OpenIMSDK/tools v0.0.14 h1:WLof/+WxyPyRST+QkoTKubYCiV73uCLiL8pgnpH/yKQ=
github.com/OpenIMSDK/tools v0.0.14/go.mod h1:eg+q4A34Qmu73xkY0mt37FHGMCMfC6CtmOnm0kFEGFI=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=

@ -100,6 +100,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg)
ContentType: params.ContentType,
Content: []byte(newContent),
CreateTime: utils.GetCurrentTimestampByMill(),
SendTime: params.SendTime,
Options: options,
OfflinePushInfo: params.OfflinePushInfo,
},
@ -207,7 +208,6 @@ func (m *MessageApi) SendMessage(c *gin.Context) {
apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap())
return
}
log.ZInfo(c, "SendMessage", "req", req)
if !authverify.IsAppManagerUid(c) {
apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message"))
return
@ -224,6 +224,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) {
respPb, err := m.Client.SendMsg(c, sendMsgReq)
if err != nil {
status = constant.MsgSendFailed
log.ZError(c, "send message err", err)
apiresp.GinError(c, err)
return
}

@ -156,6 +156,11 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken)
thirdGroup.POST("/set_app_badge", t.SetAppBadge)
logs := thirdGroup.Group("/logs")
logs.POST("/upload", t.UploadLogs)
logs.POST("/delete", t.DeleteLogs)
logs.POST("/search", t.SearchLogs)
objectGroup := r.Group("/object", ParseToken)
objectGroup.POST("/part_limit", t.PartLimit)

@ -105,3 +105,16 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) {
}
c.Redirect(http.StatusFound, resp.Url)
}
// #################### logs ####################
func (o *ThirdApi) UploadLogs(c *gin.Context) {
a2r.Call(third.ThirdClient.UploadLogs, o.Client, c)
}
func (o *ThirdApi) DeleteLogs(c *gin.Context) {
a2r.Call(third.ThirdClient.DeleteLogs, o.Client, c)
}
func (o *ThirdApi) SearchLogs(c *gin.Context) {
a2r.Call(third.ThirdClient.SearchLogs, o.Client, c)
}

@ -59,7 +59,7 @@ const (
PongMessage = 10
)
type PongHandler func(string) error
type PingPongHandler func(string) error
type Client struct {
w *sync.Mutex
@ -107,8 +107,12 @@ func (c *Client) ResetClient(
c.token = token
}
func (c *Client) pongHandler(_ string) error {
func (c *Client) pingHandler(_ string) error {
c.conn.SetReadDeadline(pongWait)
err := c.writePongMsg()
if err != nil {
return err
}
return nil
}
@ -122,10 +126,11 @@ func (c *Client) readMessage() {
}()
c.conn.SetReadLimit(maxMessageSize)
_ = c.conn.SetReadDeadline(pongWait)
c.conn.SetPongHandler(c.pongHandler)
c.conn.SetPingHandler(c.pingHandler)
for {
messageType, message, returnErr := c.conn.ReadMessage()
if returnErr != nil {
log.ZWarn(c.ctx, "readMessage", returnErr, "messageType", messageType)
c.closedErr = returnErr
return
}

@ -41,7 +41,8 @@ type LongConn interface {
SetConnNil()
// SetReadLimit sets the maximum size for a message read from the peer.bytes
SetReadLimit(limit int64)
SetPongHandler(handler PongHandler)
SetPongHandler(handler PingPongHandler)
SetPingHandler(handler PingPongHandler)
// GenerateLongConn Check the connection of the current and when it was sent are the same
GenerateLongConn(w http.ResponseWriter, r *http.Request) error
}
@ -116,9 +117,12 @@ func (d *GWebSocket) SetReadLimit(limit int64) {
d.conn.SetReadLimit(limit)
}
func (d *GWebSocket) SetPongHandler(handler PongHandler) {
func (d *GWebSocket) SetPongHandler(handler PingPongHandler) {
d.conn.SetPongHandler(handler)
}
func (d *GWebSocket) SetPingHandler(handler PingPongHandler) {
d.conn.SetPingHandler(handler)
}
//func (d *GWebSocket) CheckSendConnDiffNow() bool {
// return d.conn == d.sendConn

@ -0,0 +1,146 @@
package third
import (
"context"
"crypto/rand"
"fmt"
"time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/third"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/utils"
utils2 "github.com/OpenIMSDK/tools/utils"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
func genLogID() string {
const dataLen = 10
data := make([]byte, dataLen)
rand.Read(data)
chars := []byte("0123456789")
for i := 0; i < len(data); i++ {
if i == 0 {
data[i] = chars[1:][data[i]%9]
} else {
data[i] = chars[data[i]%10]
}
}
return string(data)
}
func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) (*third.UploadLogsResp, error) {
var DBlogs []*relationtb.Log
userID := ctx.Value(constant.OpUserID).(string)
platform := constant.PlatformID2Name[int(req.Platform)]
for _, fileURL := range req.FileURLs {
log := relationtb.Log{
Version: req.Version,
SystemType: req.SystemType,
Platform: platform,
UserID: userID,
CreateTime: time.Now(),
Url: fileURL.URL,
FileName: fileURL.Filename,
}
for i := 0; i < 20; i++ {
id := genLogID()
logs, err := t.thirdDatabase.GetLogs(ctx, []string{id}, "")
if err != nil {
return nil, err
}
if len(logs) == 0 {
log.LogID = id
break
}
}
if log.LogID == "" {
return nil, errs.ErrData.Wrap("Log id gen error")
}
DBlogs = append(DBlogs, &log)
}
err := t.thirdDatabase.UploadLogs(ctx, DBlogs)
if err != nil {
return nil, err
}
return &third.UploadLogsResp{}, nil
}
func (t *thirdServer) DeleteLogs(ctx context.Context, req *third.DeleteLogsReq) (*third.DeleteLogsResp, error) {
if err := authverify.CheckAdmin(ctx); err != nil {
return nil, err
}
userID := ""
logs, err := t.thirdDatabase.GetLogs(ctx, req.LogIDs, userID)
if err != nil {
return nil, err
}
var logIDs []string
for _, log := range logs {
logIDs = append(logIDs, log.LogID)
}
if ids := utils2.Single(req.LogIDs, logIDs); len(ids) > 0 {
return nil, errs.ErrRecordNotFound.Wrap(fmt.Sprintf("logIDs not found%#v", ids))
}
err = t.thirdDatabase.DeleteLogs(ctx, req.LogIDs, userID)
if err != nil {
return nil, err
}
return &third.DeleteLogsResp{}, nil
}
func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo {
db2pbForLogInfo := func(log *relationtb.Log) *third.LogInfo {
return &third.LogInfo{
Filename: log.FileName,
UserID: log.UserID,
Platform: utils.StringToInt32(log.Platform),
Url: log.Url,
CreateTime: log.CreateTime.UnixMilli(),
LogID: log.LogID,
SystemType: log.SystemType,
Version: log.Version,
Ex: log.Ex,
}
}
return utils.Slice(logs, db2pbForLogInfo)
}
func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq) (*third.SearchLogsResp, error) {
if err := authverify.CheckAdmin(ctx); err != nil {
return nil, err
}
var (
resp third.SearchLogsResp
userIDs []string
)
if req.StartTime > req.EndTime {
return nil, errs.ErrArgs.Wrap("startTime>endTime")
}
total, logs, err := t.thirdDatabase.SearchLogs(ctx, req.Keyword, time.UnixMilli(req.StartTime), time.UnixMilli(req.EndTime), req.Pagination.PageNumber, req.Pagination.ShowNumber)
if err != nil {
return nil, err
}
pbLogs := dbToPbLogInfos(logs)
for _, log := range logs {
userIDs = append(userIDs, log.UserID)
}
users, err := t.thirdDatabase.FindUsers(ctx, userIDs)
if err != nil {
return nil, err
}
IDtoName := make(map[string]string)
for _, user := range users {
IDtoName[user.UserID] = user.Nickname
}
for _, pbLog := range pbLogs {
pbLog.Nickname = IDtoName[pbLog.UserID]
}
resp.LogsInfos = pbLogs
resp.Total = total
return &resp, nil
}

@ -35,6 +35,7 @@ import (
"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
)
@ -79,7 +80,7 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
}
third.RegisterThirdServer(server, &thirdServer{
apiURL: apiURL,
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb)),
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb), db),
userRpcClient: rpcclient.NewUserRpcClient(client),
s3dataBase: controller.NewS3Database(o, relation.NewObjectInfo(db)),
defaultExpire: time.Hour * 24 * 7,

@ -29,6 +29,7 @@ type SendMsg struct {
SessionType int32 `json:"sessionType" binding:"required"`
IsOnlineOnly bool `json:"isOnlineOnly"`
NotOfflinePush bool `json:"notOfflinePush"`
SendTime int64 `json:"sendTime"`
OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"`
}

@ -59,11 +59,8 @@ type ConversationCache interface {
DelConversations(ownerUserID string, conversationIDs ...string) ConversationCache
DelUsersConversation(conversationID string, ownerUserIDs ...string) ConversationCache
// get one conversation from msgCache
GetConversations(
ctx context.Context,
ownerUserID string,
conversationIDs []string,
) ([]*relationtb.ConversationModel, error)
GetConversations(ctx context.Context, ownerUserID string,
conversationIDs []string) ([]*relationtb.ConversationModel, error)
// get one user's all conversations from msgCache
GetUserAllConversations(ctx context.Context, ownerUserID string) ([]*relationtb.ConversationModel, error)
// get user conversation recv msg from msgCache
@ -79,10 +76,8 @@ type ConversationCache interface {
GetUserAllHasReadSeqs(ctx context.Context, ownerUserID string) (map[string]int64, error)
DelUserAllHasReadSeqs(ownerUserID string, conversationIDs ...string) ConversationCache
GetConversationsByConversationID(
ctx context.Context,
conversationIDs []string,
) ([]*relationtb.ConversationModel, error)
GetConversationsByConversationID(ctx context.Context,
conversationIDs []string) ([]*relationtb.ConversationModel, error)
DelConversationByConversationID(conversationIDs ...string) ConversationCache
GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error)
DelConversationNotReceiveMessageUserIDs(conversationIDs ...string) ConversationCache
@ -418,10 +413,8 @@ func (c *ConversationRedisCache) GetUserAllHasReadSeqs(
)
}
func (c *ConversationRedisCache) DelUserAllHasReadSeqs(
ownerUserID string,
conversationIDs ...string,
) ConversationCache {
func (c *ConversationRedisCache) DelUserAllHasReadSeqs(ownerUserID string,
conversationIDs ...string) ConversationCache {
cache := c.NewCache()
for _, conversationID := range conversationIDs {
cache.AddKeys(c.getConversationHasReadSeqKey(ownerUserID, conversationID))

@ -16,21 +16,60 @@ package controller
import (
"context"
"time"
"gorm.io/gorm"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
dbimpl "github.com/openimsdk/open-im-server/v3/pkg/common/db/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
type ThirdDatabase interface {
FcmUpdateToken(ctx context.Context, account string, platformID int, fcmToken string, expireTime int64) error
SetAppBadge(ctx context.Context, userID string, value int) error
//about log for debug
UploadLogs(ctx context.Context, logs []*relation.Log) error
DeleteLogs(ctx context.Context, logID []string, userID string) error
SearchLogs(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relation.Log, error)
GetLogs(ctx context.Context, LogIDs []string, userID string) ([]*relation.Log, error)
FindUsers(ctx context.Context, userIDs []string) ([]*relation.UserModel, error)
}
type thirdDatabase struct {
cache cache.MsgModel
logdb relation.LogInterface
userdb relation.UserModelInterface
}
// FindUsers implements ThirdDatabase.
func (t *thirdDatabase) FindUsers(ctx context.Context, userIDs []string) ([]*relation.UserModel, error) {
return t.userdb.Find(ctx, userIDs)
}
// DeleteLogs implements ThirdDatabase.
func (t *thirdDatabase) DeleteLogs(ctx context.Context, logID []string, userID string) error {
return t.logdb.Delete(ctx, logID, userID)
}
// GetLogs implements ThirdDatabase.
func (t *thirdDatabase) GetLogs(ctx context.Context, LogIDs []string, userID string) ([]*relation.Log, error) {
return t.logdb.Get(ctx, LogIDs, userID)
}
// SearchLogs implements ThirdDatabase.
func (t *thirdDatabase) SearchLogs(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relation.Log, error) {
return t.logdb.Search(ctx, keyword, start, end, pageNumber, showNumber)
}
// UploadLogs implements ThirdDatabase.
func (t *thirdDatabase) UploadLogs(ctx context.Context, logs []*relation.Log) error {
return t.logdb.Create(ctx, logs)
}
func NewThirdDatabase(cache cache.MsgModel) ThirdDatabase {
return &thirdDatabase{cache: cache}
func NewThirdDatabase(cache cache.MsgModel, db *gorm.DB) ThirdDatabase {
return &thirdDatabase{cache: cache, logdb: dbimpl.NewLogGorm(db), userdb: dbimpl.NewUserGorm(db)}
}
func (t *thirdDatabase) FcmUpdateToken(

@ -0,0 +1,47 @@
package relation
import (
"context"
"time"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/ormutil"
"gorm.io/gorm"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
)
type LogGorm struct {
db *gorm.DB
}
func (l *LogGorm) Create(ctx context.Context, log []*relationtb.Log) error {
return errs.Wrap(l.db.WithContext(ctx).Create(log).Error)
}
func (l *LogGorm) Search(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*relationtb.Log, error) {
db := l.db.WithContext(ctx).Where("create_time >= ?", start)
if end.UnixMilli() != 0 {
db = l.db.WithContext(ctx).Where("create_time <= ?", end)
}
return ormutil.GormSearch[relationtb.Log](db, []string{"user_id"}, keyword, pageNumber, showNumber)
}
func (l *LogGorm) Delete(ctx context.Context, logIDs []string, userID string) error {
if userID == "" {
return errs.Wrap(l.db.WithContext(ctx).Where("log_id in ?", logIDs).Delete(&relationtb.Log{}).Error)
}
return errs.Wrap(l.db.WithContext(ctx).Where("log_id in ? and user_id=?", logIDs, userID).Delete(&relationtb.Log{}).Error)
}
func (l *LogGorm) Get(ctx context.Context, logIDs []string, userID string) ([]*relationtb.Log, error) {
var logs []*relationtb.Log
if userID == "" {
return logs, errs.Wrap(l.db.WithContext(ctx).Where("log_id in ?", logIDs).Find(&logs).Error)
}
return logs, errs.Wrap(l.db.WithContext(ctx).Where("log_id in ? and user_id=?", logIDs, userID).Find(&logs).Error)
}
func NewLogGorm(db *gorm.DB) relationtb.LogInterface {
db.AutoMigrate(&relationtb.Log{})
return &LogGorm{db: db}
}

@ -0,0 +1,25 @@
package relation
import (
"context"
"time"
)
type Log struct {
LogID string `gorm:"column:log_id;primary_key;type:char(64)"`
Platform string `gorm:"column:platform;type:varchar(32)"`
UserID string `gorm:"column:user_id;type:char(64)"`
CreateTime time.Time `gorm:"index:,sort:desc"`
Url string `gorm:"column:url;type varchar(255)"`
FileName string `gorm:"column:filename;type varchar(255)"`
SystemType string `gorm:"column:system_type;type varchar(255)"`
Version string `gorm:"column:version;type varchar(255)"`
Ex string `gorm:"column:ex;type varchar(255)"`
}
type LogInterface interface {
Create(ctx context.Context, log []*Log) error
Search(ctx context.Context, keyword string, start time.Time, end time.Time, pageNumber int32, showNumber int32) (uint32, []*Log, error)
Delete(ctx context.Context, logID []string, userID string) error
Get(ctx context.Context, logIDs []string, userID string) ([]*Log, error)
}

@ -22,7 +22,6 @@ import (
"net"
"net/url"
"os"
"strings"
"time"
"github.com/minio/minio-go/v7"
@ -117,16 +116,24 @@ func main() {
os.Exit(1)
}
func exactIP(urll string) string {
u, _ := url.Parse(urll)
func checkMinioIP() error {
for _, i := range []string{config.Config.Object.ApiURL, config.Config.Object.Minio.SignEndpoint} {
u, err := url.Parse(i)
if err != nil {
return utils.Wrap(err, "api format error,please check config file apiURL or Minio SignEndpoint")
}
if u.Scheme == "https" {
continue
}
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
host = u.Host
}
if strings.HasSuffix(host, ":") {
host = host[0 : len(host)-1]
if host == "127.0.0.1" {
return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1,please modify it")
}
return host
}
return nil
}
func checkMysql() error {
@ -205,8 +212,8 @@ func checkMinio() error {
return ErrComponentStart.Wrap("Minio server is offline")
}
}
if exactIP(config.Config.Object.ApiURL) == "127.0.0.1" || exactIP(config.Config.Object.Minio.SignEndpoint) == "127.0.0.1" {
return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1")
if checkMinioIP() != nil {
return checkMinioIP()
}
}
return nil

@ -14,5 +14,6 @@ require (
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/sys v0.10.0 // indirect
k8s.io/kubernetes v1.28.2
)

@ -20,5 +20,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/kubernetes v1.28.2 h1:GhcnYeNTukeaC0dD5BC+UWBvzQsFEpWj7XBVMQptfYc=
k8s.io/kubernetes v1.28.2/go.mod h1:FmB1Mlp9ua0ezuwQCTGs/y6wj/fVisN2sVxhzjj0WDk=

Loading…
Cancel
Save