pluto
1 year ago
committed by
GitHub
commit
496d178313
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="WEB_MODULE" version="4">
|
||||
<component name="Go" enabled="true" />
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="grafana" uuid="95aae14a-3593-4ff7-ab49-5e4316cbecd1">
|
||||
<driver-ref>sqlite.xerial</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
|
||||
<jdbc-url>jdbc:sqlite:C:\Users\Administrator\Desktop\Open-IM-Server\docker-compose_cfg\grafana.db</jdbc-url>
|
||||
<driver-properties>
|
||||
<property name="enable_load_extension" value="true" />
|
||||
</driver-properties>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="JavaScriptSettings">
|
||||
<option name="languageLevel" value="ES6" />
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/Open-IM-Server.iml" filepath="$PROJECT_DIR$/.idea/Open-IM-Server.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@ -1,27 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
msgTransferCmd := cmd.NewMsgTransferCmd()
|
||||
msgTransferCmd.AddPrometheusPortFlag()
|
||||
if err := msgTransferCmd.Exec(); err != nil {
|
||||
panic(err.Error())
|
||||
}
|
||||
}
|
||||
@ -1,145 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msgtransfer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type ModifyMsgConsumerHandler struct {
|
||||
modifyMsgConsumerGroup *kfk.MConsumerGroup
|
||||
|
||||
extendMsgDatabase controller.ExtendMsgDatabase
|
||||
extendSetMsgModel unRelationTb.ExtendMsgSetModel
|
||||
}
|
||||
|
||||
func NewModifyMsgConsumerHandler(database controller.ExtendMsgDatabase) *ModifyMsgConsumerHandler {
|
||||
return &ModifyMsgConsumerHandler{
|
||||
modifyMsgConsumerGroup: kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V2_0_0_0,
|
||||
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.MsgToModify.Topic},
|
||||
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToModify),
|
||||
extendMsgDatabase: database,
|
||||
}
|
||||
}
|
||||
|
||||
func (ModifyMsgConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (ModifyMsgConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (mmc *ModifyMsgConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
|
||||
claim sarama.ConsumerGroupClaim) error {
|
||||
for msg := range claim.Messages() {
|
||||
ctx := mmc.modifyMsgConsumerGroup.GetContextFromMsg(msg)
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"kafka get info to mysql",
|
||||
"ModifyMsgConsumerHandler",
|
||||
msg.Topic,
|
||||
"msgPartition",
|
||||
msg.Partition,
|
||||
"msg",
|
||||
string(msg.Value),
|
||||
"key",
|
||||
string(msg.Key),
|
||||
)
|
||||
if len(msg.Value) != 0 {
|
||||
mmc.ModifyMsg(ctx, msg, string(msg.Key), sess)
|
||||
} else {
|
||||
log.ZError(ctx, "msg get from kafka but is nil", nil, "key", msg.Key)
|
||||
}
|
||||
sess.MarkMessage(msg, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mmc *ModifyMsgConsumerHandler) ModifyMsg(
|
||||
ctx context.Context,
|
||||
cMsg *sarama.ConsumerMessage,
|
||||
msgKey string,
|
||||
_ sarama.ConsumerGroupSession,
|
||||
) {
|
||||
msgFromMQ := pbMsg.MsgDataToModifyByMQ{}
|
||||
operationID := mcontext.GetOperationID(ctx)
|
||||
err := proto.Unmarshal(cMsg.Value, &msgFromMQ)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "msg_transfer Unmarshal msg err", err, "msg", string(cMsg.Value))
|
||||
return
|
||||
}
|
||||
log.ZDebug(ctx, "proto.Unmarshal MsgDataToMQ", "msgs", msgFromMQ.String())
|
||||
for _, msg := range msgFromMQ.Messages {
|
||||
isReactionFromCache := utils.GetSwitchFromOptions(msg.Options, constant.IsReactionFromCache)
|
||||
if !isReactionFromCache {
|
||||
continue
|
||||
}
|
||||
ctx = mcontext.SetOperationID(ctx, operationID)
|
||||
if msg.ContentType == constant.ReactionMessageModifier {
|
||||
notification := &sdkws.ReactionMessageModifierNotification{}
|
||||
if err := json.Unmarshal(msg.Content, notification); err != nil {
|
||||
continue
|
||||
}
|
||||
if notification.IsExternalExtensions {
|
||||
continue
|
||||
}
|
||||
if !notification.IsReact {
|
||||
// first time to modify
|
||||
var reactionExtensionList = make(map[string]unRelationTb.KeyValueModel)
|
||||
extendMsg := unRelationTb.ExtendMsgModel{
|
||||
ReactionExtensionList: reactionExtensionList,
|
||||
ClientMsgID: notification.ClientMsgID,
|
||||
MsgFirstModifyTime: notification.MsgFirstModifyTime,
|
||||
}
|
||||
for _, v := range notification.SuccessReactionExtensions {
|
||||
reactionExtensionList[v.TypeKey] = unRelationTb.KeyValueModel{
|
||||
TypeKey: v.TypeKey,
|
||||
Value: v.Value,
|
||||
LatestUpdateTime: v.LatestUpdateTime,
|
||||
}
|
||||
}
|
||||
|
||||
if err := mmc.extendMsgDatabase.InsertExtendMsg(ctx, notification.ConversationID, notification.SessionType, &extendMsg); err != nil {
|
||||
// log.ZError(ctx, "MsgFirstModify InsertExtendMsg failed", notification.ConversationID,
|
||||
// notification.SessionType, extendMsg, err.Error())
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if err := mmc.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
|
||||
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
|
||||
}
|
||||
}
|
||||
} else if msg.ContentType == constant.ReactionMessageDeleter {
|
||||
notification := &sdkws.ReactionMessageDeleteNotification{}
|
||||
if err := json.Unmarshal(msg.Content, notification); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := mmc.extendMsgDatabase.DeleteReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
|
||||
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package group
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/group"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCreateCountReq) (*group.GroupCreateCountResp, error) {
|
||||
if req.Start > req.End {
|
||||
return nil, errs.ErrArgs.Wrap("start > end")
|
||||
}
|
||||
total, err := s.GroupDatabase.CountTotal(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start := time.UnixMilli(req.Start)
|
||||
before, err := s.GroupDatabase.CountTotal(ctx, &start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count, err := s.GroupDatabase.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &group.GroupCreateCountResp{Total: total, Before: before, Count: count}, nil
|
||||
}
|
||||
@ -1,414 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
)
|
||||
|
||||
func (m *msgServer) SetMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.SetMessageReactionExtensionsReq,
|
||||
) (resp *msg.SetMessageReactionExtensionsResp, err error) {
|
||||
//resp = &msg.SetMessageReactionExtensionsResp{}
|
||||
////resp.ClientMsgID = req.ClientMsgID
|
||||
////resp.MsgFirstModifyTime = req.MsgFirstModifyTime
|
||||
//
|
||||
//if err := CallbackSetMessageReactionExtensions(ctx, req); err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
////if ExternalExtension
|
||||
//if req.IsExternalExtensions {
|
||||
// resp.MsgFirstModifyTime = req.MsgFirstModifyTime
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, !req.IsReact, false)
|
||||
// return resp, nil
|
||||
//}
|
||||
//isExists, err := m.MsgDatabase.JudgeMessageReactionExist(ctx, req.ClientMsgID, req.SessionType)
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
//
|
||||
//if !isExists {
|
||||
// if !req.IsReact {
|
||||
// resp.MsgFirstModifyTime = utils.GetCurrentTimestampByMill()
|
||||
// for k, v := range req.ReactionExtensions {
|
||||
// err := m.MessageLocker.LockMessageTypeKey(ctx, req.ClientMsgID, k)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// if err := m.MsgDatabase.SetMessageTypeKeyValue(ctx, req.ClientMsgID, req.SessionType, k,
|
||||
// utils.StructToJsonString(v)); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// }
|
||||
// resp.IsReact = true
|
||||
// _, err := m.MsgDatabase.SetMessageReactionExpire(ctx, req.ClientMsgID, req.SessionType,
|
||||
// time.Duration(24*3)*time.Hour)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// } else {
|
||||
// err := m.MessageLocker.LockGlobalMessage(ctx, req.ClientMsgID)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// mongoValue, err := m.MsgDatabase.GetExtendMsg(ctx, req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// setValue := make(map[string]*sdkws.KeyValue)
|
||||
// for k, v := range req.ReactionExtensions {
|
||||
//
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// if vv, ok := mongoValue.ReactionExtensions[k]; ok {
|
||||
// utils.CopyStructFields(temp, &vv)
|
||||
// if v.LatestUpdateTime != vv.LatestUpdateTime {
|
||||
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// }
|
||||
// }
|
||||
// temp.TypeKey = k
|
||||
// temp.Value = v.Value
|
||||
// temp.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// setValue[k] = temp
|
||||
// }
|
||||
// err = db.DB.InsertOrUpdateReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime, setValue)
|
||||
// if err != nil {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// resp.Result = append(resp.Result, temp)
|
||||
// }
|
||||
// } else {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// resp.Result = append(resp.Result, temp)
|
||||
// }
|
||||
// }
|
||||
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
|
||||
// if lockErr != nil {
|
||||
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//} else {
|
||||
// log.Debug(req.OperationID, "redis handle secondly", req.String())
|
||||
//
|
||||
// for k, v := range req.Pb2Model {
|
||||
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, k)
|
||||
// if err != nil {
|
||||
// setKeyResultInfo(&resp, 100, err.Error(), req.ClientMsgID, k, v)
|
||||
// continue
|
||||
// }
|
||||
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k)
|
||||
// if err != nil && err != go_redis.Nil {
|
||||
// setKeyResultInfo(&resp, 200, err.Error(), req.ClientMsgID, k, v)
|
||||
// continue
|
||||
// }
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(redisValue, temp)
|
||||
// if v.LatestUpdateTime != temp.LatestUpdateTime {
|
||||
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// } else {
|
||||
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// newerr := db.DB.SetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k, utils.StructToJsonString(v))
|
||||
// if newerr != nil {
|
||||
// setKeyResultInfo(&resp, 201, newerr.Error(), req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// }
|
||||
// setKeyResultInfo(&resp, 0, "", req.ClientMsgID, k, v)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//}
|
||||
//if !isExists {
|
||||
// if !req.IsReact {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, true, true)
|
||||
// } else {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, false, false)
|
||||
// }
|
||||
//} else {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, false, true)
|
||||
//}
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", resp.String())
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func (m *msgServer) setKeyResultInfo(
|
||||
ctx context.Context,
|
||||
r *msg.SetMessageReactionExtensionsResp,
|
||||
errCode int32,
|
||||
errMsg, clientMsgID, typeKey string,
|
||||
keyValue *sdkws.KeyValue,
|
||||
) {
|
||||
temp := new(msg.KeyValueResp)
|
||||
temp.KeyValue = keyValue
|
||||
temp.ErrCode = errCode
|
||||
temp.ErrMsg = errMsg
|
||||
r.Result = append(r.Result, temp)
|
||||
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
|
||||
}
|
||||
|
||||
func (m *msgServer) setDeleteKeyResultInfo(
|
||||
ctx context.Context,
|
||||
r *msg.DeleteMessagesReactionExtensionsResp,
|
||||
errCode int32,
|
||||
errMsg, clientMsgID, typeKey string,
|
||||
keyValue *sdkws.KeyValue,
|
||||
) {
|
||||
temp := new(msg.KeyValueResp)
|
||||
temp.KeyValue = keyValue
|
||||
temp.ErrCode = errCode
|
||||
temp.ErrMsg = errMsg
|
||||
r.Result = append(r.Result, temp)
|
||||
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
|
||||
}
|
||||
|
||||
func (m *msgServer) GetMessagesReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.GetMessagesReactionExtensionsReq,
|
||||
) (resp *msg.GetMessagesReactionExtensionsResp, err error) {
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
|
||||
//var rResp msg.GetMessageListReactionExtensionsResp
|
||||
//for _, messageValue := range req.MessageReactionKeyList {
|
||||
// var oneMessage msg.SingleMessageExtensionResult
|
||||
// oneMessage.ClientMsgID = messageValue.ClientMsgID
|
||||
//
|
||||
// isExists, err := db.DB.JudgeMessageReactionExist(messageValue.ClientMsgID, req.SessionType)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// if isExists {
|
||||
// redisValue, err := db.DB.GetOneMessageAllReactionList(messageValue.ClientMsgID, req.SessionType)
|
||||
// if err != nil {
|
||||
// oneMessage.ErrCode = 100
|
||||
// oneMessage.ErrMsg = err.Error()
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
// continue
|
||||
// }
|
||||
// keyMap := make(map[string]*sdkws.KeyValue)
|
||||
//
|
||||
// for k, v := range redisValue {
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(v, temp)
|
||||
// keyMap[k] = temp
|
||||
// }
|
||||
// oneMessage.Pb2Model = keyMap
|
||||
//
|
||||
// } else {
|
||||
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, messageValue.ClientMsgID,
|
||||
// messageValue.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// oneMessage.ErrCode = 100
|
||||
// oneMessage.ErrMsg = err.Error()
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
// continue
|
||||
// }
|
||||
// keyMap := make(map[string]*sdkws.KeyValue)
|
||||
//
|
||||
// for k, v := range mongoValue.Pb2Model {
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// temp.TypeKey = v.TypeKey
|
||||
// temp.Value = v.Value
|
||||
// temp.LatestUpdateTime = v.LatestUpdateTime
|
||||
// keyMap[k] = temp
|
||||
// }
|
||||
// oneMessage.Pb2Model = keyMap
|
||||
// }
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
//}
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func (m *msgServer) AddMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.ModifyMessageReactionExtensionsReq,
|
||||
) (resp *msg.ModifyMessageReactionExtensionsResp, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (m *msgServer) DeleteMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.DeleteMessagesReactionExtensionsReq,
|
||||
) (resp *msg.DeleteMessagesReactionExtensionsResp, err error) {
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
|
||||
//var rResp msg.DeleteMessagesReactionExtensionsResp
|
||||
//callbackResp := notification.callbackDeleteMessageReactionExtensions(req)
|
||||
//if callbackResp.ActionCode != constant.ActionAllow || callbackResp.ErrCode != 0 {
|
||||
// rResp.ErrCode = int32(callbackResp.ErrCode)
|
||||
// rResp.ErrMsg = callbackResp.ErrMsg
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = callbackResp.ErrMsg
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
//}
|
||||
////if ExternalExtension
|
||||
//if req.IsExternalExtensions {
|
||||
// rResp.Result = callbackResp.ResultReactionExtensionList
|
||||
// notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType,
|
||||
// req, &rResp, false, false)
|
||||
// return &rResp, nil
|
||||
//
|
||||
//}
|
||||
//for _, v := range callbackResp.ResultReactionExtensions {
|
||||
// if v.ErrCode != 0 {
|
||||
// func(req *[]*sdkws.KeyValue, typeKey string) {
|
||||
// for i := 0; i < len(*req); i++ {
|
||||
// if (*req)[i].TypeKey == typeKey {
|
||||
// *req = append((*req)[:i], (*req)[i+1:]...)
|
||||
// }
|
||||
// }
|
||||
// }(&req.Pb2Model, v.KeyValue.TypeKey)
|
||||
// rResp.Result = append(rResp.Result, v)
|
||||
// }
|
||||
//}
|
||||
//isExists, err := db.DB.JudgeMessageReactionExist(req.ClientMsgID, req.SessionType)
|
||||
//if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
//}
|
||||
//
|
||||
//if isExists {
|
||||
// log.Debug(req.OperationID, "redis handle this delete", req.String())
|
||||
// for _, v := range req.Pb2Model {
|
||||
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, v.TypeKey)
|
||||
// if err != nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 100, err.Error(), req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
//
|
||||
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, v.TypeKey)
|
||||
// if err != nil && err != go_redis.Nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 200, err.Error(), req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(redisValue, temp)
|
||||
// if v.LatestUpdateTime != temp.LatestUpdateTime {
|
||||
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// } else {
|
||||
// newErr := db.DB.DeleteOneMessageKey(req.ClientMsgID, req.SessionType, v.TypeKey)
|
||||
// if newErr != nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 201, newErr.Error(), req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// }
|
||||
// setDeleteKeyResultInfo(&rResp, 0, "", req.ClientMsgID, v.TypeKey, v)
|
||||
// }
|
||||
// }
|
||||
//} else {
|
||||
// err := m.dMessageLocker.LockGlobalMessage(req.ClientMsgID)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 200
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// setValue := make(map[string]*sdkws.KeyValue)
|
||||
// for _, v := range req.Pb2Model {
|
||||
//
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// if vv, ok := mongoValue.Pb2Model[v.TypeKey]; ok {
|
||||
// utils.CopyStructFields(temp, &vv)
|
||||
// if v.LatestUpdateTime != vv.LatestUpdateTime {
|
||||
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// }
|
||||
// } else {
|
||||
// setDeleteKeyResultInfo(&rResp, 400, "key not in", req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
// temp.TypeKey = v.TypeKey
|
||||
// setValue[v.TypeKey] = temp
|
||||
// }
|
||||
// err = db.DB.DeleteReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime, setValue)
|
||||
// if err != nil {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// } else {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// }
|
||||
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
|
||||
// if lockErr != nil {
|
||||
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
|
||||
// }
|
||||
//
|
||||
//}
|
||||
// notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType,
|
||||
// req, &rResp, false, isExists)
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
|
||||
return resp, nil
|
||||
}
|
||||
@ -1,105 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
cbapi "github.com/OpenIMSDK/Open-IM-Server/pkg/callbackstruct"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/http"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
)
|
||||
|
||||
func callbackSetMessageReactionExtensions(ctx context.Context, setReq *msg.SetMessageReactionExtensionsReq) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackBeforeSetMessageReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackBeforeSetMessageReactionExtensionCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: mcontext.GetOpUserID(ctx),
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsReact: setReq.IsReact,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackBeforeSetMessageReactionExtResp{}
|
||||
if err := http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
setReq.MsgFirstModifyTime = resp.MsgFirstModifyTime
|
||||
return nil
|
||||
}
|
||||
|
||||
func callbackDeleteMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
setReq *msg.DeleteMessagesReactionExtensionsReq,
|
||||
) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackDeleteMessageReactionExtReq{
|
||||
OperationID: setReq.OperationID,
|
||||
CallbackCommand: constant.CallbackBeforeDeleteMessageReactionExtensionsCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: setReq.OpUserID,
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackDeleteMessageReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
||||
|
||||
func callbackGetMessageListReactionExtensions(ctx context.Context, getReq *msg.GetMessagesReactionExtensionsReq) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackGetMessageListReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackGetMessageListReactionExtensionsCommand,
|
||||
ConversationID: getReq.ConversationID,
|
||||
OpUserID: mcontext.GetOperationID(ctx),
|
||||
SessionType: getReq.SessionType,
|
||||
TypeKeyList: getReq.TypeKeys,
|
||||
}
|
||||
resp := &cbapi.CallbackGetMessageListReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
||||
|
||||
func callbackAddMessageReactionExtensions(ctx context.Context, setReq *msg.ModifyMessageReactionExtensionsReq) error {
|
||||
req := &cbapi.CallbackAddMessageReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackAddMessageListReactionExtensionsCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: mcontext.GetOperationID(ctx),
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsReact: setReq.IsReact,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackAddMessageReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
||||
@ -0,0 +1,84 @@
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq) (*msg.GetActiveUserResp, error) {
|
||||
msgCount, userCount, users, dateCount, err := m.MsgDatabase.RangeUserSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Group, req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pbUsers []*msg.ActiveUser
|
||||
if len(users) > 0 {
|
||||
userIDs := utils.Slice(users, func(e *unrelation.UserCount) string { return e.UserID })
|
||||
userMap, err := m.User.GetUsersInfoMap(ctx, userIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pbUsers = make([]*msg.ActiveUser, 0, len(users))
|
||||
for _, user := range users {
|
||||
pbUser := userMap[user.UserID]
|
||||
if pbUser == nil {
|
||||
pbUser = &sdkws.UserInfo{
|
||||
UserID: user.UserID,
|
||||
Nickname: user.UserID,
|
||||
}
|
||||
}
|
||||
pbUsers = append(pbUsers, &msg.ActiveUser{
|
||||
User: pbUser,
|
||||
Count: user.Count,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &msg.GetActiveUserResp{
|
||||
MsgCount: msgCount,
|
||||
UserCount: userCount,
|
||||
DateCount: dateCount,
|
||||
Users: pbUsers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *msgServer) GetActiveGroup(ctx context.Context, req *msg.GetActiveGroupReq) (*msg.GetActiveGroupResp, error) {
|
||||
msgCount, groupCount, groups, dateCount, err := m.MsgDatabase.RangeGroupSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pbGroups []*msg.ActiveGroup
|
||||
if len(groups) > 0 {
|
||||
groupIDs := utils.Slice(groups, func(e *unrelation.GroupCount) string { return e.GroupID })
|
||||
resp, err := m.Group.GetGroupInfos(ctx, groupIDs, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groupMap := make(map[string]*sdkws.GroupInfo, len(groups))
|
||||
for i, group := range groups {
|
||||
groupMap[group.GroupID] = resp[i]
|
||||
}
|
||||
pbGroups = make([]*msg.ActiveGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
pbGroup := groupMap[group.GroupID]
|
||||
if pbGroup == nil {
|
||||
pbGroup = &sdkws.GroupInfo{
|
||||
GroupID: group.GroupID,
|
||||
GroupName: group.GroupID,
|
||||
}
|
||||
}
|
||||
pbGroups = append(pbGroups, &msg.ActiveGroup{
|
||||
Group: pbGroup,
|
||||
Count: group.Count,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &msg.GetActiveGroupResp{
|
||||
MsgCount: msgCount,
|
||||
GroupCount: groupCount,
|
||||
DateCount: dateCount,
|
||||
Groups: pbGroups,
|
||||
}, nil
|
||||
}
|
||||
@ -1,52 +1,151 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package third
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (t *thirdServer) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
|
||||
return t.s3dataBase.ApplyPut(ctx, req)
|
||||
func (t *thirdServer) PartLimit(ctx context.Context, req *third.PartLimitReq) (*third.PartLimitResp, error) {
|
||||
limit := t.s3dataBase.PartLimit()
|
||||
return &third.PartLimitResp{
|
||||
MinPartSize: limit.MinPartSize,
|
||||
MaxPartSize: limit.MaxPartSize,
|
||||
MaxNumSize: int32(limit.MaxNumSize),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
|
||||
return t.s3dataBase.GetPut(ctx, req)
|
||||
func (t *thirdServer) PartSize(ctx context.Context, req *third.PartSizeReq) (*third.PartSizeResp, error) {
|
||||
size, err := t.s3dataBase.PartSize(ctx, req.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.PartSizeResp{Size: size}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error) {
|
||||
return t.s3dataBase.ConfirmPut(ctx, req)
|
||||
func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if err := checkUploadName(ctx, req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expireTime := time.Now().Add(t.defaultExpire)
|
||||
result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, t.defaultExpire, int(req.MaxParts))
|
||||
if err != nil {
|
||||
if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok {
|
||||
obj := &relation.ObjectModel{
|
||||
Name: req.Name,
|
||||
UserID: mcontext.GetOpUserID(ctx),
|
||||
Hash: req.Hash,
|
||||
Key: haErr.Object.Key,
|
||||
Size: haErr.Object.Size,
|
||||
ContentType: req.ContentType,
|
||||
Cause: req.Cause,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.InitiateMultipartUploadResp{
|
||||
Url: t.apiAddress(obj.Name),
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var sign *third.AuthSignParts
|
||||
if result.Sign != nil && len(result.Sign.Parts) > 0 {
|
||||
sign = &third.AuthSignParts{
|
||||
Url: result.Sign.URL,
|
||||
Query: toPbMapArray(result.Sign.Query),
|
||||
Header: toPbMapArray(result.Sign.Header),
|
||||
Parts: make([]*third.SignPart, len(result.Sign.Parts)),
|
||||
}
|
||||
for i, part := range result.Sign.Parts {
|
||||
sign.Parts[i] = &third.SignPart{
|
||||
PartNumber: int32(part.PartNumber),
|
||||
Url: part.URL,
|
||||
Query: toPbMapArray(part.Query),
|
||||
Header: toPbMapArray(part.Header),
|
||||
}
|
||||
}
|
||||
}
|
||||
return &third.InitiateMultipartUploadResp{
|
||||
Upload: &third.UploadInfo{
|
||||
UploadID: result.UploadID,
|
||||
PartSize: result.PartSize,
|
||||
Sign: sign,
|
||||
ExpireTime: expireTime.UnixMilli(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
|
||||
if req.Expires <= 0 {
|
||||
if err := tokenverify.CheckAdmin(ctx); err != nil {
|
||||
return nil, err
|
||||
func (t *thirdServer) AuthSign(ctx context.Context, req *third.AuthSignReq) (*third.AuthSignResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
partNumbers := utils.Slice(req.PartNumbers, func(partNumber int32) int { return int(partNumber) })
|
||||
result, err := t.s3dataBase.AuthSign(ctx, req.UploadID, partNumbers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp := &third.AuthSignResp{
|
||||
Url: result.URL,
|
||||
Query: toPbMapArray(result.Query),
|
||||
Header: toPbMapArray(result.Header),
|
||||
Parts: make([]*third.SignPart, len(result.Parts)),
|
||||
}
|
||||
for i, part := range result.Parts {
|
||||
resp.Parts[i] = &third.SignPart{
|
||||
PartNumber: int32(part.PartNumber),
|
||||
Url: part.URL,
|
||||
Query: toPbMapArray(part.Query),
|
||||
Header: toPbMapArray(part.Header),
|
||||
}
|
||||
}
|
||||
return t.s3dataBase.GetUrl(ctx, req)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (*third.CompleteMultipartUploadResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if err := checkUploadName(ctx, req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := t.s3dataBase.CompleteMultipartUpload(ctx, req.UploadID, req.Parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
obj := &relation.ObjectModel{
|
||||
Name: req.Name,
|
||||
UserID: mcontext.GetOpUserID(ctx),
|
||||
Hash: result.Hash,
|
||||
Key: result.Key,
|
||||
Size: result.Size,
|
||||
ContentType: req.ContentType,
|
||||
Cause: req.Cause,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.CompleteMultipartUploadResp{
|
||||
Url: t.apiAddress(obj.Name),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
|
||||
return t.s3dataBase.GetHashInfo(ctx, req)
|
||||
func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) {
|
||||
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.AccessURLResp{
|
||||
Url: rawURL,
|
||||
ExpireTime: expireTime.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) CleanObject(ctx context.Context, now time.Time) {
|
||||
t.s3dataBase.CleanExpirationObject(ctx, now)
|
||||
func (t *thirdServer) apiAddress(name string) string {
|
||||
return t.apiURL + name
|
||||
}
|
||||
|
||||
@ -0,0 +1,63 @@
|
||||
package third
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func toPbMapArray(m map[string][]string) []*third.KeyValues {
|
||||
res := make([]*third.KeyValues, 0, len(m))
|
||||
for key := range m {
|
||||
res = append(res, &third.KeyValues{
|
||||
Key: key,
|
||||
Values: m[key],
|
||||
})
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func checkUploadName(ctx context.Context, name string) error {
|
||||
if name == "" {
|
||||
return errs.ErrArgs.Wrap("name is empty")
|
||||
}
|
||||
if name[0] == '/' {
|
||||
return errs.ErrArgs.Wrap("name cannot start with `/`")
|
||||
}
|
||||
if err := checkValidObjectName(name); err != nil {
|
||||
return errs.ErrArgs.Wrap(err.Error())
|
||||
}
|
||||
opUserID := mcontext.GetOpUserID(ctx)
|
||||
if opUserID == "" {
|
||||
return errs.ErrNoPermission.Wrap("opUserID is empty")
|
||||
}
|
||||
if !tokenverify.IsManagerUserID(opUserID) {
|
||||
if !strings.HasPrefix(name, opUserID+"/") {
|
||||
return errs.ErrNoPermission.Wrap(fmt.Sprintf("name must start with `%s/`", opUserID))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkValidObjectNamePrefix(objectName string) error {
|
||||
if len(objectName) > 1024 {
|
||||
return errors.New("object name cannot be longer than 1024 characters")
|
||||
}
|
||||
if !utf8.ValidString(objectName) {
|
||||
return errors.New("object name with non UTF-8 strings are not supported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkValidObjectName(objectName string) error {
|
||||
if strings.TrimSpace(objectName) == "" {
|
||||
return errors.New("object name cannot be empty")
|
||||
}
|
||||
return checkValidObjectNamePrefix(objectName)
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
func (c *MsgTool) ConversationsDestructMsgs() {
|
||||
log.ZInfo(context.Background(), "start msg destruct cron task")
|
||||
ctx := mcontext.NewCtx(utils.GetSelfFuncName())
|
||||
conversations, err := c.conversationDatabase.GetConversationIDsNeedDestruct(ctx)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "get conversation id need destruct failed", err)
|
||||
return
|
||||
}
|
||||
log.ZDebug(context.Background(), "nums conversations need destruct", "nums", len(conversations))
|
||||
for _, conversation := range conversations {
|
||||
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "lastMsgDestructTime", conversation.LatestMsgDestructTime)
|
||||
seqs, err := c.msgDatabase.UserMsgsDestruct(ctx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
continue
|
||||
}
|
||||
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]interface{}{"latest_msg_destruct_time": time.Now()}); err != nil {
|
||||
log.ZError(ctx, "updateUsersConversationFiled failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
continue
|
||||
}
|
||||
if len(seqs) > 0 {
|
||||
if err := c.msgNotificationSender.UserDeleteMsgsNotification(ctx, conversation.OwnerUserID, conversation.ConversationID, seqs); err != nil {
|
||||
log.ZError(ctx, "userDeleteMsgsNotification failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
package apiresp
|
||||
|
||||
type ApiFormat interface {
|
||||
ApiFormat()
|
||||
}
|
||||
@ -1,101 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/dtm-labs/rockscache"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
)
|
||||
|
||||
const (
|
||||
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
|
||||
extendMsgCache = "EXTEND_MSG_CACHE:"
|
||||
)
|
||||
|
||||
type ExtendMsgSetCache interface {
|
||||
metaCache
|
||||
NewCache() ExtendMsgSetCache
|
||||
GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
firstModifyTime int64,
|
||||
) (extendMsg *unrelation.ExtendMsgModel, err error)
|
||||
DelExtendMsg(clientMsgID string) ExtendMsgSetCache
|
||||
}
|
||||
|
||||
type ExtendMsgSetCacheRedis struct {
|
||||
metaCache
|
||||
expireTime time.Duration
|
||||
rcClient *rockscache.Client
|
||||
extendMsgSetDB unrelation.ExtendMsgSetModelInterface
|
||||
}
|
||||
|
||||
func NewExtendMsgSetCacheRedis(
|
||||
rdb redis.UniversalClient,
|
||||
extendMsgSetDB unrelation.ExtendMsgSetModelInterface,
|
||||
options rockscache.Options,
|
||||
) ExtendMsgSetCache {
|
||||
rcClient := rockscache.NewClient(rdb, options)
|
||||
return &ExtendMsgSetCacheRedis{
|
||||
metaCache: NewMetaCacheRedis(rcClient),
|
||||
expireTime: time.Second * 30 * 60,
|
||||
extendMsgSetDB: extendMsgSetDB,
|
||||
rcClient: rcClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) NewCache() ExtendMsgSetCache {
|
||||
return &ExtendMsgSetCacheRedis{
|
||||
metaCache: NewMetaCacheRedis(e.rcClient, e.metaCache.GetPreDelKeys()...),
|
||||
expireTime: e.expireTime,
|
||||
extendMsgSetDB: e.extendMsgSetDB,
|
||||
rcClient: e.rcClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) getKey(clientMsgID string) string {
|
||||
return extendMsgCache + clientMsgID
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
firstModifyTime int64,
|
||||
) (extendMsg *unrelation.ExtendMsgModel, err error) {
|
||||
return getCache(
|
||||
ctx,
|
||||
e.rcClient,
|
||||
e.getKey(clientMsgID),
|
||||
e.expireTime,
|
||||
func(ctx context.Context) (*unrelation.ExtendMsgModel, error) {
|
||||
return e.extendMsgSetDB.TakeExtendMsg(ctx, conversationID, sessionType, clientMsgID, firstModifyTime)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) DelExtendMsg(clientMsgID string) ExtendMsgSetCache {
|
||||
new := e.NewCache()
|
||||
new.AddKeys(e.getKey(clientMsgID))
|
||||
return new
|
||||
}
|
||||
@ -1,158 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
|
||||
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
|
||||
)
|
||||
|
||||
// for mongoDB
|
||||
type ExtendMsgDatabase interface {
|
||||
CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error
|
||||
GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
ID string,
|
||||
opts *unRelationTb.GetAllExtendMsgSetOpts,
|
||||
) (sets []*unRelationTb.ExtendMsgSetModel, err error)
|
||||
GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*unRelationTb.ExtendMsgSetModel, error)
|
||||
InsertExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
msg *unRelationTb.ExtendMsgModel,
|
||||
) error
|
||||
InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error
|
||||
DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error
|
||||
GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *unRelationTb.ExtendMsgModel, err error)
|
||||
}
|
||||
|
||||
type extendMsgDatabase struct {
|
||||
database unRelationTb.ExtendMsgSetModelInterface
|
||||
cache cache.ExtendMsgSetCache
|
||||
ctxTx tx.CtxTx
|
||||
}
|
||||
|
||||
func NewExtendMsgDatabase(
|
||||
extendMsgModel unRelationTb.ExtendMsgSetModelInterface,
|
||||
cache cache.ExtendMsgSetCache,
|
||||
ctxTx tx.CtxTx,
|
||||
) ExtendMsgDatabase {
|
||||
return &extendMsgDatabase{database: extendMsgModel, cache: cache, ctxTx: ctxTx}
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
|
||||
return e.database.CreateExtendMsgSet(ctx, set)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
opts *unRelationTb.GetAllExtendMsgSetOpts,
|
||||
) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
|
||||
return e.database.GetAllExtendMsgSet(ctx, conversationID, opts)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*unRelationTb.ExtendMsgSetModel, error) {
|
||||
return e.database.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) InsertExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
msg *unRelationTb.ExtendMsgModel,
|
||||
) error {
|
||||
return e.database.InsertExtendMsg(ctx, conversationID, sessionType, msg)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
return e.database.InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx,
|
||||
conversationID,
|
||||
sessionType,
|
||||
clientMsgID,
|
||||
msgFirstModifyTime,
|
||||
reactionExtensionList,
|
||||
)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
return e.database.DeleteReactionExtendMsgSet(
|
||||
ctx,
|
||||
conversationID,
|
||||
sessionType,
|
||||
clientMsgID,
|
||||
msgFirstModifyTime,
|
||||
reactionExtensionList,
|
||||
)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
|
||||
return e.cache.GetExtendMsg(ctx, conversationID, sessionType, clientMsgID, maxMsgUpdateTime)
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
package controller
|
||||
|
||||
import "C"
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
type S3Database interface {
|
||||
PartLimit() *s3.PartLimit
|
||||
PartSize(ctx context.Context, size int64) (int64, error)
|
||||
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
|
||||
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
|
||||
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
|
||||
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error)
|
||||
SetObject(ctx context.Context, info *relation.ObjectModel) error
|
||||
}
|
||||
|
||||
func NewS3Database(s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
|
||||
return &s3Database{
|
||||
s3: cont.New(s3),
|
||||
obj: obj,
|
||||
}
|
||||
}
|
||||
|
||||
type s3Database struct {
|
||||
s3 *cont.Controller
|
||||
obj relation.ObjectInfoModelInterface
|
||||
}
|
||||
|
||||
func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
return s.s3.PartSize(ctx, size)
|
||||
}
|
||||
|
||||
func (s *s3Database) PartLimit() *s3.PartLimit {
|
||||
return s.s3.PartLimit()
|
||||
}
|
||||
|
||||
func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
return s.s3.AuthSign(ctx, uploadID, partNumbers)
|
||||
}
|
||||
|
||||
func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) {
|
||||
return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts)
|
||||
}
|
||||
|
||||
func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) {
|
||||
return s.s3.CompleteUpload(ctx, uploadID, parts)
|
||||
}
|
||||
|
||||
func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) error {
|
||||
return s.obj.SetObject(ctx, info)
|
||||
}
|
||||
|
||||
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) {
|
||||
obj, err := s.obj.Take(ctx, name)
|
||||
if err != nil {
|
||||
return time.Time{}, "", err
|
||||
}
|
||||
opt := &s3.AccessURLOption{
|
||||
ContentType: obj.ContentType,
|
||||
}
|
||||
if filename := filepath.Base(obj.Name); filename != "" {
|
||||
opt.ContentDisposition = `attachment; filename=` + filename
|
||||
}
|
||||
expireTime := time.Now().Add(expire)
|
||||
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)
|
||||
if err != nil {
|
||||
return time.Time{}, "", err
|
||||
}
|
||||
return expireTime, rawURL, nil
|
||||
}
|
||||
@ -1,567 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
hashPrefix = "hash"
|
||||
tempPrefix = "temp"
|
||||
fragmentPrefix = "fragment_"
|
||||
urlsName = "urls.json"
|
||||
)
|
||||
|
||||
type S3Database interface {
|
||||
ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
|
||||
GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
|
||||
ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
|
||||
GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error)
|
||||
GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error)
|
||||
CleanExpirationObject(ctx context.Context, t time.Time)
|
||||
}
|
||||
|
||||
func NewS3Database(
|
||||
obj obj.Interface,
|
||||
hash relation.ObjectHashModelInterface,
|
||||
info relation.ObjectInfoModelInterface,
|
||||
put relation.ObjectPutModelInterface,
|
||||
url *url.URL,
|
||||
) S3Database {
|
||||
return &s3Database{
|
||||
url: url,
|
||||
obj: obj,
|
||||
hash: hash,
|
||||
info: info,
|
||||
put: put,
|
||||
}
|
||||
}
|
||||
|
||||
type s3Database struct {
|
||||
url *url.URL
|
||||
obj obj.Interface
|
||||
hash relation.ObjectHashModelInterface
|
||||
info relation.ObjectInfoModelInterface
|
||||
put relation.ObjectPutModelInterface
|
||||
}
|
||||
|
||||
// today 今天的日期
|
||||
func (c *s3Database) today() string {
|
||||
return time.Now().Format("20060102")
|
||||
}
|
||||
|
||||
// fragmentName 根据序号生成文件名
|
||||
func (c *s3Database) fragmentName(index int) string {
|
||||
return fragmentPrefix + strconv.Itoa(index+1)
|
||||
}
|
||||
|
||||
// getFragmentNum 获取分片大小和分片数量
|
||||
func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
|
||||
if size := c.obj.MinFragmentSize(); fragmentSize < size {
|
||||
fragmentSize = size
|
||||
}
|
||||
if fragmentSize <= 0 || objectSize <= fragmentSize {
|
||||
return objectSize, 1
|
||||
} else {
|
||||
num := int(objectSize / fragmentSize)
|
||||
if objectSize%fragmentSize > 0 {
|
||||
num++
|
||||
}
|
||||
if n := c.obj.MaxFragmentNum(); num > n {
|
||||
num = n
|
||||
}
|
||||
return fragmentSize, num
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) CheckHash(hash string) error {
|
||||
val, err := hex.DecodeString(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(val) != md5.Size {
|
||||
return errs.ErrArgs.Wrap("invalid hash")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3Database) urlName(name string) string {
|
||||
u := url.URL{
|
||||
Scheme: c.url.Scheme,
|
||||
Opaque: c.url.Opaque,
|
||||
User: c.url.User,
|
||||
Host: c.url.Host,
|
||||
Path: c.url.Path,
|
||||
RawPath: c.url.RawPath,
|
||||
ForceQuery: c.url.ForceQuery,
|
||||
RawQuery: c.url.RawQuery,
|
||||
Fragment: c.url.Fragment,
|
||||
RawFragment: c.url.RawFragment,
|
||||
}
|
||||
v := make(url.Values, 1)
|
||||
v.Set("name", name)
|
||||
u.RawQuery = v.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (c *s3Database) UUID() string {
|
||||
return uuid.New().String()
|
||||
}
|
||||
|
||||
func (c *s3Database) HashName(hash string) string {
|
||||
return path.Join(hashPrefix, hash+"_"+c.today()+"_"+c.UUID())
|
||||
}
|
||||
|
||||
func (c *s3Database) isNotFound(err error) bool {
|
||||
return relation.IsNotFound(err)
|
||||
}
|
||||
|
||||
func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
|
||||
if err := c.CheckHash(req.Hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.obj.CheckName(req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if req.ValidTime != 0 && req.ValidTime <= time.Now().UnixMilli() {
|
||||
return nil, errors.New("invalid ValidTime")
|
||||
}
|
||||
var expirationTime *time.Time
|
||||
if req.ValidTime != 0 {
|
||||
expirationTime = utils.ToPtr(time.UnixMilli(req.ValidTime))
|
||||
}
|
||||
if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
|
||||
o := relation.ObjectInfoModel{
|
||||
Name: req.Name,
|
||||
Hash: hash.Hash,
|
||||
ValidTime: expirationTime,
|
||||
ContentType: req.ContentType,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, &o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
// 新上传
|
||||
var fragmentNum int
|
||||
const effective = time.Hour * 24 * 2
|
||||
req.FragmentSize, fragmentNum = c.getFragmentNum(req.FragmentSize, req.Size)
|
||||
put := relation.ObjectPutModel{
|
||||
PutID: req.PutID,
|
||||
Hash: req.Hash,
|
||||
Name: req.Name,
|
||||
ObjectSize: req.Size,
|
||||
ContentType: req.ContentType,
|
||||
FragmentSize: req.FragmentSize,
|
||||
ValidTime: expirationTime,
|
||||
EffectiveTime: time.Now().Add(effective),
|
||||
}
|
||||
if put.PutID == "" {
|
||||
put.PutID = c.UUID()
|
||||
}
|
||||
if v, err := c.put.Take(ctx, put.PutID); err == nil {
|
||||
now := time.Now().UnixMilli()
|
||||
if v.EffectiveTime.UnixMilli() <= now {
|
||||
if err := c.put.DelPut(ctx, []string{v.PutID}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, errs.ErrDuplicateKey.Wrap(fmt.Sprintf("duplicate put id %s", put.PutID))
|
||||
}
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
put.Path = path.Join(tempPrefix, c.today(), req.Hash, put.PutID)
|
||||
putURLs := make([]string, 0, fragmentNum)
|
||||
for i := 0; i < fragmentNum; i++ {
|
||||
url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: path.Join(put.Path, c.fragmentName(i)),
|
||||
Effective: effective,
|
||||
MaxObjectSize: req.FragmentSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
putURLs = append(putURLs, url)
|
||||
}
|
||||
urlsJsonData, err := json.Marshal(putURLs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := md5.Sum(urlsJsonData)
|
||||
put.PutURLsHash = hex.EncodeToString(t[:])
|
||||
_, err = c.obj.PutObject(
|
||||
ctx,
|
||||
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(put.Path, urlsName)},
|
||||
bytes.NewReader(urlsJsonData),
|
||||
int64(len(urlsJsonData)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
put.CreateTime = time.Now()
|
||||
if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.ApplyPutResp{
|
||||
PutID: put.PutID,
|
||||
FragmentSize: put.FragmentSize,
|
||||
PutURLs: putURLs,
|
||||
ValidTime: put.EffectiveTime.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
|
||||
up, err := c.put.Take(ctx, req.PutID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := c.obj.GetObject(
|
||||
ctx,
|
||||
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(up.Path, urlsName)},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urlsData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := md5.Sum(urlsData)
|
||||
if h := hex.EncodeToString(t[:]); h != up.PutURLsHash {
|
||||
return nil, fmt.Errorf("invalid put urls hash %s %s", h, up.PutURLsHash)
|
||||
}
|
||||
var urls []string
|
||||
if err := json.Unmarshal(urlsData, &urls); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, fragmentNum := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
|
||||
if len(urls) != fragmentNum {
|
||||
return nil, fmt.Errorf("invalid urls length %d fragment %d", len(urls), fragmentNum)
|
||||
}
|
||||
fragments := make([]*third.GetPutFragment, fragmentNum)
|
||||
for i := 0; i < fragmentNum; i++ {
|
||||
name := path.Join(up.Path, c.fragmentName(i))
|
||||
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
if c.obj.IsNotFound(err) {
|
||||
fragments[i] = &third.GetPutFragment{Url: urls[i]}
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash, Url: urls[i]}
|
||||
}
|
||||
var validTime int64
|
||||
if up.ValidTime != nil {
|
||||
validTime = up.ValidTime.UnixMilli()
|
||||
}
|
||||
return &third.GetPutResp{
|
||||
FragmentSize: up.FragmentSize,
|
||||
Size: up.ObjectSize,
|
||||
Name: up.Name,
|
||||
Hash: up.Hash,
|
||||
Fragments: fragments,
|
||||
PutURLsHash: up.PutURLsHash,
|
||||
ContentType: up.ContentType,
|
||||
ValidTime: validTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
|
||||
put, err := c.put.Take(ctx, req.PutID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, pack := c.getFragmentNum(put.FragmentSize, put.ObjectSize)
|
||||
defer func() {
|
||||
if _err == nil {
|
||||
// 清理上传的碎片
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "deleteObject failed", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
}
|
||||
}
|
||||
}()
|
||||
now := time.Now().UnixMilli()
|
||||
if put.EffectiveTime.UnixMilli() < now {
|
||||
return nil, errs.ErrFileUploadedExpired.Wrap("put expired")
|
||||
}
|
||||
if put.ValidTime != nil && put.ValidTime.UnixMilli() < now {
|
||||
return nil, errs.ErrFileUploadedExpired.Wrap("object expired")
|
||||
}
|
||||
if hash, err := c.hash.Take(ctx, put.Hash, c.obj.Name()); err == nil {
|
||||
o := relation.ObjectInfoModel{
|
||||
Name: put.Name,
|
||||
Hash: hash.Hash,
|
||||
ValidTime: put.ValidTime,
|
||||
ContentType: put.ContentType,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, &o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: put.Path,
|
||||
})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
}
|
||||
}()
|
||||
// 服务端已存在
|
||||
return &third.ConfirmPutResp{
|
||||
Url: c.urlName(o.Name),
|
||||
}, nil
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
src := make([]obj.BucketObject, pack)
|
||||
for i := 0; i < pack; i++ {
|
||||
name := path.Join(put.Path, c.fragmentName(i))
|
||||
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i+1 == pack { // 最后一个
|
||||
size := put.ObjectSize - put.FragmentSize*int64(i)
|
||||
if size != o.Size {
|
||||
return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
|
||||
}
|
||||
} else {
|
||||
if o.Size != put.FragmentSize {
|
||||
return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, put.FragmentSize, o.Hash)
|
||||
}
|
||||
}
|
||||
src[i] = obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
}
|
||||
}
|
||||
dst := &obj.BucketObject{
|
||||
Bucket: c.obj.DataBucket(),
|
||||
Name: c.HashName(put.Hash),
|
||||
}
|
||||
if len(src) == 1 { // 未分片直接触发copy
|
||||
// 检查数据完整性,避免脏数据
|
||||
o, err := c.obj.GetObjectInfo(ctx, &src[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if put.ObjectSize != o.Size {
|
||||
return nil, fmt.Errorf("size mismatching should %d reality %d", put.ObjectSize, o.Size)
|
||||
}
|
||||
if put.Hash != o.Hash {
|
||||
return nil, fmt.Errorf("hash mismatching should %s reality %s", put.Hash, o.Hash)
|
||||
}
|
||||
if err := c.obj.CopyObject(ctx, &src[0], dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
tempBucket := &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: path.Join(put.Path, "merge_"+c.UUID()),
|
||||
}
|
||||
defer func() { // 清理合成的文件
|
||||
if err := c.obj.DeleteObject(ctx, tempBucket); err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", tempBucket.Bucket, "Path", tempBucket.Name)
|
||||
}
|
||||
}()
|
||||
err := c.obj.ComposeObject(ctx, src, tempBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := c.obj.GetObjectInfo(ctx, tempBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if put.ObjectSize != info.Size {
|
||||
return nil, fmt.Errorf("size mismatch should %d reality %d", put.ObjectSize, info.Size)
|
||||
}
|
||||
if put.Hash != info.Hash {
|
||||
return nil, fmt.Errorf("hash mismatch should %s reality %s", put.Hash, info.Hash)
|
||||
}
|
||||
if err := c.obj.CopyObject(ctx, tempBucket, dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
h := &relation.ObjectHashModel{
|
||||
Hash: put.Hash,
|
||||
Engine: c.obj.Name(),
|
||||
Size: put.ObjectSize,
|
||||
Bucket: c.obj.DataBucket(),
|
||||
Name: dst.Name,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := &relation.ObjectInfoModel{
|
||||
Name: put.Name,
|
||||
Hash: put.Hash,
|
||||
ContentType: put.ContentType,
|
||||
ValidTime: put.ValidTime,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.put.DelPut(ctx, []string{put.PutID}); err != nil {
|
||||
log.ZError(ctx, "DelPut", err, "PutID", put.PutID)
|
||||
}
|
||||
return &third.ConfirmPutResp{
|
||||
Url: c.urlName(o.Name),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
|
||||
info, err := c.info.Take(ctx, req.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.ValidTime != nil && info.ValidTime.Before(time.Now()) {
|
||||
return nil, errs.ErrRecordNotFound.Wrap("object expired")
|
||||
}
|
||||
hash, err := c.hash.Take(ctx, info.Hash, c.obj.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt := obj.HeaderOption{ContentType: info.ContentType}
|
||||
if req.Attachment {
|
||||
opt.Filename = info.Name
|
||||
}
|
||||
u, err := c.obj.PresignedGetURL(ctx, hash.Bucket, hash.Name, time.Duration(req.Expires)*time.Millisecond, &opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.GetUrlResp{
|
||||
Url: u,
|
||||
Size: hash.Size,
|
||||
Hash: hash.Hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) CleanExpirationObject(ctx context.Context, t time.Time) {
|
||||
// 清理上传产生的临时文件
|
||||
c.cleanPutTemp(ctx, t, 10)
|
||||
// 清理hash引用全过期的文件
|
||||
c.cleanExpirationObject(ctx, t)
|
||||
// 清理没有引用的hash对象
|
||||
c.clearNoCitation(ctx, c.obj.Name(), 10)
|
||||
}
|
||||
|
||||
func (c *s3Database) cleanPutTemp(ctx context.Context, t time.Time, num int) {
|
||||
for {
|
||||
puts, err := c.put.FindExpirationPut(ctx, t, num)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "FindExpirationPut", err, "Time", t, "Num", num)
|
||||
return
|
||||
}
|
||||
if len(puts) == 0 {
|
||||
return
|
||||
}
|
||||
for _, put := range puts {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
return
|
||||
}
|
||||
}
|
||||
ids := utils.Slice(puts, func(e *relation.ObjectPutModel) string { return e.PutID })
|
||||
err = c.put.DelPut(ctx, ids)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DelPut", err, "PutID", ids)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) cleanExpirationObject(ctx context.Context, t time.Time) {
|
||||
err := c.info.DeleteExpiration(ctx, t)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteExpiration", err, "Time", t)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) clearNoCitation(ctx context.Context, engine string, limit int) {
|
||||
for {
|
||||
list, err := c.hash.DeleteNoCitation(ctx, engine, limit)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteNoCitation", err, "Engine", engine, "Limit", limit)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
var hasErr bool
|
||||
for _, h := range list {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: h.Bucket, Name: h.Name})
|
||||
if err != nil {
|
||||
hasErr = true
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", h.Bucket, "Path", h.Name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if hasErr {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
|
||||
if err := c.CheckHash(req.Hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o, err := c.hash.Take(ctx, req.Hash, c.obj.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.GetHashInfoResp{
|
||||
Hash: o.Hash,
|
||||
Size: o.Size,
|
||||
}, nil
|
||||
}
|
||||
@ -1,258 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package obj
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
func NewMinioInterface() (Interface, error) {
|
||||
conf := config.Config.Object.Minio
|
||||
u, err := url.Parse(conf.Endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio endpoint parse %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid minio endpoint scheme %s", u.Scheme)
|
||||
}
|
||||
client, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio new client %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
|
||||
defer cancel()
|
||||
initBucket := func(ctx context.Context) error {
|
||||
for _, bucket := range utils.Distinct([]string{conf.TempBucket, conf.DataBucket}) {
|
||||
exists, err := client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio bucket %s exists %w", bucket, err)
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
opt := minio.MakeBucketOptions{
|
||||
Region: conf.Location,
|
||||
ObjectLocking: conf.IsDistributedMod,
|
||||
}
|
||||
if err := client.MakeBucket(ctx, bucket, opt); err != nil {
|
||||
return fmt.Errorf("minio make bucket %s %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := initBucket(ctx); err != nil {
|
||||
fmt.Println("minio init error:", err)
|
||||
}
|
||||
return &minioImpl{
|
||||
client: client,
|
||||
tempBucket: conf.TempBucket,
|
||||
dataBucket: conf.DataBucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type minioImpl struct {
|
||||
tempBucket string // 上传桶
|
||||
dataBucket string // 永久桶
|
||||
urlstr string // 访问地址
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func (m *minioImpl) Name() string {
|
||||
return "minio"
|
||||
}
|
||||
|
||||
func (m *minioImpl) MinFragmentSize() int64 {
|
||||
return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
|
||||
}
|
||||
|
||||
func (m *minioImpl) MaxFragmentNum() int {
|
||||
return 1000 // 最大分片数量 minio.maxPartsCount
|
||||
}
|
||||
|
||||
func (m *minioImpl) MinExpirationTime() time.Duration {
|
||||
return time.Hour * 24
|
||||
}
|
||||
|
||||
func (m *minioImpl) TempBucket() string {
|
||||
return m.tempBucket
|
||||
}
|
||||
|
||||
func (m *minioImpl) DataBucket() string {
|
||||
return m.dataBucket
|
||||
}
|
||||
|
||||
func (m *minioImpl) PresignedGetURL(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
name string,
|
||||
expires time.Duration,
|
||||
opt *HeaderOption,
|
||||
) (string, error) {
|
||||
var reqParams url.Values
|
||||
if opt != nil {
|
||||
reqParams = make(url.Values)
|
||||
if opt.ContentType != "" {
|
||||
reqParams.Set("response-content-type", opt.ContentType)
|
||||
}
|
||||
if opt.Filename != "" {
|
||||
reqParams.Set("response-content-disposition", "attachment;filename="+opt.Filename)
|
||||
}
|
||||
}
|
||||
u, err := m.client.PresignedGetObject(ctx, bucket, name, expires, reqParams)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
|
||||
if args.Effective <= 0 {
|
||||
return "", errors.New("EffectiveTime <= 0")
|
||||
}
|
||||
_, err := m.GetObjectInfo(ctx, &BucketObject{
|
||||
Bucket: m.tempBucket,
|
||||
Name: args.Name,
|
||||
})
|
||||
if err == nil {
|
||||
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
|
||||
} else if !m.IsNotFound(err) {
|
||||
return "", err
|
||||
}
|
||||
u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio apply error: %w", err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
|
||||
info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ObjectInfo{
|
||||
Size: info.Size,
|
||||
Hash: info.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
|
||||
_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
|
||||
Bucket: dst.Bucket,
|
||||
Object: dst.Name,
|
||||
}, minio.CopySrcOptions{
|
||||
Bucket: src.Bucket,
|
||||
Object: src.Name,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *minioImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
|
||||
return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (m *minioImpl) MoveObjectInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
|
||||
if err := m.CopyObject(ctx, src, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.DeleteObject(ctx, src)
|
||||
}
|
||||
|
||||
func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
|
||||
destOptions := minio.CopyDestOptions{
|
||||
Bucket: dst.Bucket,
|
||||
Object: dst.Name + ".temp",
|
||||
}
|
||||
sources := make([]minio.CopySrcOptions, len(src))
|
||||
for i, s := range src {
|
||||
sources[i] = minio.CopySrcOptions{
|
||||
Bucket: s.Bucket,
|
||||
Object: s.Name,
|
||||
}
|
||||
}
|
||||
_, err := m.client.ComposeObject(ctx, destOptions, sources...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.MoveObjectInfo(ctx, &BucketObject{
|
||||
Bucket: destOptions.Bucket,
|
||||
Name: destOptions.Object,
|
||||
}, &BucketObject{
|
||||
Bucket: dst.Bucket,
|
||||
Name: dst.Name,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *minioImpl) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (m *minioImpl) PutObject(
|
||||
ctx context.Context,
|
||||
info *BucketObject,
|
||||
reader io.Reader,
|
||||
size int64,
|
||||
) (*ObjectInfo, error) {
|
||||
update, err := m.client.PutObject(ctx, info.Bucket, info.Name, reader, size, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ObjectInfo{
|
||||
Size: update.Size,
|
||||
Hash: update.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
|
||||
object, err := m.client.GetObject(ctx, info.Bucket, info.Name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := object.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewSizeReader(object, stat.Size), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) CheckName(name string) error {
|
||||
return s3utils.CheckValidObjectName(name)
|
||||
}
|
||||
@ -1,110 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package obj
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BucketObject struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type ApplyPutArgs struct {
|
||||
Bucket string
|
||||
Name string
|
||||
Effective time.Duration // 申请有效时间
|
||||
Header http.Header // header
|
||||
MaxObjectSize int64
|
||||
}
|
||||
|
||||
type HeaderOption struct {
|
||||
ContentType string
|
||||
Filename string
|
||||
}
|
||||
|
||||
type ObjectInfo struct {
|
||||
Size int64
|
||||
Hash string
|
||||
}
|
||||
|
||||
type SizeReader interface {
|
||||
io.ReadCloser
|
||||
Size() int64
|
||||
}
|
||||
|
||||
func NewSizeReader(r io.ReadCloser, size int64) SizeReader {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
return &sizeReader{
|
||||
size: size,
|
||||
ReadCloser: r,
|
||||
}
|
||||
}
|
||||
|
||||
type sizeReader struct {
|
||||
size int64
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *sizeReader) Size() int64 {
|
||||
return r.size
|
||||
}
|
||||
|
||||
type Interface interface {
|
||||
// Name 存储名字
|
||||
Name() string
|
||||
// MinFragmentSize 最小允许的分片大小
|
||||
MinFragmentSize() int64
|
||||
// MaxFragmentNum 最大允许的分片数量
|
||||
MaxFragmentNum() int
|
||||
// MinExpirationTime 最小过期时间
|
||||
MinExpirationTime() time.Duration
|
||||
// TempBucket 临时桶名，用于上传
|
||||
TempBucket() string
|
||||
// DataBucket 永久存储的桶名
|
||||
DataBucket() string
|
||||
// PresignedGetURL 通过桶名和对象名返回URL
|
||||
PresignedGetURL(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
name string,
|
||||
expires time.Duration,
|
||||
opt *HeaderOption,
|
||||
) (string, error)
|
||||
// PresignedPutURL 申请上传,返回PUT的上传地址
|
||||
PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
|
||||
// GetObjectInfo 获取对象信息
|
||||
GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
|
||||
// CopyObject 复制对象
|
||||
CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error
|
||||
// DeleteObject 删除对象(不存在返回nil)
|
||||
DeleteObject(ctx context.Context, info *BucketObject) error
|
||||
// ComposeObject 合并对象
|
||||
ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
|
||||
// IsNotFound 判断是不是不存在导致的错误
|
||||
IsNotFound(err error) bool
|
||||
// CheckName 检查名字是否可用
|
||||
CheckName(name string) error
|
||||
// PutObject 上传文件
|
||||
PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error)
|
||||
// GetObject 下载文件
|
||||
GetObject(ctx context.Context, info *BucketObject) (SizeReader, error)
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectHashGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
|
||||
return &ObjectHashGorm{
|
||||
NewMetaDB(db, &relation.ObjectHashModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
|
||||
return &ObjectHashGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectHashModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) Take(
|
||||
ctx context.Context,
|
||||
hash string,
|
||||
engine string,
|
||||
) (oh *relation.ObjectHashModel, err error) {
|
||||
oh = &relation.ObjectHashModel{}
|
||||
return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
|
||||
return utils.Wrap1(o.DB.Create(h).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) DeleteNoCitation(
|
||||
ctx context.Context,
|
||||
engine string,
|
||||
num int,
|
||||
) (list []*relation.ObjectHashModel, err error) {
|
||||
err = o.DB.Table(relation.ObjectHashModelTableName, "as h").Select("h.*").
|
||||
Joins("LEFT JOIN "+relation.ObjectInfoModelTableName+" as i ON h.hash = i.hash").
|
||||
Where("h.engine = ? AND i.hash IS NULL", engine).
|
||||
Limit(num).
|
||||
Find(&list).Error
|
||||
return list, utils.Wrap1(err)
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectInfoGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(db, &relation.ObjectInfoModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectInfoModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
|
||||
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
return errs.Wrap(o.DB.WithContext(ctx).Create(obj).Error)
|
||||
//return errs.Wrap(o.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
|
||||
// return errs.Wrap(err)
|
||||
// }
|
||||
// return errs.Wrap(tx.Create(obj).Error)
|
||||
//}))
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
|
||||
info = &relation.ObjectInfoModel{}
|
||||
return info, utils.Wrap1(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) DeleteExpiration(ctx context.Context, expiration time.Time) (err error) {
|
||||
return utils.Wrap1(
|
||||
o.DB.WithContext(ctx).
|
||||
Where("expiration_time IS NOT NULL AND expiration_time <= ?", expiration).
|
||||
Delete(&relation.ObjectInfoModel{}).
|
||||
Error,
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type ObjectInfoGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(db, &relation.ObjectModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectModel) (err error) {
|
||||
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).FirstOrCreate(obj).Error; err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectModel, err error) {
|
||||
info = &relation.ObjectModel{}
|
||||
return info, errs.Wrap(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectPutGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
|
||||
return &ObjectPutGorm{
|
||||
NewMetaDB(db, &relation.ObjectPutModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
|
||||
return &ObjectPutGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectPutModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
|
||||
return utils.Wrap1(o.DB.Create(m).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
|
||||
put = &relation.ObjectPutModel{}
|
||||
return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
|
||||
return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) FindExpirationPut(
|
||||
ctx context.Context,
|
||||
expirationTime time.Time,
|
||||
num int,
|
||||
) (list []*relation.ObjectPutModel, err error) {
|
||||
err = o.DB.Where("effective_time <= ?", expirationTime).Limit(num).Find(&list).Error
|
||||
return list, utils.Wrap1(err)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) DelPut(ctx context.Context, ids []string) (err error) {
|
||||
return utils.Wrap1(o.DB.Where("put_id IN ?", ids).Delete(&relation.ObjectPutModel{}).Error)
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package cont
|
||||
|
||||
const (
|
||||
hashPath = "openim/data/hash/"
|
||||
tempPath = "openim/temp/"
|
||||
UploadTypeMultipart = 1 // 分片上传
|
||||
UploadTypePresigned = 2 // 预签名上传
|
||||
partSeparator = ","
|
||||
)
|
||||
@ -0,0 +1,244 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/google/uuid"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func New(impl s3.Interface) *Controller {
|
||||
return &Controller{impl: impl}
|
||||
}
|
||||
|
||||
type Controller struct {
|
||||
impl s3.Interface
|
||||
}
|
||||
|
||||
func (c *Controller) HashPath(md5 string) string {
|
||||
return path.Join(hashPath, md5)
|
||||
}
|
||||
|
||||
func (c *Controller) NowPath() string {
|
||||
now := time.Now()
|
||||
return path.Join(
|
||||
fmt.Sprintf("%04d", now.Year()),
|
||||
fmt.Sprintf("%02d", now.Month()),
|
||||
fmt.Sprintf("%02d", now.Day()),
|
||||
fmt.Sprintf("%02d", now.Hour()),
|
||||
fmt.Sprintf("%02d", now.Minute()),
|
||||
fmt.Sprintf("%02d", now.Second()),
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Controller) UUID() string {
|
||||
id := uuid.New()
|
||||
return hex.EncodeToString(id[:])
|
||||
}
|
||||
|
||||
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
return c.impl.PartSize(ctx, size)
|
||||
}
|
||||
|
||||
func (c *Controller) PartLimit() *s3.PartLimit {
|
||||
return c.impl.PartLimit()
|
||||
}
|
||||
|
||||
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
|
||||
return c.impl.StatObject(ctx, c.HashPath(hash))
|
||||
}
|
||||
|
||||
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if size < 0 {
|
||||
return nil, errors.New("invalid size")
|
||||
}
|
||||
if hashBytes, err := hex.DecodeString(hash); err != nil {
|
||||
return nil, err
|
||||
} else if len(hashBytes) != md5.Size {
|
||||
return nil, errors.New("invalid md5")
|
||||
}
|
||||
partSize, err := c.impl.PartSize(ctx, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partNumber := int(size / partSize)
|
||||
if size%partSize > 0 {
|
||||
partNumber++
|
||||
}
|
||||
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
|
||||
return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
|
||||
}
|
||||
if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
|
||||
return nil, &HashAlreadyExistsError{Object: info}
|
||||
} else if !c.impl.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
if size <= partSize {
|
||||
// 预签名上传
|
||||
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
|
||||
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &InitiateUploadResult{
|
||||
UploadID: newMultipartUploadID(multipartUploadID{
|
||||
Type: UploadTypePresigned,
|
||||
ID: "",
|
||||
Key: key,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
}),
|
||||
PartSize: partSize,
|
||||
Sign: &s3.AuthSignResult{
|
||||
Parts: []s3.SignPart{
|
||||
{
|
||||
PartNumber: 1,
|
||||
URL: rawURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
} else {
|
||||
// 分片上传
|
||||
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if maxParts < 0 {
|
||||
maxParts = partNumber
|
||||
}
|
||||
var authSign *s3.AuthSignResult
|
||||
if maxParts > 0 {
|
||||
partNumbers := make([]int, partNumber)
|
||||
for i := 0; i < maxParts; i++ {
|
||||
partNumbers[i] = i + 1
|
||||
}
|
||||
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &InitiateUploadResult{
|
||||
UploadID: newMultipartUploadID(multipartUploadID{
|
||||
Type: UploadTypeMultipart,
|
||||
ID: upload.UploadID,
|
||||
Key: upload.Key,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
}),
|
||||
PartSize: partSize,
|
||||
Sign: authSign,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
upload, err := parseMultipartUploadID(uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
|
||||
fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
|
||||
return nil, errors.New("md5 mismatching")
|
||||
}
|
||||
if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
|
||||
return &UploadResult{
|
||||
Key: info.Key,
|
||||
Size: info.Size,
|
||||
Hash: info.ETag,
|
||||
}, nil
|
||||
} else if !c.impl.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject := make(map[string]struct{})
|
||||
defer func() {
|
||||
for key := range cleanObject {
|
||||
_ = c.impl.DeleteObject(ctx, key)
|
||||
}
|
||||
}()
|
||||
var targetKey string
|
||||
switch upload.Type {
|
||||
case UploadTypeMultipart:
|
||||
parts := make([]s3.Part, len(partHashs))
|
||||
for i, part := range partHashs {
|
||||
parts[i] = s3.Part{
|
||||
PartNumber: i + 1,
|
||||
ETag: part,
|
||||
}
|
||||
}
|
||||
// todo: 验证大小
|
||||
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetKey = result.Key
|
||||
case UploadTypePresigned:
|
||||
uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject[uploadInfo.Key] = struct{}{}
|
||||
if uploadInfo.Size != upload.Size {
|
||||
return nil, errors.New("upload size mismatching")
|
||||
}
|
||||
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
|
||||
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
|
||||
return nil, errs.ErrArgs.Wrap(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
|
||||
}
|
||||
// 防止在这个时候，并发操作，导致文件被覆盖
|
||||
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject[copyInfo.Key] = struct{}{}
|
||||
if copyInfo.ETag != uploadInfo.ETag {
|
||||
return nil, errors.New("[concurrency]copy md5 mismatching")
|
||||
}
|
||||
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
|
||||
targetKey = hashCopyInfo.Key
|
||||
default:
|
||||
return nil, errors.New("invalid upload id type")
|
||||
}
|
||||
return &UploadResult{
|
||||
Key: targetKey,
|
||||
Size: upload.Size,
|
||||
Hash: upload.Hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
upload, err := parseMultipartUploadID(uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch upload.Type {
|
||||
case UploadTypeMultipart:
|
||||
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
|
||||
case UploadTypePresigned:
|
||||
return nil, errors.New("presigned id not support auth sign")
|
||||
default:
|
||||
return nil, errors.New("invalid upload id type")
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) IsNotFound(err error) bool {
|
||||
return c.impl.IsNotFound(err)
|
||||
}
|
||||
|
||||
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
return c.impl.AccessURL(ctx, name, expire, opt)
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
)
|
||||
|
||||
type HashAlreadyExistsError struct {
|
||||
Object *s3.ObjectInfo
|
||||
}
|
||||
|
||||
func (e *HashAlreadyExistsError) Error() string {
|
||||
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type multipartUploadID struct {
|
||||
Type int `json:"a,omitempty"`
|
||||
ID string `json:"b,omitempty"`
|
||||
Key string `json:"c,omitempty"`
|
||||
Size int64 `json:"d,omitempty"`
|
||||
Hash string `json:"e,omitempty"`
|
||||
}
|
||||
|
||||
func newMultipartUploadID(id multipartUploadID) string {
|
||||
data, err := json.Marshal(id)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
|
||||
data, err := base64.StdEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
||||
}
|
||||
var upload multipartUploadID
|
||||
if err := json.Unmarshal(data, &upload); err != nil {
|
||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
||||
}
|
||||
return &upload, nil
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package cont
|
||||
|
||||
import "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
|
||||
type InitiateUploadResult struct {
|
||||
UploadID string `json:"uploadID"` // 上传ID
|
||||
PartSize int64 `json:"partSize"` // 分片大小
|
||||
Sign *s3.AuthSignResult `json:"sign"` // 分片信息
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
Hash string `json:"hash"`
|
||||
Size int64 `json:"size"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
@ -0,0 +1,254 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 1 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 1000
|
||||
)
|
||||
|
||||
func NewCos() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Cos
|
||||
u, err := url.Parse(conf.BucketURL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: conf.SecretID,
|
||||
SecretKey: conf.SecretKey,
|
||||
SessionToken: conf.SessionToken,
|
||||
},
|
||||
})
|
||||
return &Cos{
|
||||
copyURL: u.Host + "/",
|
||||
client: client,
|
||||
credential: client.GetCredential(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Cos struct {
|
||||
copyURL string
|
||||
client *cos.Client
|
||||
credential *cos.Credential
|
||||
}
|
||||
|
||||
func (c *Cos) Engine() string {
|
||||
return "tencent-cos"
|
||||
}
|
||||
|
||||
func (c *Cos) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
UploadID: result.UploadID,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
opts := &cos.CompleteMultipartUploadOptions{
|
||||
Parts: make([]cos.Object, len(parts)),
|
||||
}
|
||||
for i, part := range parts {
|
||||
opts.Parts[i] = cos.Object{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
|
||||
}
|
||||
}
|
||||
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: result.Location,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
ETag: result.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
result := s3.AuthSignResult{
|
||||
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Header: make(http.Header),
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
|
||||
result.Header = req.Header
|
||||
for i, partNumber := range partNumbers {
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
||||
|
||||
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
|
||||
_, err := c.client.Object.Delete(ctx, name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
if name != "" && name[0] == '/' {
|
||||
name = name[1:]
|
||||
}
|
||||
info, err := c.client.Object.Head(ctx, name, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ObjectInfo{Key: name}
|
||||
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
|
||||
return nil, errors.New("StatObject etag not found")
|
||||
}
|
||||
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
|
||||
return nil, errors.New("StatObject content-length not found")
|
||||
} else {
|
||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
|
||||
}
|
||||
if res.Size < 0 {
|
||||
return nil, errors.New("StatObject content-length must be greater than 0")
|
||||
}
|
||||
}
|
||||
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
|
||||
return nil, errors.New("StatObject last-modified not found")
|
||||
} else {
|
||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
sourceURL := c.copyURL + src
|
||||
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *cos.ErrorResponse:
|
||||
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
|
||||
MaxParts: strconv.Itoa(maxParts),
|
||||
PartNumberMarker: strconv.Itoa(partNumberMarker),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
|
||||
}
|
||||
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
|
||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
||||
for i, part := range result.Parts {
|
||||
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: lastModified,
|
||||
ETag: part.ETag,
|
||||
Size: part.Size,
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//reqParams := make(url.Values)
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// reqParams.Set("Content-Type", opt.ContentType)
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
||||
@ -0,0 +1,250 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
unsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 5 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 10000
|
||||
)
|
||||
|
||||
func NewMinio() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Minio
|
||||
u, err := url.Parse(conf.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
}
|
||||
client, err := minio.New(u.Host, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Minio{
|
||||
bucket: conf.Bucket,
|
||||
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
|
||||
opts: opts,
|
||||
core: &minio.Core{Client: client},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Minio struct {
|
||||
bucket string
|
||||
bucketURL string
|
||||
opts *minio.Options
|
||||
core *minio.Core
|
||||
}
|
||||
|
||||
func (m *Minio) Engine() string {
|
||||
return "minio"
|
||||
}
|
||||
|
||||
func (m *Minio) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
Bucket: m.bucket,
|
||||
Key: name,
|
||||
UploadID: uploadID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
minioParts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
minioParts[i] = minio.CompletePart{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ToLower(part.ETag),
|
||||
}
|
||||
}
|
||||
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: upload.Location,
|
||||
Bucket: upload.Bucket,
|
||||
Key: upload.Key,
|
||||
ETag: strings.ToLower(upload.ETag),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
creds, err := m.opts.Creds.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := s3.AuthSignResult{
|
||||
URL: m.bucketURL + name,
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
for i, partNumber := range partNumbers {
|
||||
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
|
||||
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
URL: request.URL.String(),
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
Header: request.Header,
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
||||
|
||||
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
|
||||
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.ObjectInfo{
|
||||
ETag: strings.ToLower(info.ETag),
|
||||
Key: info.Key,
|
||||
Size: info.Size,
|
||||
LastModified: info.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
|
||||
Bucket: m.bucket,
|
||||
Object: dst,
|
||||
}, minio.CopySrcOptions{
|
||||
Bucket: m.bucket,
|
||||
Object: src,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ToLower(result.ETag),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
|
||||
}
|
||||
|
||||
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
MaxParts: result.MaxParts,
|
||||
NextPartNumberMarker: result.NextPartNumberMarker,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
|
||||
}
|
||||
for i, part := range result.ObjectParts {
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: part.LastModified,
|
||||
ETag: part.ETag,
|
||||
Size: part.Size,
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//reqParams := make(url.Values)
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// reqParams.Set("Content-Type", opt.ContentType)
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
@ -0,0 +1,259 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 1 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 10000
|
||||
)
|
||||
|
||||
func NewOSS() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Oss
|
||||
if conf.BucketURL == "" {
|
||||
return nil, errors.New("bucket url is empty")
|
||||
}
|
||||
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, err := client.Bucket(conf.Bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
|
||||
conf.BucketURL += "/"
|
||||
}
|
||||
return &OSS{
|
||||
bucketURL: conf.BucketURL,
|
||||
bucket: bucket,
|
||||
credentials: client.Config.GetCredentials(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type OSS struct {
|
||||
bucketURL string
|
||||
bucket *oss.Bucket
|
||||
credentials oss.Credentials
|
||||
}
|
||||
|
||||
func (o *OSS) Engine() string {
|
||||
return "ali-oss"
|
||||
}
|
||||
|
||||
func (o *OSS) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
result, err := o.bucket.InitiateMultipartUpload(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
UploadID: result.UploadID,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
ossParts := make([]oss.UploadPart, len(parts))
|
||||
for i, part := range parts {
|
||||
ossParts[i] = oss.UploadPart{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ToUpper(part.ETag),
|
||||
}
|
||||
}
|
||||
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Bucket: o.bucket.BucketName,
|
||||
Key: name,
|
||||
}, ossParts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: result.Location,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
result := s3.AuthSignResult{
|
||||
URL: o.bucketURL + name,
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Header: make(http.Header),
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
for i, partNumber := range partNumbers {
|
||||
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o.credentials.GetSecurityToken() != "" {
|
||||
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
|
||||
}
|
||||
request.Header.Set(oss.HTTPHeaderHost, request.Host)
|
||||
request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
|
||||
authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
|
||||
request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
URL: request.URL.String(),
|
||||
Header: request.Header,
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
|
||||
}
|
||||
|
||||
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
header, err := o.bucket.GetObjectMeta(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ObjectInfo{Key: name}
|
||||
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
|
||||
return nil, errors.New("StatObject etag not found")
|
||||
}
|
||||
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
|
||||
return nil, errors.New("StatObject content-length not found")
|
||||
} else {
|
||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
|
||||
}
|
||||
if res.Size < 0 {
|
||||
return nil, errors.New("StatObject content-length must be greater than 0")
|
||||
}
|
||||
}
|
||||
if lastModified := header.Get("Last-Modified"); lastModified == "" {
|
||||
return nil, errors.New("StatObject last-modified not found")
|
||||
} else {
|
||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
|
||||
return o.bucket.DeleteObject(name)
|
||||
}
|
||||
|
||||
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
result, err := o.bucket.CopyObject(src, dst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case oss.ServiceError:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *oss.ServiceError:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Key: name,
|
||||
Bucket: o.bucket.BucketName,
|
||||
})
|
||||
}
|
||||
|
||||
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Key: name,
|
||||
Bucket: o.bucket.BucketName,
|
||||
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
MaxParts: result.MaxParts,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
|
||||
}
|
||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
||||
for i, part := range result.UploadedParts {
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: part.LastModified,
|
||||
ETag: part.ETag,
|
||||
Size: int64(part.Size),
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//var opts []oss.Option
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// opts = append(opts, oss.ContentType(opt.ContentType))
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// opts = append(opts, oss.ContentDisposition(opt.ContentDisposition))
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second))
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
|
||||
var keysList []string
|
||||
keysMap := make(map[string]string)
|
||||
srcKeys := make(map[string]string)
|
||||
|
||||
for k := range req.Header {
|
||||
srcKeys[strings.ToLower(k)] = ""
|
||||
}
|
||||
|
||||
for _, v := range o.bucket.Client.Config.AdditionalHeaders {
|
||||
if _, ok := srcKeys[strings.ToLower(v)]; ok {
|
||||
keysMap[strings.ToLower(v)] = ""
|
||||
}
|
||||
}
|
||||
|
||||
for k := range keysMap {
|
||||
keysList = append(keysList, k)
|
||||
}
|
||||
sort.Strings(keysList)
|
||||
return keysList, keysMap
|
||||
}
|
||||
|
||||
func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
|
||||
// Find out the "x-oss-"'s address in header of the request
|
||||
ossHeadersMap := make(map[string]string)
|
||||
additionalList, additionalMap := o.getAdditionalHeaderKeys(req)
|
||||
for k, v := range req.Header {
|
||||
if strings.HasPrefix(strings.ToLower(k), "x-oss-") {
|
||||
ossHeadersMap[strings.ToLower(k)] = v[0]
|
||||
} else if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
|
||||
if _, ok := additionalMap[strings.ToLower(k)]; ok {
|
||||
ossHeadersMap[strings.ToLower(k)] = v[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
hs := newHeaderSorter(ossHeadersMap)
|
||||
|
||||
// Sort the ossHeadersMap by the ascending order
|
||||
hs.Sort()
|
||||
|
||||
// Get the canonicalizedOSSHeaders
|
||||
canonicalizedOSSHeaders := ""
|
||||
for i := range hs.Keys {
|
||||
canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n"
|
||||
}
|
||||
|
||||
// Give other parameters values
|
||||
// when sign URL, date is expires
|
||||
date := req.Header.Get(oss.HTTPHeaderDate)
|
||||
contentType := req.Header.Get(oss.HTTPHeaderContentType)
|
||||
contentMd5 := req.Header.Get(oss.HTTPHeaderContentMD5)
|
||||
|
||||
// default is v1 signature
|
||||
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
|
||||
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
|
||||
|
||||
// v2 signature
|
||||
if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
|
||||
signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
|
||||
h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
|
||||
}
|
||||
_, _ = io.WriteString(h, signStr)
|
||||
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
|
||||
|
||||
return signedStr
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// headerSorter defines the key-value structure for storing the sorted data in signHeader.
|
||||
type headerSorter struct {
|
||||
Keys []string
|
||||
Vals []string
|
||||
}
|
||||
|
||||
// newHeaderSorter is an additional function for function SignHeader.
|
||||
func newHeaderSorter(m map[string]string) *headerSorter {
|
||||
hs := &headerSorter{
|
||||
Keys: make([]string, 0, len(m)),
|
||||
Vals: make([]string, 0, len(m)),
|
||||
}
|
||||
|
||||
for k, v := range m {
|
||||
hs.Keys = append(hs.Keys, k)
|
||||
hs.Vals = append(hs.Vals, v)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// Sort is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Sort() {
|
||||
sort.Sort(hs)
|
||||
}
|
||||
|
||||
// Len is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Len() int {
|
||||
return len(hs.Vals)
|
||||
}
|
||||
|
||||
// Less is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Less(i, j int) bool {
|
||||
return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0
|
||||
}
|
||||
|
||||
// Swap is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Swap(i, j int) {
|
||||
hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i]
|
||||
hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i]
|
||||
}
|
||||
@ -0,0 +1,134 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PartLimit struct {
|
||||
MinPartSize int64 `json:"minPartSize"`
|
||||
MaxPartSize int64 `json:"maxPartSize"`
|
||||
MaxNumSize int `json:"maxNumSize"`
|
||||
}
|
||||
|
||||
type InitiateMultipartUploadResult struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
UploadID string `json:"uploadID"`
|
||||
}
|
||||
|
||||
type MultipartUploadRequest struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
Query url.Values `json:"query"`
|
||||
Header http.Header `json:"header"`
|
||||
PartKey string `json:"partKey"`
|
||||
PartSize int64 `json:"partSize"`
|
||||
FirstPart int `json:"firstPart"`
|
||||
}
|
||||
|
||||
type Part struct {
|
||||
PartNumber int `json:"partNumber"`
|
||||
ETag string `json:"etag"`
|
||||
}
|
||||
|
||||
type CompleteMultipartUploadResult struct {
|
||||
Location string `json:"location"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
ETag string `json:"etag"`
|
||||
}
|
||||
|
||||
type SignResult struct {
|
||||
Parts []SignPart `json:"parts"`
|
||||
}
|
||||
|
||||
type ObjectInfo struct {
|
||||
ETag string `json:"etag"`
|
||||
Key string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
LastModified time.Time `json:"lastModified"`
|
||||
}
|
||||
|
||||
type CopyObjectInfo struct {
|
||||
Key string `json:"name"`
|
||||
ETag string `json:"etag"`
|
||||
}
|
||||
|
||||
type SignPart struct {
|
||||
PartNumber int `json:"partNumber"`
|
||||
URL string `json:"url"`
|
||||
Query url.Values `json:"query"`
|
||||
Header http.Header `json:"header"`
|
||||
}
|
||||
|
||||
type AuthSignResult struct {
|
||||
URL string `json:"url"`
|
||||
Query url.Values `json:"query"`
|
||||
Header http.Header `json:"header"`
|
||||
Parts []SignPart `json:"parts"`
|
||||
}
|
||||
|
||||
type InitiateUpload struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
Query url.Values `json:"query"`
|
||||
Header http.Header `json:"header"`
|
||||
PartKey string `json:"partKey"`
|
||||
PartSize int64 `json:"partSize"`
|
||||
FirstPart int `json:"firstPart"`
|
||||
}
|
||||
|
||||
type UploadedPart struct {
|
||||
PartNumber int `json:"partNumber"`
|
||||
LastModified time.Time `json:"lastModified"`
|
||||
ETag string `json:"etag"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type ListUploadedPartsResult struct {
|
||||
Key string `xml:"Key"`
|
||||
UploadID string `xml:"UploadId"`
|
||||
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
|
||||
MaxParts int `xml:"MaxParts"`
|
||||
UploadedParts []UploadedPart `xml:"Part"`
|
||||
}
|
||||
|
||||
type AccessURLOption struct {
|
||||
ContentType string `json:"contentType"`
|
||||
ContentDisposition string `json:"contentDisposition"`
|
||||
}
|
||||
|
||||
type Interface interface {
|
||||
Engine() string
|
||||
PartLimit() *PartLimit
|
||||
|
||||
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
|
||||
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
|
||||
|
||||
PartSize(ctx context.Context, size int64) (int64, error)
|
||||
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
|
||||
|
||||
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
|
||||
|
||||
DeleteObject(ctx context.Context, name string) error
|
||||
|
||||
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
|
||||
|
||||
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
|
||||
|
||||
IsNotFound(err error) bool
|
||||
|
||||
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
|
||||
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
|
||||
|
||||
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ObjectInfoModelTableName = "object"
|
||||
)
|
||||
|
||||
type ObjectModel struct {
|
||||
Name string `gorm:"column:name;primary_key"`
|
||||
UserID string `gorm:"column:user_id"`
|
||||
Hash string `gorm:"column:hash"`
|
||||
Key string `gorm:"column:key"`
|
||||
Size int64 `gorm:"column:size"`
|
||||
ContentType string `gorm:"column:content_type"`
|
||||
Cause string `gorm:"column:cause"`
|
||||
CreateTime time.Time `gorm:"column:create_time"`
|
||||
}
|
||||
|
||||
func (ObjectModel) TableName() string {
|
||||
return ObjectInfoModelTableName
|
||||
}
|
||||
|
||||
type ObjectInfoModelInterface interface {
|
||||
NewTx(tx any) ObjectInfoModelInterface
|
||||
SetObject(ctx context.Context, obj *ObjectModel) error
|
||||
Take(ctx context.Context, name string) (*ObjectModel, error)
|
||||
}
|
||||
@ -1,44 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ObjectHashModelTableName = "object_hash"
|
||||
)
|
||||
|
||||
type ObjectHashModel struct {
|
||||
Hash string `gorm:"column:hash;primary_key;size:32"`
|
||||
Engine string `gorm:"column:engine;primary_key;size:16"`
|
||||
Size int64 `gorm:"column:size"`
|
||||
Bucket string `gorm:"column:bucket"`
|
||||
Name string `gorm:"column:name"`
|
||||
CreateTime time.Time `gorm:"column:create_time"`
|
||||
}
|
||||
|
||||
func (ObjectHashModel) TableName() string {
|
||||
return ObjectHashModelTableName
|
||||
}
|
||||
|
||||
type ObjectHashModelInterface interface {
|
||||
NewTx(tx any) ObjectHashModelInterface
|
||||
Take(ctx context.Context, hash string, engine string) (*ObjectHashModel, error)
|
||||
Create(ctx context.Context, h []*ObjectHashModel) error
|
||||
DeleteNoCitation(ctx context.Context, engine string, num int) (list []*ObjectHashModel, err error)
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ObjectInfoModelTableName = "object_info"
|
||||
)
|
||||
|
||||
type ObjectInfoModel struct {
|
||||
Name string `gorm:"column:name;primary_key"`
|
||||
Hash string `gorm:"column:hash"`
|
||||
ContentType string `gorm:"column:content_type"`
|
||||
ValidTime *time.Time `gorm:"column:valid_time"`
|
||||
CreateTime time.Time `gorm:"column:create_time"`
|
||||
}
|
||||
|
||||
func (ObjectInfoModel) TableName() string {
|
||||
return ObjectInfoModelTableName
|
||||
}
|
||||
|
||||
type ObjectInfoModelInterface interface {
|
||||
NewTx(tx any) ObjectInfoModelInterface
|
||||
SetObject(ctx context.Context, obj *ObjectInfoModel) error
|
||||
Take(ctx context.Context, name string) (*ObjectInfoModel, error)
|
||||
DeleteExpiration(ctx context.Context, expiration time.Time) error
|
||||
}
|
||||
@ -1,51 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ObjectPutModelTableName = "object_put"
|
||||
)
|
||||
|
||||
type ObjectPutModel struct {
|
||||
PutID string `gorm:"column:put_id;primary_key"`
|
||||
Hash string `gorm:"column:hash"`
|
||||
Path string `gorm:"column:path"`
|
||||
Name string `gorm:"column:name"`
|
||||
ContentType string `gorm:"column:content_type"`
|
||||
ObjectSize int64 `gorm:"column:object_size"`
|
||||
FragmentSize int64 `gorm:"column:fragment_size"`
|
||||
PutURLsHash string `gorm:"column:put_urls_hash"`
|
||||
ValidTime *time.Time `gorm:"column:valid_time"`
|
||||
EffectiveTime time.Time `gorm:"column:effective_time"`
|
||||
CreateTime time.Time `gorm:"column:create_time"`
|
||||
}
|
||||
|
||||
func (ObjectPutModel) TableName() string {
|
||||
return ObjectPutModelTableName
|
||||
}
|
||||
|
||||
type ObjectPutModelInterface interface {
|
||||
NewTx(tx any) ObjectPutModelInterface
|
||||
Create(ctx context.Context, m []*ObjectPutModel) error
|
||||
Take(ctx context.Context, putID string) (*ObjectPutModel, error)
|
||||
SetCompleted(ctx context.Context, putID string) error
|
||||
FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) ([]*ObjectPutModel, error)
|
||||
DelPut(ctx context.Context, ids []string) error
|
||||
}
|
||||
@ -1,125 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package unrelation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
)
|
||||
|
||||
const (
|
||||
CExtendMsgSet = "extend_msgs"
|
||||
|
||||
ExtendMsgMaxNum = 100
|
||||
)
|
||||
|
||||
type ExtendMsgSetModel struct {
|
||||
ConversationID string `bson:"source_id" json:"conversationID"`
|
||||
SessionType int32 `bson:"session_type" json:"sessionType"`
|
||||
ExtendMsgs map[string]ExtendMsgModel `bson:"extend_msgs" json:"extendMsgs"`
|
||||
ExtendMsgNum int32 `bson:"extend_msg_num" json:"extendMsgNum"`
|
||||
CreateTime int64 `bson:"create_time" json:"createTime"` // this block's create time
|
||||
MaxMsgUpdateTime int64 `bson:"max_msg_update_time" json:"maxMsgUpdateTime"` // index find msg
|
||||
}
|
||||
|
||||
type KeyValueModel struct {
|
||||
TypeKey string `bson:"type_key" json:"typeKey"`
|
||||
Value string `bson:"value" json:"value"`
|
||||
LatestUpdateTime int64 `bson:"latest_update_time" json:"latestUpdateTime"`
|
||||
}
|
||||
|
||||
type ExtendMsgModel struct {
|
||||
ReactionExtensionList map[string]KeyValueModel `bson:"reaction_extension_list" json:"reactionExtensionList"`
|
||||
ClientMsgID string `bson:"client_msg_id" json:"clientMsgID"`
|
||||
MsgFirstModifyTime int64 `bson:"msg_first_modify_time" json:"msgFirstModifyTime"` // this extendMsg create time
|
||||
AttachedInfo string `bson:"attached_info" json:"attachedInfo"`
|
||||
Ex string `bson:"ex" json:"ex"`
|
||||
}
|
||||
|
||||
type ExtendMsgSetModelInterface interface {
|
||||
CreateExtendMsgSet(ctx context.Context, set *ExtendMsgSetModel) error
|
||||
GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
opts *GetAllExtendMsgSetOpts,
|
||||
) (sets []*ExtendMsgSetModel, err error)
|
||||
GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*ExtendMsgSetModel, error)
|
||||
InsertExtendMsg(ctx context.Context, conversationID string, sessionType int32, msg *ExtendMsgModel) error
|
||||
InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*KeyValueModel,
|
||||
) error
|
||||
DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*KeyValueModel,
|
||||
) error
|
||||
TakeExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *ExtendMsgModel, err error)
|
||||
}
|
||||
|
||||
func (ExtendMsgSetModel) TableName() string {
|
||||
return CExtendMsgSet
|
||||
}
|
||||
|
||||
func (ExtendMsgSetModel) GetExtendMsgMaxNum() int32 {
|
||||
return ExtendMsgMaxNum
|
||||
}
|
||||
|
||||
func (ExtendMsgSetModel) GetConversationID(ID string, index int32) string {
|
||||
return ID + ":" + strconv.Itoa(int(index))
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetModel) SplitConversationIDAndGetIndex() int32 {
|
||||
l := strings.Split(e.ConversationID, ":")
|
||||
index, _ := strconv.Atoi(l[len(l)-1])
|
||||
return int32(index)
|
||||
}
|
||||
|
||||
type GetAllExtendMsgSetOpts struct {
|
||||
ExcludeExtendMsgs bool
|
||||
}
|
||||
|
||||
func (ExtendMsgSetModel) Pb2Model(reactionExtensionList map[string]*sdkws.KeyValue) map[string]*KeyValueModel {
|
||||
r := make(map[string]*KeyValueModel)
|
||||
for key, value := range reactionExtensionList {
|
||||
r[key] = &KeyValueModel{
|
||||
TypeKey: value.TypeKey,
|
||||
Value: value.Value,
|
||||
LatestUpdateTime: value.LatestUpdateTime,
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
@ -1,230 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package unrelation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
|
||||
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ExtendMsgSetMongoDriver struct {
|
||||
mgoDB *mongo.Database
|
||||
ExtendMsgSetCollection *mongo.Collection
|
||||
}
|
||||
|
||||
func NewExtendMsgSetMongoDriver(mgoDB *mongo.Database) unRelationTb.ExtendMsgSetModelInterface {
|
||||
return &ExtendMsgSetMongoDriver{mgoDB: mgoDB, ExtendMsgSetCollection: mgoDB.Collection(unRelationTb.CExtendMsgSet)}
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetMongoDriver) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
|
||||
_, err := e.ExtendMsgSetCollection.InsertOne(ctx, set)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetMongoDriver) GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
ID string,
|
||||
opts *unRelationTb.GetAllExtendMsgSetOpts,
|
||||
) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
|
||||
regex := fmt.Sprintf("^%s", ID)
|
||||
var findOpts *options.FindOptions
|
||||
if opts != nil {
|
||||
if opts.ExcludeExtendMsgs {
|
||||
findOpts = &options.FindOptions{}
|
||||
findOpts.SetProjection(bson.M{"extend_msgs": 0})
|
||||
}
|
||||
}
|
||||
cursor, err := e.ExtendMsgSetCollection.Find(ctx, bson.M{"doc_id": primitive.Regex{Pattern: regex}}, findOpts)
|
||||
if err != nil {
|
||||
return nil, utils.Wrap(err, "")
|
||||
}
|
||||
err = cursor.All(ctx, &sets)
|
||||
if err != nil {
|
||||
return nil, utils.Wrap(err, fmt.Sprintf("cursor is %s", cursor.Current.String()))
|
||||
}
|
||||
return sets, nil
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetMongoDriver) GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*unRelationTb.ExtendMsgSetModel, error) {
|
||||
var err error
|
||||
findOpts := options.Find().
|
||||
SetLimit(1).
|
||||
SetSkip(0).
|
||||
SetSort(bson.M{"source_id": -1}).
|
||||
SetProjection(bson.M{"extend_msgs": 0})
|
||||
// update newest
|
||||
find := bson.M{
|
||||
"source_id": primitive.Regex{Pattern: fmt.Sprintf("^%s", conversationID)},
|
||||
"session_type": sessionType,
|
||||
}
|
||||
if maxMsgUpdateTime > 0 {
|
||||
find["max_msg_update_time"] = maxMsgUpdateTime
|
||||
}
|
||||
result, err := e.ExtendMsgSetCollection.Find(ctx, find, findOpts)
|
||||
if err != nil {
|
||||
return nil, utils.Wrap(err, "")
|
||||
}
|
||||
var setList []unRelationTb.ExtendMsgSetModel
|
||||
if err := result.All(ctx, &setList); err != nil {
|
||||
return nil, utils.Wrap(err, "")
|
||||
}
|
||||
if len(setList) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &setList[0], nil
|
||||
}
|
||||
|
||||
// first modify msg
|
||||
func (e *ExtendMsgSetMongoDriver) InsertExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
msg *unRelationTb.ExtendMsgModel,
|
||||
) error {
|
||||
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, 0)
|
||||
if err != nil {
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
if set == nil || set.ExtendMsgNum >= set.GetExtendMsgMaxNum() {
|
||||
var index int32
|
||||
if set != nil {
|
||||
index = set.SplitConversationIDAndGetIndex()
|
||||
}
|
||||
err = e.CreateExtendMsgSet(ctx, &unRelationTb.ExtendMsgSetModel{
|
||||
ConversationID: set.GetConversationID(conversationID, index),
|
||||
SessionType: sessionType,
|
||||
ExtendMsgs: map[string]unRelationTb.ExtendMsgModel{msg.ClientMsgID: *msg},
|
||||
ExtendMsgNum: 1,
|
||||
CreateTime: msg.MsgFirstModifyTime,
|
||||
MaxMsgUpdateTime: msg.MsgFirstModifyTime,
|
||||
})
|
||||
} else {
|
||||
_, err = e.ExtendMsgSetCollection.UpdateOne(ctx, bson.M{"conversation_id": set.ConversationID, "session_type": sessionType}, bson.M{"$set": bson.M{"max_msg_update_time": msg.MsgFirstModifyTime, "$inc": bson.M{"extend_msg_num": 1}, fmt.Sprintf("extend_msgs.%s", msg.ClientMsgID): msg}})
|
||||
}
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
|
||||
// insert or update
|
||||
func (e *ExtendMsgSetMongoDriver) InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
var updateBson = bson.M{}
|
||||
for _, v := range reactionExtensionList {
|
||||
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = v
|
||||
}
|
||||
upsert := true
|
||||
opt := &options.UpdateOptions{
|
||||
Upsert: &upsert,
|
||||
}
|
||||
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
|
||||
if err != nil {
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
if set == nil {
|
||||
return fmt.Errorf("conversationID %s has no set", conversationID)
|
||||
}
|
||||
_, err = e.ExtendMsgSetCollection.UpdateOne(
|
||||
ctx,
|
||||
bson.M{"source_id": set.ConversationID, "session_type": sessionType},
|
||||
bson.M{"$set": updateBson},
|
||||
opt,
|
||||
)
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
|
||||
// delete TypeKey
|
||||
func (e *ExtendMsgSetMongoDriver) DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
var updateBson = bson.M{}
|
||||
for _, v := range reactionExtensionList {
|
||||
updateBson[fmt.Sprintf("extend_msgs.%s.%s", clientMsgID, v.TypeKey)] = ""
|
||||
}
|
||||
set, err := e.GetExtendMsgSet(ctx, conversationID, sessionType, msgFirstModifyTime)
|
||||
if err != nil {
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
if set == nil {
|
||||
return fmt.Errorf("conversationID %s has no set", conversationID)
|
||||
}
|
||||
_, err = e.ExtendMsgSetCollection.UpdateOne(
|
||||
ctx,
|
||||
bson.M{"source_id": set.ConversationID, "session_type": sessionType},
|
||||
bson.M{"$unset": updateBson},
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetMongoDriver) TakeExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
|
||||
findOpts := options.Find().
|
||||
SetLimit(1).
|
||||
SetSkip(0).
|
||||
SetSort(bson.M{"source_id": -1}).
|
||||
SetProjection(bson.M{fmt.Sprintf("extend_msgs.%s", clientMsgID): 1})
|
||||
regex := fmt.Sprintf("^%s", conversationID)
|
||||
result, err := e.ExtendMsgSetCollection.Find(
|
||||
ctx,
|
||||
bson.M{
|
||||
"source_id": primitive.Regex{Pattern: regex},
|
||||
"session_type": sessionType,
|
||||
"max_msg_update_time": bson.M{"$lte": maxMsgUpdateTime},
|
||||
},
|
||||
findOpts,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, utils.Wrap(err, "")
|
||||
}
|
||||
var setList []unRelationTb.ExtendMsgSetModel
|
||||
if err := result.All(ctx, &setList); err != nil {
|
||||
return nil, utils.Wrap(err, "")
|
||||
}
|
||||
if len(setList) == 0 {
|
||||
return nil, utils.Wrap(errors.New("GetExtendMsg failed, len(setList) == 0"), "")
|
||||
}
|
||||
if v, ok := setList[0].ExtendMsgs[clientMsgID]; ok {
|
||||
return &v, nil
|
||||
}
|
||||
return nil, fmt.Errorf("cant find client msg id: %s", clientMsgID)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue