# Conflicts: # pkg/common/db/unrelation/extend_msg.gopull/518/head
@ -0,0 +1,69 @@
|
||||
name: OpenIM Start Execute Scripts
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "README_zh-CN.md"
|
||||
- "CONTRIBUTING.md"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- "README.md"
|
||||
- "README_zh-CN.md"
|
||||
- "CONTRIBUTING.md"
|
||||
- "docs/**"
|
||||
|
||||
jobs:
|
||||
execute-scripts:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Start Docker Compose
|
||||
run: |
|
||||
docker-compose stop
|
||||
docker-compose up -d
|
||||
sleep 60
|
||||
|
||||
- name: Stop all services
|
||||
run: |
|
||||
chmod +x ./scripts/stop_all.sh
|
||||
./scripts/stop_all.sh
|
||||
shell: bash
|
||||
|
||||
- name: Build all services
|
||||
run: |
|
||||
chmod +x ./scripts/build_all_service.sh
|
||||
./scripts/build_all_service.sh
|
||||
cat logs/openIM.log
|
||||
shell: bash
|
||||
|
||||
- name: Start all services
|
||||
run: |
|
||||
chmod +x ./scripts/start_all.sh
|
||||
./scripts/start_all.sh
|
||||
cat logs/openIM.log
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
|
||||
- name: Check all services
|
||||
run: |
|
||||
chmod +x ./scripts/check_all.sh
|
||||
./scripts/check_all.sh
|
||||
cat logs/openIM.log
|
||||
shell: bash
|
||||
continue-on-error: true
|
||||
|
||||
- name: Print openIM.log
|
||||
run: |
|
||||
cat logs/openIM.log
|
||||
cat logs/openIM.log >> "$GITHUB_OUTPUT"
|
||||
shell: bash
|
||||
continue-on-error: true
|
@ -0,0 +1,32 @@
|
||||
# `/assets`
|
||||
|
||||
The `/assets` directory in the OpenIM repository contains various assets such as images, logos, and animated GIFs. These assets serve different purposes and contribute to the functionality and aesthetics of the OpenIM project.
|
||||
|
||||
## Directory Structure:
|
||||
|
||||
```bash
|
||||
assets/
|
||||
├── README.md # Documentation for the assets directory
|
||||
├── images # Directory holding images related to OpenIM
|
||||
│ ├── architecture.png # Image depicting the architecture of OpenIM
|
||||
│ └── mvc.png # Image illustrating the Model-View-Controller (MVC) pattern
|
||||
├── intive-slack.png # Image displaying the Intive Slack logo
|
||||
├── logo # Directory containing various logo variations for OpenIM
|
||||
│ ├── openim-logo-black.png # OpenIM logo with a black background
|
||||
│ ├── openim-logo-blue.png # OpenIM logo with a blue background
|
||||
│ ├── openim-logo-green.png # OpenIM logo with a green background
|
||||
│ ├── openim-logo-purple.png # OpenIM logo with a purple background
|
||||
│ ├── openim-logo-white.png # OpenIM logo with a white background
|
||||
│ ├── openim-logo-yellow.png # OpenIM logo with a yellow background
|
||||
│ └── openim-logo.png # OpenIM logo with a transparent background
|
||||
└── logo-gif # Directory containing animated GIF versions of the OpenIM logo
|
||||
└── openim-log.gif # Animated OpenIM logo with a transparent background
|
||||
```
|
||||
|
||||
## Copyright Notice:
|
||||
|
||||
The OpenIM logo, including its variations and animated versions, displayed in this repository [OpenIM](https://github.com/OpenIMSDK/openim) under the `/assets/logo` and `/assets/logo-gif` directories, are protected by copyright laws.
|
||||
|
||||
The logo design is credited to @Xx(席欣).
|
||||
|
||||
Please respect the intellectual property rights and refrain from unauthorized use and distribution of these assets.
|
After Width: | Height: | Size: 21 KiB |
@ -0,0 +1 @@
|
||||
# The OpenIM logo files are licensed under a choice of either Apache-2.0 or CC-BY-4.0 (Creative Commons Attribution 4.0 International).
|
After Width: | Height: | Size: 1.6 MiB |
@ -0,0 +1 @@
|
||||
# The OpenIM logo files are licensed under a choice of either Apache-2.0 or CC-BY-4.0 (Creative Commons Attribution 4.0 International).
|
After Width: | Height: | Size: 32 KiB |
After Width: | Height: | Size: 35 KiB |
After Width: | Height: | Size: 38 KiB |
After Width: | Height: | Size: 32 KiB |
After Width: | Height: | Size: 36 KiB |
After Width: | Height: | Size: 32 KiB |
After Width: | Height: | Size: 31 KiB |
After Width: | Height: | Size: 36 KiB |
After Width: | Height: | Size: 37 KiB |
After Width: | Height: | Size: 58 KiB |
After Width: | Height: | Size: 58 KiB |
@ -1,27 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
internet_ip=`curl ifconfig.me -s`
|
||||
|
||||
# Get the public internet IP address
|
||||
internet_ip=$(curl ifconfig.me -s)
|
||||
echo $internet_ip
|
||||
|
||||
# Load environment variables from .env file
|
||||
source .env
|
||||
echo $MINIO_ENDPOINT
|
||||
|
||||
# Replace local IP address with the public IP address in .env file
|
||||
if [ $MINIO_ENDPOINT == "http://127.0.0.1:10005" ]; then
|
||||
sed -i "s/127.0.0.1/${internet_ip}/" .env
|
||||
|
||||
fi
|
||||
|
||||
cd scripts ;
|
||||
chmod +x *.sh ;
|
||||
# Change directory to scripts folder
|
||||
cd scripts
|
||||
chmod +x *.sh
|
||||
|
||||
# Execute necessary scripts
|
||||
./init_pwd.sh
|
||||
./env_check.sh;
|
||||
cd .. ;
|
||||
./env_check.sh
|
||||
|
||||
# Go back to the previous directory
|
||||
cd ..
|
||||
|
||||
# Check if docker-compose command is available
|
||||
if command -v docker-compose &> /dev/null
|
||||
then
|
||||
docker-compose up -d ;
|
||||
docker-compose up -d
|
||||
else
|
||||
docker compose up -d ;
|
||||
docker compose up -d
|
||||
fi
|
||||
|
||||
# Change directory to scripts folder again
|
||||
cd scripts
|
||||
|
||||
cd scripts ;
|
||||
# Check docker services
|
||||
./docker_check_service.sh
|
@ -1,145 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msgtransfer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
|
||||
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
kfk "github.com/OpenIMSDK/Open-IM-Server/pkg/common/kafka"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
pbMsg "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type ModifyMsgConsumerHandler struct {
|
||||
modifyMsgConsumerGroup *kfk.MConsumerGroup
|
||||
|
||||
extendMsgDatabase controller.ExtendMsgDatabase
|
||||
extendSetMsgModel unRelationTb.ExtendMsgSetModel
|
||||
}
|
||||
|
||||
func NewModifyMsgConsumerHandler(database controller.ExtendMsgDatabase) *ModifyMsgConsumerHandler {
|
||||
return &ModifyMsgConsumerHandler{
|
||||
modifyMsgConsumerGroup: kfk.NewMConsumerGroup(&kfk.MConsumerGroupConfig{KafkaVersion: sarama.V2_0_0_0,
|
||||
OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false}, []string{config.Config.Kafka.MsgToModify.Topic},
|
||||
config.Config.Kafka.Addr, config.Config.Kafka.ConsumerGroupID.MsgToModify),
|
||||
extendMsgDatabase: database,
|
||||
}
|
||||
}
|
||||
|
||||
func (ModifyMsgConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (ModifyMsgConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil }
|
||||
func (mmc *ModifyMsgConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
|
||||
claim sarama.ConsumerGroupClaim) error {
|
||||
for msg := range claim.Messages() {
|
||||
ctx := mmc.modifyMsgConsumerGroup.GetContextFromMsg(msg)
|
||||
log.ZDebug(
|
||||
ctx,
|
||||
"kafka get info to mysql",
|
||||
"ModifyMsgConsumerHandler",
|
||||
msg.Topic,
|
||||
"msgPartition",
|
||||
msg.Partition,
|
||||
"msg",
|
||||
string(msg.Value),
|
||||
"key",
|
||||
string(msg.Key),
|
||||
)
|
||||
if len(msg.Value) != 0 {
|
||||
mmc.ModifyMsg(ctx, msg, string(msg.Key), sess)
|
||||
} else {
|
||||
log.ZError(ctx, "msg get from kafka but is nil", nil, "key", msg.Key)
|
||||
}
|
||||
sess.MarkMessage(msg, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mmc *ModifyMsgConsumerHandler) ModifyMsg(
|
||||
ctx context.Context,
|
||||
cMsg *sarama.ConsumerMessage,
|
||||
msgKey string,
|
||||
_ sarama.ConsumerGroupSession,
|
||||
) {
|
||||
msgFromMQ := pbMsg.MsgDataToModifyByMQ{}
|
||||
operationID := mcontext.GetOperationID(ctx)
|
||||
err := proto.Unmarshal(cMsg.Value, &msgFromMQ)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "msg_transfer Unmarshal msg err", err, "msg", string(cMsg.Value))
|
||||
return
|
||||
}
|
||||
log.ZDebug(ctx, "proto.Unmarshal MsgDataToMQ", "msgs", msgFromMQ.String())
|
||||
for _, msg := range msgFromMQ.Messages {
|
||||
isReactionFromCache := utils.GetSwitchFromOptions(msg.Options, constant.IsReactionFromCache)
|
||||
if !isReactionFromCache {
|
||||
continue
|
||||
}
|
||||
ctx = mcontext.SetOperationID(ctx, operationID)
|
||||
if msg.ContentType == constant.ReactionMessageModifier {
|
||||
notification := &sdkws.ReactionMessageModifierNotification{}
|
||||
if err := json.Unmarshal(msg.Content, notification); err != nil {
|
||||
continue
|
||||
}
|
||||
if notification.IsExternalExtensions {
|
||||
continue
|
||||
}
|
||||
if !notification.IsReact {
|
||||
// first time to modify
|
||||
var reactionExtensionList = make(map[string]unRelationTb.KeyValueModel)
|
||||
extendMsg := unRelationTb.ExtendMsgModel{
|
||||
ReactionExtensionList: reactionExtensionList,
|
||||
ClientMsgID: notification.ClientMsgID,
|
||||
MsgFirstModifyTime: notification.MsgFirstModifyTime,
|
||||
}
|
||||
for _, v := range notification.SuccessReactionExtensions {
|
||||
reactionExtensionList[v.TypeKey] = unRelationTb.KeyValueModel{
|
||||
TypeKey: v.TypeKey,
|
||||
Value: v.Value,
|
||||
LatestUpdateTime: v.LatestUpdateTime,
|
||||
}
|
||||
}
|
||||
|
||||
if err := mmc.extendMsgDatabase.InsertExtendMsg(ctx, notification.ConversationID, notification.SessionType, &extendMsg); err != nil {
|
||||
// log.ZError(ctx, "MsgFirstModify InsertExtendMsg failed", notification.ConversationID,
|
||||
// notification.SessionType, extendMsg, err.Error())
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if err := mmc.extendMsgDatabase.InsertOrUpdateReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
|
||||
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
|
||||
}
|
||||
}
|
||||
} else if msg.ContentType == constant.ReactionMessageDeleter {
|
||||
notification := &sdkws.ReactionMessageDeleteNotification{}
|
||||
if err := json.Unmarshal(msg.Content, notification); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := mmc.extendMsgDatabase.DeleteReactionExtendMsgSet(ctx, notification.ConversationID, notification.SessionType, notification.ClientMsgID, notification.MsgFirstModifyTime, mmc.extendSetMsgModel.Pb2Model(notification.SuccessReactionExtensions)); err != nil {
|
||||
// log.NewError(operationID, "InsertOrUpdateReactionExtendMsgSet failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,28 @@
|
||||
package group
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/group"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCreateCountReq) (*group.GroupCreateCountResp, error) {
|
||||
if req.Start > req.End {
|
||||
return nil, errs.ErrArgs.Wrap("start > end")
|
||||
}
|
||||
total, err := s.GroupDatabase.CountTotal(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start := time.UnixMilli(req.Start)
|
||||
before, err := s.GroupDatabase.CountTotal(ctx, &start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count, err := s.GroupDatabase.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &group.GroupCreateCountResp{Total: total, Before: before, Count: count}, nil
|
||||
}
|
@ -1,414 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
)
|
||||
|
||||
func (m *msgServer) SetMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.SetMessageReactionExtensionsReq,
|
||||
) (resp *msg.SetMessageReactionExtensionsResp, err error) {
|
||||
//resp = &msg.SetMessageReactionExtensionsResp{}
|
||||
////resp.ClientMsgID = req.ClientMsgID
|
||||
////resp.MsgFirstModifyTime = req.MsgFirstModifyTime
|
||||
//
|
||||
//if err := CallbackSetMessageReactionExtensions(ctx, req); err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
////if ExternalExtension
|
||||
//if req.IsExternalExtensions {
|
||||
// resp.MsgFirstModifyTime = req.MsgFirstModifyTime
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, !req.IsReact, false)
|
||||
// return resp, nil
|
||||
//}
|
||||
//isExists, err := m.MsgDatabase.JudgeMessageReactionExist(ctx, req.ClientMsgID, req.SessionType)
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
//
|
||||
//if !isExists {
|
||||
// if !req.IsReact {
|
||||
// resp.MsgFirstModifyTime = utils.GetCurrentTimestampByMill()
|
||||
// for k, v := range req.ReactionExtensions {
|
||||
// err := m.MessageLocker.LockMessageTypeKey(ctx, req.ClientMsgID, k)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// if err := m.MsgDatabase.SetMessageTypeKeyValue(ctx, req.ClientMsgID, req.SessionType, k,
|
||||
// utils.StructToJsonString(v)); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// }
|
||||
// resp.IsReact = true
|
||||
// _, err := m.MsgDatabase.SetMessageReactionExpire(ctx, req.ClientMsgID, req.SessionType,
|
||||
// time.Duration(24*3)*time.Hour)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// } else {
|
||||
// err := m.MessageLocker.LockGlobalMessage(ctx, req.ClientMsgID)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// mongoValue, err := m.MsgDatabase.GetExtendMsg(ctx, req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// setValue := make(map[string]*sdkws.KeyValue)
|
||||
// for k, v := range req.ReactionExtensions {
|
||||
//
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// if vv, ok := mongoValue.ReactionExtensions[k]; ok {
|
||||
// utils.CopyStructFields(temp, &vv)
|
||||
// if v.LatestUpdateTime != vv.LatestUpdateTime {
|
||||
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// }
|
||||
// }
|
||||
// temp.TypeKey = k
|
||||
// temp.Value = v.Value
|
||||
// temp.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// setValue[k] = temp
|
||||
// }
|
||||
// err = db.DB.InsertOrUpdateReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime, setValue)
|
||||
// if err != nil {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// resp.Result = append(resp.Result, temp)
|
||||
// }
|
||||
// } else {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// resp.Result = append(resp.Result, temp)
|
||||
// }
|
||||
// }
|
||||
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
|
||||
// if lockErr != nil {
|
||||
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//} else {
|
||||
// log.Debug(req.OperationID, "redis handle secondly", req.String())
|
||||
//
|
||||
// for k, v := range req.Pb2Model {
|
||||
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, k)
|
||||
// if err != nil {
|
||||
// setKeyResultInfo(&resp, 100, err.Error(), req.ClientMsgID, k, v)
|
||||
// continue
|
||||
// }
|
||||
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k)
|
||||
// if err != nil && err != go_redis.Nil {
|
||||
// setKeyResultInfo(&resp, 200, err.Error(), req.ClientMsgID, k, v)
|
||||
// continue
|
||||
// }
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(redisValue, temp)
|
||||
// if v.LatestUpdateTime != temp.LatestUpdateTime {
|
||||
// setKeyResultInfo(&resp, 300, "message have update", req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// } else {
|
||||
// v.LatestUpdateTime = utils.GetCurrentTimestampByMill()
|
||||
// newerr := db.DB.SetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, k, utils.StructToJsonString(v))
|
||||
// if newerr != nil {
|
||||
// setKeyResultInfo(&resp, 201, newerr.Error(), req.ClientMsgID, k, temp)
|
||||
// continue
|
||||
// }
|
||||
// setKeyResultInfo(&resp, 0, "", req.ClientMsgID, k, v)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//}
|
||||
//if !isExists {
|
||||
// if !req.IsReact {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, true, true)
|
||||
// } else {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, false, false)
|
||||
// }
|
||||
//} else {
|
||||
// notification.ExtendMessageUpdatedNotification(req.OperationID, req.OpUserID, req.conversationID,
|
||||
// req.SessionType, req, &resp, false, true)
|
||||
//}
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", resp.String())
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func (m *msgServer) setKeyResultInfo(
|
||||
ctx context.Context,
|
||||
r *msg.SetMessageReactionExtensionsResp,
|
||||
errCode int32,
|
||||
errMsg, clientMsgID, typeKey string,
|
||||
keyValue *sdkws.KeyValue,
|
||||
) {
|
||||
temp := new(msg.KeyValueResp)
|
||||
temp.KeyValue = keyValue
|
||||
temp.ErrCode = errCode
|
||||
temp.ErrMsg = errMsg
|
||||
r.Result = append(r.Result, temp)
|
||||
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
|
||||
}
|
||||
|
||||
func (m *msgServer) setDeleteKeyResultInfo(
|
||||
ctx context.Context,
|
||||
r *msg.DeleteMessagesReactionExtensionsResp,
|
||||
errCode int32,
|
||||
errMsg, clientMsgID, typeKey string,
|
||||
keyValue *sdkws.KeyValue,
|
||||
) {
|
||||
temp := new(msg.KeyValueResp)
|
||||
temp.KeyValue = keyValue
|
||||
temp.ErrCode = errCode
|
||||
temp.ErrMsg = errMsg
|
||||
r.Result = append(r.Result, temp)
|
||||
_ = m.MessageLocker.UnLockMessageTypeKey(ctx, clientMsgID, typeKey)
|
||||
}
|
||||
|
||||
func (m *msgServer) GetMessagesReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.GetMessagesReactionExtensionsReq,
|
||||
) (resp *msg.GetMessagesReactionExtensionsResp, err error) {
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
|
||||
//var rResp msg.GetMessageListReactionExtensionsResp
|
||||
//for _, messageValue := range req.MessageReactionKeyList {
|
||||
// var oneMessage msg.SingleMessageExtensionResult
|
||||
// oneMessage.ClientMsgID = messageValue.ClientMsgID
|
||||
//
|
||||
// isExists, err := db.DB.JudgeMessageReactionExist(messageValue.ClientMsgID, req.SessionType)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// if isExists {
|
||||
// redisValue, err := db.DB.GetOneMessageAllReactionList(messageValue.ClientMsgID, req.SessionType)
|
||||
// if err != nil {
|
||||
// oneMessage.ErrCode = 100
|
||||
// oneMessage.ErrMsg = err.Error()
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
// continue
|
||||
// }
|
||||
// keyMap := make(map[string]*sdkws.KeyValue)
|
||||
//
|
||||
// for k, v := range redisValue {
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(v, temp)
|
||||
// keyMap[k] = temp
|
||||
// }
|
||||
// oneMessage.Pb2Model = keyMap
|
||||
//
|
||||
// } else {
|
||||
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, messageValue.ClientMsgID,
|
||||
// messageValue.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// oneMessage.ErrCode = 100
|
||||
// oneMessage.ErrMsg = err.Error()
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
// continue
|
||||
// }
|
||||
// keyMap := make(map[string]*sdkws.KeyValue)
|
||||
//
|
||||
// for k, v := range mongoValue.Pb2Model {
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// temp.TypeKey = v.TypeKey
|
||||
// temp.Value = v.Value
|
||||
// temp.LatestUpdateTime = v.LatestUpdateTime
|
||||
// keyMap[k] = temp
|
||||
// }
|
||||
// oneMessage.Pb2Model = keyMap
|
||||
// }
|
||||
// rResp.SingleMessageResult = append(rResp.SingleMessageResult, &oneMessage)
|
||||
//}
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
func (m *msgServer) AddMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.ModifyMessageReactionExtensionsReq,
|
||||
) (resp *msg.ModifyMessageReactionExtensionsResp, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (m *msgServer) DeleteMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
req *msg.DeleteMessagesReactionExtensionsReq,
|
||||
) (resp *msg.DeleteMessagesReactionExtensionsResp, err error) {
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m args is:", req.String())
|
||||
//var rResp msg.DeleteMessagesReactionExtensionsResp
|
||||
//callbackResp := notification.callbackDeleteMessageReactionExtensions(req)
|
||||
//if callbackResp.ActionCode != constant.ActionAllow || callbackResp.ErrCode != 0 {
|
||||
// rResp.ErrCode = int32(callbackResp.ErrCode)
|
||||
// rResp.ErrMsg = callbackResp.ErrMsg
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = callbackResp.ErrMsg
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
//}
|
||||
////if ExternalExtension
|
||||
//if req.IsExternalExtensions {
|
||||
// rResp.Result = callbackResp.ResultReactionExtensionList
|
||||
// notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType,
|
||||
// req, &rResp, false, false)
|
||||
// return &rResp, nil
|
||||
//
|
||||
//}
|
||||
//for _, v := range callbackResp.ResultReactionExtensions {
|
||||
// if v.ErrCode != 0 {
|
||||
// func(req *[]*sdkws.KeyValue, typeKey string) {
|
||||
// for i := 0; i < len(*req); i++ {
|
||||
// if (*req)[i].TypeKey == typeKey {
|
||||
// *req = append((*req)[:i], (*req)[i+1:]...)
|
||||
// }
|
||||
// }
|
||||
// }(&req.Pb2Model, v.KeyValue.TypeKey)
|
||||
// rResp.Result = append(rResp.Result, v)
|
||||
// }
|
||||
//}
|
||||
//isExists, err := db.DB.JudgeMessageReactionExist(req.ClientMsgID, req.SessionType)
|
||||
//if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
//}
|
||||
//
|
||||
//if isExists {
|
||||
// log.Debug(req.OperationID, "redis handle this delete", req.String())
|
||||
// for _, v := range req.Pb2Model {
|
||||
// err := m.dMessageLocker.LockMessageTypeKey(req.ClientMsgID, v.TypeKey)
|
||||
// if err != nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 100, err.Error(), req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
//
|
||||
// redisValue, err := db.DB.GetMessageTypeKeyValue(req.ClientMsgID, req.SessionType, v.TypeKey)
|
||||
// if err != nil && err != go_redis.Nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 200, err.Error(), req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// utils.JsonStringToStruct(redisValue, temp)
|
||||
// if v.LatestUpdateTime != temp.LatestUpdateTime {
|
||||
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// } else {
|
||||
// newErr := db.DB.DeleteOneMessageKey(req.ClientMsgID, req.SessionType, v.TypeKey)
|
||||
// if newErr != nil {
|
||||
// setDeleteKeyResultInfo(&rResp, 201, newErr.Error(), req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// }
|
||||
// setDeleteKeyResultInfo(&rResp, 0, "", req.ClientMsgID, v.TypeKey, v)
|
||||
// }
|
||||
// }
|
||||
//} else {
|
||||
// err := m.dMessageLocker.LockGlobalMessage(req.ClientMsgID)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 100
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// mongoValue, err := db.DB.GetExtendMsg(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime)
|
||||
// if err != nil {
|
||||
// rResp.ErrCode = 200
|
||||
// rResp.ErrMsg = err.Error()
|
||||
// for _, value := range req.Pb2Model {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// return &rResp, nil
|
||||
// }
|
||||
// setValue := make(map[string]*sdkws.KeyValue)
|
||||
// for _, v := range req.Pb2Model {
|
||||
//
|
||||
// temp := new(sdkws.KeyValue)
|
||||
// if vv, ok := mongoValue.Pb2Model[v.TypeKey]; ok {
|
||||
// utils.CopyStructFields(temp, &vv)
|
||||
// if v.LatestUpdateTime != vv.LatestUpdateTime {
|
||||
// setDeleteKeyResultInfo(&rResp, 300, "message have update", req.ClientMsgID, v.TypeKey, temp)
|
||||
// continue
|
||||
// }
|
||||
// } else {
|
||||
// setDeleteKeyResultInfo(&rResp, 400, "key not in", req.ClientMsgID, v.TypeKey, v)
|
||||
// continue
|
||||
// }
|
||||
// temp.TypeKey = v.TypeKey
|
||||
// setValue[v.TypeKey] = temp
|
||||
// }
|
||||
// err = db.DB.DeleteReactionExtendMsgSet(req.conversationID, req.SessionType, req.ClientMsgID,
|
||||
// req.MsgFirstModifyTime, setValue)
|
||||
// if err != nil {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// temp.ErrMsg = err.Error()
|
||||
// temp.ErrCode = 100
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// } else {
|
||||
// for _, value := range setValue {
|
||||
// temp := new(msg.KeyValueResp)
|
||||
// temp.KeyValue = value
|
||||
// rResp.Result = append(rResp.Result, temp)
|
||||
// }
|
||||
// }
|
||||
// lockErr := m.dMessageLocker.UnLockGlobalMessage(req.ClientMsgID)
|
||||
// if lockErr != nil {
|
||||
// log.Error(req.OperationID, "UnLockGlobalMessage err:", lockErr.Error())
|
||||
// }
|
||||
//
|
||||
//}
|
||||
// notification.ExtendMessageDeleteNotification(req.OperationID, req.OpUserID, req.conversationID, req.SessionType,
|
||||
// req, &rResp, false, isExists)
|
||||
//log.Debug(req.OperationID, utils.GetSelfFuncName(), "m return is:", rResp.String())
|
||||
return resp, nil
|
||||
}
|
@ -1,105 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
cbapi "github.com/OpenIMSDK/Open-IM-Server/pkg/callbackstruct"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/http"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
)
|
||||
|
||||
func callbackSetMessageReactionExtensions(ctx context.Context, setReq *msg.SetMessageReactionExtensionsReq) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackBeforeSetMessageReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackBeforeSetMessageReactionExtensionCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: mcontext.GetOpUserID(ctx),
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsReact: setReq.IsReact,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackBeforeSetMessageReactionExtResp{}
|
||||
if err := http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg); err != nil {
|
||||
return err
|
||||
}
|
||||
setReq.MsgFirstModifyTime = resp.MsgFirstModifyTime
|
||||
return nil
|
||||
}
|
||||
|
||||
func callbackDeleteMessageReactionExtensions(
|
||||
ctx context.Context,
|
||||
setReq *msg.DeleteMessagesReactionExtensionsReq,
|
||||
) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackDeleteMessageReactionExtReq{
|
||||
OperationID: setReq.OperationID,
|
||||
CallbackCommand: constant.CallbackBeforeDeleteMessageReactionExtensionsCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: setReq.OpUserID,
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackDeleteMessageReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
||||
|
||||
func callbackGetMessageListReactionExtensions(ctx context.Context, getReq *msg.GetMessagesReactionExtensionsReq) error {
|
||||
if !config.Config.Callback.CallbackAfterSendGroupMsg.Enable {
|
||||
return nil
|
||||
}
|
||||
req := &cbapi.CallbackGetMessageListReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackGetMessageListReactionExtensionsCommand,
|
||||
ConversationID: getReq.ConversationID,
|
||||
OpUserID: mcontext.GetOperationID(ctx),
|
||||
SessionType: getReq.SessionType,
|
||||
TypeKeyList: getReq.TypeKeys,
|
||||
}
|
||||
resp := &cbapi.CallbackGetMessageListReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
||||
|
||||
func callbackAddMessageReactionExtensions(ctx context.Context, setReq *msg.ModifyMessageReactionExtensionsReq) error {
|
||||
req := &cbapi.CallbackAddMessageReactionExtReq{
|
||||
OperationID: mcontext.GetOperationID(ctx),
|
||||
CallbackCommand: constant.CallbackAddMessageListReactionExtensionsCommand,
|
||||
ConversationID: setReq.ConversationID,
|
||||
OpUserID: mcontext.GetOperationID(ctx),
|
||||
SessionType: setReq.SessionType,
|
||||
ReactionExtensionList: setReq.ReactionExtensions,
|
||||
ClientMsgID: setReq.ClientMsgID,
|
||||
IsReact: setReq.IsReact,
|
||||
IsExternalExtensions: setReq.IsExternalExtensions,
|
||||
MsgFirstModifyTime: setReq.MsgFirstModifyTime,
|
||||
}
|
||||
resp := &cbapi.CallbackAddMessageReactionExtResp{}
|
||||
return http.CallBackPostReturn(ctx, cbURL(), req, resp, config.Config.Callback.CallbackAfterSendGroupMsg)
|
||||
}
|
@ -0,0 +1,84 @@
|
||||
package msg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msg"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/sdkws"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq) (*msg.GetActiveUserResp, error) {
|
||||
msgCount, userCount, users, dateCount, err := m.MsgDatabase.RangeUserSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Group, req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pbUsers []*msg.ActiveUser
|
||||
if len(users) > 0 {
|
||||
userIDs := utils.Slice(users, func(e *unrelation.UserCount) string { return e.UserID })
|
||||
userMap, err := m.User.GetUsersInfoMap(ctx, userIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pbUsers = make([]*msg.ActiveUser, 0, len(users))
|
||||
for _, user := range users {
|
||||
pbUser := userMap[user.UserID]
|
||||
if pbUser == nil {
|
||||
pbUser = &sdkws.UserInfo{
|
||||
UserID: user.UserID,
|
||||
Nickname: user.UserID,
|
||||
}
|
||||
}
|
||||
pbUsers = append(pbUsers, &msg.ActiveUser{
|
||||
User: pbUser,
|
||||
Count: user.Count,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &msg.GetActiveUserResp{
|
||||
MsgCount: msgCount,
|
||||
UserCount: userCount,
|
||||
DateCount: dateCount,
|
||||
Users: pbUsers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *msgServer) GetActiveGroup(ctx context.Context, req *msg.GetActiveGroupReq) (*msg.GetActiveGroupResp, error) {
|
||||
msgCount, groupCount, groups, dateCount, err := m.MsgDatabase.RangeGroupSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pbGroups []*msg.ActiveGroup
|
||||
if len(groups) > 0 {
|
||||
groupIDs := utils.Slice(groups, func(e *unrelation.GroupCount) string { return e.GroupID })
|
||||
resp, err := m.Group.GetGroupInfos(ctx, groupIDs, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groupMap := make(map[string]*sdkws.GroupInfo, len(groups))
|
||||
for i, group := range groups {
|
||||
groupMap[group.GroupID] = resp[i]
|
||||
}
|
||||
pbGroups = make([]*msg.ActiveGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
pbGroup := groupMap[group.GroupID]
|
||||
if pbGroup == nil {
|
||||
pbGroup = &sdkws.GroupInfo{
|
||||
GroupID: group.GroupID,
|
||||
GroupName: group.GroupID,
|
||||
}
|
||||
}
|
||||
pbGroups = append(pbGroups, &msg.ActiveGroup{
|
||||
Group: pbGroup,
|
||||
Count: group.Count,
|
||||
})
|
||||
}
|
||||
}
|
||||
return &msg.GetActiveGroupResp{
|
||||
MsgCount: msgCount,
|
||||
GroupCount: groupCount,
|
||||
DateCount: dateCount,
|
||||
Groups: pbGroups,
|
||||
}, nil
|
||||
}
|
@ -1,52 +1,151 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package third
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (t *thirdServer) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
|
||||
return t.s3dataBase.ApplyPut(ctx, req)
|
||||
func (t *thirdServer) PartLimit(ctx context.Context, req *third.PartLimitReq) (*third.PartLimitResp, error) {
|
||||
limit := t.s3dataBase.PartLimit()
|
||||
return &third.PartLimitResp{
|
||||
MinPartSize: limit.MinPartSize,
|
||||
MaxPartSize: limit.MaxPartSize,
|
||||
MaxNumSize: int32(limit.MaxNumSize),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
|
||||
return t.s3dataBase.GetPut(ctx, req)
|
||||
func (t *thirdServer) PartSize(ctx context.Context, req *third.PartSizeReq) (*third.PartSizeResp, error) {
|
||||
size, err := t.s3dataBase.PartSize(ctx, req.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.PartSizeResp{Size: size}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error) {
|
||||
return t.s3dataBase.ConfirmPut(ctx, req)
|
||||
func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if err := checkUploadName(ctx, req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expireTime := time.Now().Add(t.defaultExpire)
|
||||
result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, t.defaultExpire, int(req.MaxParts))
|
||||
if err != nil {
|
||||
if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok {
|
||||
obj := &relation.ObjectModel{
|
||||
Name: req.Name,
|
||||
UserID: mcontext.GetOpUserID(ctx),
|
||||
Hash: req.Hash,
|
||||
Key: haErr.Object.Key,
|
||||
Size: haErr.Object.Size,
|
||||
ContentType: req.ContentType,
|
||||
Cause: req.Cause,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.InitiateMultipartUploadResp{
|
||||
Url: t.apiAddress(obj.Name),
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var sign *third.AuthSignParts
|
||||
if result.Sign != nil && len(result.Sign.Parts) > 0 {
|
||||
sign = &third.AuthSignParts{
|
||||
Url: result.Sign.URL,
|
||||
Query: toPbMapArray(result.Sign.Query),
|
||||
Header: toPbMapArray(result.Sign.Header),
|
||||
Parts: make([]*third.SignPart, len(result.Sign.Parts)),
|
||||
}
|
||||
for i, part := range result.Sign.Parts {
|
||||
sign.Parts[i] = &third.SignPart{
|
||||
PartNumber: int32(part.PartNumber),
|
||||
Url: part.URL,
|
||||
Query: toPbMapArray(part.Query),
|
||||
Header: toPbMapArray(part.Header),
|
||||
}
|
||||
}
|
||||
}
|
||||
return &third.InitiateMultipartUploadResp{
|
||||
Upload: &third.UploadInfo{
|
||||
UploadID: result.UploadID,
|
||||
PartSize: result.PartSize,
|
||||
Sign: sign,
|
||||
ExpireTime: expireTime.UnixMilli(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
|
||||
if req.Expires <= 0 {
|
||||
if err := tokenverify.CheckAdmin(ctx); err != nil {
|
||||
func (t *thirdServer) AuthSign(ctx context.Context, req *third.AuthSignReq) (*third.AuthSignResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
partNumbers := utils.Slice(req.PartNumbers, func(partNumber int32) int { return int(partNumber) })
|
||||
result, err := t.s3dataBase.AuthSign(ctx, req.UploadID, partNumbers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp := &third.AuthSignResp{
|
||||
Url: result.URL,
|
||||
Query: toPbMapArray(result.Query),
|
||||
Header: toPbMapArray(result.Header),
|
||||
Parts: make([]*third.SignPart, len(result.Parts)),
|
||||
}
|
||||
for i, part := range result.Parts {
|
||||
resp.Parts[i] = &third.SignPart{
|
||||
PartNumber: int32(part.PartNumber),
|
||||
Url: part.URL,
|
||||
Query: toPbMapArray(part.Query),
|
||||
Header: toPbMapArray(part.Header),
|
||||
}
|
||||
}
|
||||
return t.s3dataBase.GetUrl(ctx, req)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
|
||||
return t.s3dataBase.GetHashInfo(ctx, req)
|
||||
func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (*third.CompleteMultipartUploadResp, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if err := checkUploadName(ctx, req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := t.s3dataBase.CompleteMultipartUpload(ctx, req.UploadID, req.Parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
obj := &relation.ObjectModel{
|
||||
Name: req.Name,
|
||||
UserID: mcontext.GetOpUserID(ctx),
|
||||
Hash: result.Hash,
|
||||
Key: result.Key,
|
||||
Size: result.Size,
|
||||
ContentType: req.ContentType,
|
||||
Cause: req.Cause,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.CompleteMultipartUploadResp{
|
||||
Url: t.apiAddress(obj.Name),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) {
|
||||
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.AccessURLResp{
|
||||
Url: rawURL,
|
||||
ExpireTime: expireTime.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *thirdServer) CleanObject(ctx context.Context, now time.Time) {
|
||||
t.s3dataBase.CleanExpirationObject(ctx, now)
|
||||
func (t *thirdServer) apiAddress(name string) string {
|
||||
return t.apiURL + name
|
||||
}
|
||||
|
@ -0,0 +1,63 @@
|
||||
package third
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func toPbMapArray(m map[string][]string) []*third.KeyValues {
|
||||
res := make([]*third.KeyValues, 0, len(m))
|
||||
for key := range m {
|
||||
res = append(res, &third.KeyValues{
|
||||
Key: key,
|
||||
Values: m[key],
|
||||
})
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func checkUploadName(ctx context.Context, name string) error {
|
||||
if name == "" {
|
||||
return errs.ErrArgs.Wrap("name is empty")
|
||||
}
|
||||
if name[0] == '/' {
|
||||
return errs.ErrArgs.Wrap("name cannot start with `/`")
|
||||
}
|
||||
if err := checkValidObjectName(name); err != nil {
|
||||
return errs.ErrArgs.Wrap(err.Error())
|
||||
}
|
||||
opUserID := mcontext.GetOpUserID(ctx)
|
||||
if opUserID == "" {
|
||||
return errs.ErrNoPermission.Wrap("opUserID is empty")
|
||||
}
|
||||
if !tokenverify.IsManagerUserID(opUserID) {
|
||||
if !strings.HasPrefix(name, opUserID+"/") {
|
||||
return errs.ErrNoPermission.Wrap(fmt.Sprintf("name must start with `%s/`", opUserID))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkValidObjectNamePrefix(objectName string) error {
|
||||
if len(objectName) > 1024 {
|
||||
return errors.New("object name cannot be longer than 1024 characters")
|
||||
}
|
||||
if !utf8.ValidString(objectName) {
|
||||
return errors.New("object name with non UTF-8 strings are not supported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkValidObjectName(objectName string) error {
|
||||
if strings.TrimSpace(objectName) == "" {
|
||||
return errors.New("object name cannot be empty")
|
||||
}
|
||||
return checkValidObjectNamePrefix(objectName)
|
||||
}
|
@ -0,0 +1,38 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
func (c *MsgTool) ConversationsDestructMsgs() {
|
||||
log.ZInfo(context.Background(), "start msg destruct cron task")
|
||||
ctx := mcontext.NewCtx(utils.GetSelfFuncName())
|
||||
conversations, err := c.conversationDatabase.GetConversationIDsNeedDestruct(ctx)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "get conversation id need destruct failed", err)
|
||||
return
|
||||
}
|
||||
log.ZDebug(context.Background(), "nums conversations need destruct", "nums", len(conversations))
|
||||
for _, conversation := range conversations {
|
||||
log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "lastMsgDestructTime", conversation.LatestMsgDestructTime)
|
||||
seqs, err := c.msgDatabase.UserMsgsDestruct(ctx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
continue
|
||||
}
|
||||
if err := c.conversationDatabase.UpdateUsersConversationFiled(ctx, []string{conversation.OwnerUserID}, conversation.ConversationID, map[string]interface{}{"latest_msg_destruct_time": time.Now()}); err != nil {
|
||||
log.ZError(ctx, "updateUsersConversationFiled failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
continue
|
||||
}
|
||||
if len(seqs) > 0 {
|
||||
if err := c.msgNotificationSender.UserDeleteMsgsNotification(ctx, conversation.OwnerUserID, conversation.ConversationID, seqs); err != nil {
|
||||
log.ZError(ctx, "userDeleteMsgsNotification failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,5 @@
|
||||
package apiresp
|
||||
|
||||
type ApiFormat interface {
|
||||
ApiFormat()
|
||||
}
|
@ -1,101 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/dtm-labs/rockscache"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
)
|
||||
|
||||
const (
|
||||
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
|
||||
extendMsgCache = "EXTEND_MSG_CACHE:"
|
||||
)
|
||||
|
||||
type ExtendMsgSetCache interface {
|
||||
metaCache
|
||||
NewCache() ExtendMsgSetCache
|
||||
GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
firstModifyTime int64,
|
||||
) (extendMsg *unrelation.ExtendMsgModel, err error)
|
||||
DelExtendMsg(clientMsgID string) ExtendMsgSetCache
|
||||
}
|
||||
|
||||
type ExtendMsgSetCacheRedis struct {
|
||||
metaCache
|
||||
expireTime time.Duration
|
||||
rcClient *rockscache.Client
|
||||
extendMsgSetDB unrelation.ExtendMsgSetModelInterface
|
||||
}
|
||||
|
||||
func NewExtendMsgSetCacheRedis(
|
||||
rdb redis.UniversalClient,
|
||||
extendMsgSetDB unrelation.ExtendMsgSetModelInterface,
|
||||
options rockscache.Options,
|
||||
) ExtendMsgSetCache {
|
||||
rcClient := rockscache.NewClient(rdb, options)
|
||||
return &ExtendMsgSetCacheRedis{
|
||||
metaCache: NewMetaCacheRedis(rcClient),
|
||||
expireTime: time.Second * 30 * 60,
|
||||
extendMsgSetDB: extendMsgSetDB,
|
||||
rcClient: rcClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) NewCache() ExtendMsgSetCache {
|
||||
return &ExtendMsgSetCacheRedis{
|
||||
metaCache: NewMetaCacheRedis(e.rcClient, e.metaCache.GetPreDelKeys()...),
|
||||
expireTime: e.expireTime,
|
||||
extendMsgSetDB: e.extendMsgSetDB,
|
||||
rcClient: e.rcClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) getKey(clientMsgID string) string {
|
||||
return extendMsgCache + clientMsgID
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
firstModifyTime int64,
|
||||
) (extendMsg *unrelation.ExtendMsgModel, err error) {
|
||||
return getCache(
|
||||
ctx,
|
||||
e.rcClient,
|
||||
e.getKey(clientMsgID),
|
||||
e.expireTime,
|
||||
func(ctx context.Context) (*unrelation.ExtendMsgModel, error) {
|
||||
return e.extendMsgSetDB.TakeExtendMsg(ctx, conversationID, sessionType, clientMsgID, firstModifyTime)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (e *ExtendMsgSetCacheRedis) DelExtendMsg(clientMsgID string) ExtendMsgSetCache {
|
||||
new := e.NewCache()
|
||||
new.AddKeys(e.getKey(clientMsgID))
|
||||
return new
|
||||
}
|
@ -1,158 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
|
||||
unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx"
|
||||
)
|
||||
|
||||
// for mongoDB
|
||||
type ExtendMsgDatabase interface {
|
||||
CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error
|
||||
GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
ID string,
|
||||
opts *unRelationTb.GetAllExtendMsgSetOpts,
|
||||
) (sets []*unRelationTb.ExtendMsgSetModel, err error)
|
||||
GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*unRelationTb.ExtendMsgSetModel, error)
|
||||
InsertExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
msg *unRelationTb.ExtendMsgModel,
|
||||
) error
|
||||
InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error
|
||||
DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error
|
||||
GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *unRelationTb.ExtendMsgModel, err error)
|
||||
}
|
||||
|
||||
type extendMsgDatabase struct {
|
||||
database unRelationTb.ExtendMsgSetModelInterface
|
||||
cache cache.ExtendMsgSetCache
|
||||
ctxTx tx.CtxTx
|
||||
}
|
||||
|
||||
func NewExtendMsgDatabase(
|
||||
extendMsgModel unRelationTb.ExtendMsgSetModelInterface,
|
||||
cache cache.ExtendMsgSetCache,
|
||||
ctxTx tx.CtxTx,
|
||||
) ExtendMsgDatabase {
|
||||
return &extendMsgDatabase{database: extendMsgModel, cache: cache, ctxTx: ctxTx}
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) CreateExtendMsgSet(ctx context.Context, set *unRelationTb.ExtendMsgSetModel) error {
|
||||
return e.database.CreateExtendMsgSet(ctx, set)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetAllExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
opts *unRelationTb.GetAllExtendMsgSetOpts,
|
||||
) (sets []*unRelationTb.ExtendMsgSetModel, err error) {
|
||||
return e.database.GetAllExtendMsgSet(ctx, conversationID, opts)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
maxMsgUpdateTime int64,
|
||||
) (*unRelationTb.ExtendMsgSetModel, error) {
|
||||
return e.database.GetExtendMsgSet(ctx, conversationID, sessionType, maxMsgUpdateTime)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) InsertExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
msg *unRelationTb.ExtendMsgModel,
|
||||
) error {
|
||||
return e.database.InsertExtendMsg(ctx, conversationID, sessionType, msg)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
return e.database.InsertOrUpdateReactionExtendMsgSet(
|
||||
ctx,
|
||||
conversationID,
|
||||
sessionType,
|
||||
clientMsgID,
|
||||
msgFirstModifyTime,
|
||||
reactionExtensionList,
|
||||
)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) DeleteReactionExtendMsgSet(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
msgFirstModifyTime int64,
|
||||
reactionExtensionList map[string]*unRelationTb.KeyValueModel,
|
||||
) error {
|
||||
return e.database.DeleteReactionExtendMsgSet(
|
||||
ctx,
|
||||
conversationID,
|
||||
sessionType,
|
||||
clientMsgID,
|
||||
msgFirstModifyTime,
|
||||
reactionExtensionList,
|
||||
)
|
||||
}
|
||||
|
||||
func (e *extendMsgDatabase) GetExtendMsg(
|
||||
ctx context.Context,
|
||||
conversationID string,
|
||||
sessionType int32,
|
||||
clientMsgID string,
|
||||
maxMsgUpdateTime int64,
|
||||
) (extendMsg *unRelationTb.ExtendMsgModel, err error) {
|
||||
return e.cache.GetExtendMsg(ctx, conversationID, sessionType, clientMsgID, maxMsgUpdateTime)
|
||||
}
|
@ -0,0 +1,75 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
type S3Database interface {
|
||||
PartLimit() *s3.PartLimit
|
||||
PartSize(ctx context.Context, size int64) (int64, error)
|
||||
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
|
||||
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
|
||||
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
|
||||
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error)
|
||||
SetObject(ctx context.Context, info *relation.ObjectModel) error
|
||||
}
|
||||
|
||||
func NewS3Database(s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
|
||||
return &s3Database{
|
||||
s3: cont.New(s3),
|
||||
obj: obj,
|
||||
}
|
||||
}
|
||||
|
||||
type s3Database struct {
|
||||
s3 *cont.Controller
|
||||
obj relation.ObjectInfoModelInterface
|
||||
}
|
||||
|
||||
func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
return s.s3.PartSize(ctx, size)
|
||||
}
|
||||
|
||||
func (s *s3Database) PartLimit() *s3.PartLimit {
|
||||
return s.s3.PartLimit()
|
||||
}
|
||||
|
||||
func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
return s.s3.AuthSign(ctx, uploadID, partNumbers)
|
||||
}
|
||||
|
||||
func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) {
|
||||
return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts)
|
||||
}
|
||||
|
||||
func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) {
|
||||
return s.s3.CompleteUpload(ctx, uploadID, parts)
|
||||
}
|
||||
|
||||
func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) error {
|
||||
return s.obj.SetObject(ctx, info)
|
||||
}
|
||||
|
||||
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) {
|
||||
obj, err := s.obj.Take(ctx, name)
|
||||
if err != nil {
|
||||
return time.Time{}, "", err
|
||||
}
|
||||
opt := &s3.AccessURLOption{
|
||||
ContentType: obj.ContentType,
|
||||
}
|
||||
if filename := filepath.Base(obj.Name); filename != "" {
|
||||
opt.ContentDisposition = `attachment; filename=` + filename
|
||||
}
|
||||
expireTime := time.Now().Add(expire)
|
||||
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)
|
||||
if err != nil {
|
||||
return time.Time{}, "", err
|
||||
}
|
||||
return expireTime, rawURL, nil
|
||||
}
|
@ -1,567 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
hashPrefix = "hash"
|
||||
tempPrefix = "temp"
|
||||
fragmentPrefix = "fragment_"
|
||||
urlsName = "urls.json"
|
||||
)
|
||||
|
||||
type S3Database interface {
|
||||
ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
|
||||
GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
|
||||
ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
|
||||
GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error)
|
||||
GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error)
|
||||
CleanExpirationObject(ctx context.Context, t time.Time)
|
||||
}
|
||||
|
||||
func NewS3Database(
|
||||
obj obj.Interface,
|
||||
hash relation.ObjectHashModelInterface,
|
||||
info relation.ObjectInfoModelInterface,
|
||||
put relation.ObjectPutModelInterface,
|
||||
url *url.URL,
|
||||
) S3Database {
|
||||
return &s3Database{
|
||||
url: url,
|
||||
obj: obj,
|
||||
hash: hash,
|
||||
info: info,
|
||||
put: put,
|
||||
}
|
||||
}
|
||||
|
||||
type s3Database struct {
|
||||
url *url.URL
|
||||
obj obj.Interface
|
||||
hash relation.ObjectHashModelInterface
|
||||
info relation.ObjectInfoModelInterface
|
||||
put relation.ObjectPutModelInterface
|
||||
}
|
||||
|
||||
// today 今天的日期
|
||||
func (c *s3Database) today() string {
|
||||
return time.Now().Format("20060102")
|
||||
}
|
||||
|
||||
// fragmentName 根据序号生成文件名
|
||||
func (c *s3Database) fragmentName(index int) string {
|
||||
return fragmentPrefix + strconv.Itoa(index+1)
|
||||
}
|
||||
|
||||
// getFragmentNum 获取分片大小和分片数量
|
||||
func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
|
||||
if size := c.obj.MinFragmentSize(); fragmentSize < size {
|
||||
fragmentSize = size
|
||||
}
|
||||
if fragmentSize <= 0 || objectSize <= fragmentSize {
|
||||
return objectSize, 1
|
||||
} else {
|
||||
num := int(objectSize / fragmentSize)
|
||||
if objectSize%fragmentSize > 0 {
|
||||
num++
|
||||
}
|
||||
if n := c.obj.MaxFragmentNum(); num > n {
|
||||
num = n
|
||||
}
|
||||
return fragmentSize, num
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) CheckHash(hash string) error {
|
||||
val, err := hex.DecodeString(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(val) != md5.Size {
|
||||
return errs.ErrArgs.Wrap("invalid hash")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *s3Database) urlName(name string) string {
|
||||
u := url.URL{
|
||||
Scheme: c.url.Scheme,
|
||||
Opaque: c.url.Opaque,
|
||||
User: c.url.User,
|
||||
Host: c.url.Host,
|
||||
Path: c.url.Path,
|
||||
RawPath: c.url.RawPath,
|
||||
ForceQuery: c.url.ForceQuery,
|
||||
RawQuery: c.url.RawQuery,
|
||||
Fragment: c.url.Fragment,
|
||||
RawFragment: c.url.RawFragment,
|
||||
}
|
||||
v := make(url.Values, 1)
|
||||
v.Set("name", name)
|
||||
u.RawQuery = v.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (c *s3Database) UUID() string {
|
||||
return uuid.New().String()
|
||||
}
|
||||
|
||||
func (c *s3Database) HashName(hash string) string {
|
||||
return path.Join(hashPrefix, hash+"_"+c.today()+"_"+c.UUID())
|
||||
}
|
||||
|
||||
func (c *s3Database) isNotFound(err error) bool {
|
||||
return relation.IsNotFound(err)
|
||||
}
|
||||
|
||||
func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
|
||||
if err := c.CheckHash(req.Hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.obj.CheckName(req.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if req.ValidTime != 0 && req.ValidTime <= time.Now().UnixMilli() {
|
||||
return nil, errors.New("invalid ValidTime")
|
||||
}
|
||||
var expirationTime *time.Time
|
||||
if req.ValidTime != 0 {
|
||||
expirationTime = utils.ToPtr(time.UnixMilli(req.ValidTime))
|
||||
}
|
||||
if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
|
||||
o := relation.ObjectInfoModel{
|
||||
Name: req.Name,
|
||||
Hash: hash.Hash,
|
||||
ValidTime: expirationTime,
|
||||
ContentType: req.ContentType,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, &o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
// 新上传
|
||||
var fragmentNum int
|
||||
const effective = time.Hour * 24 * 2
|
||||
req.FragmentSize, fragmentNum = c.getFragmentNum(req.FragmentSize, req.Size)
|
||||
put := relation.ObjectPutModel{
|
||||
PutID: req.PutID,
|
||||
Hash: req.Hash,
|
||||
Name: req.Name,
|
||||
ObjectSize: req.Size,
|
||||
ContentType: req.ContentType,
|
||||
FragmentSize: req.FragmentSize,
|
||||
ValidTime: expirationTime,
|
||||
EffectiveTime: time.Now().Add(effective),
|
||||
}
|
||||
if put.PutID == "" {
|
||||
put.PutID = c.UUID()
|
||||
}
|
||||
if v, err := c.put.Take(ctx, put.PutID); err == nil {
|
||||
now := time.Now().UnixMilli()
|
||||
if v.EffectiveTime.UnixMilli() <= now {
|
||||
if err := c.put.DelPut(ctx, []string{v.PutID}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, errs.ErrDuplicateKey.Wrap(fmt.Sprintf("duplicate put id %s", put.PutID))
|
||||
}
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
put.Path = path.Join(tempPrefix, c.today(), req.Hash, put.PutID)
|
||||
putURLs := make([]string, 0, fragmentNum)
|
||||
for i := 0; i < fragmentNum; i++ {
|
||||
url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: path.Join(put.Path, c.fragmentName(i)),
|
||||
Effective: effective,
|
||||
MaxObjectSize: req.FragmentSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
putURLs = append(putURLs, url)
|
||||
}
|
||||
urlsJsonData, err := json.Marshal(putURLs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := md5.Sum(urlsJsonData)
|
||||
put.PutURLsHash = hex.EncodeToString(t[:])
|
||||
_, err = c.obj.PutObject(
|
||||
ctx,
|
||||
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(put.Path, urlsName)},
|
||||
bytes.NewReader(urlsJsonData),
|
||||
int64(len(urlsJsonData)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
put.CreateTime = time.Now()
|
||||
if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.ApplyPutResp{
|
||||
PutID: put.PutID,
|
||||
FragmentSize: put.FragmentSize,
|
||||
PutURLs: putURLs,
|
||||
ValidTime: put.EffectiveTime.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
|
||||
up, err := c.put.Take(ctx, req.PutID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := c.obj.GetObject(
|
||||
ctx,
|
||||
&obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(up.Path, urlsName)},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urlsData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := md5.Sum(urlsData)
|
||||
if h := hex.EncodeToString(t[:]); h != up.PutURLsHash {
|
||||
return nil, fmt.Errorf("invalid put urls hash %s %s", h, up.PutURLsHash)
|
||||
}
|
||||
var urls []string
|
||||
if err := json.Unmarshal(urlsData, &urls); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, fragmentNum := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
|
||||
if len(urls) != fragmentNum {
|
||||
return nil, fmt.Errorf("invalid urls length %d fragment %d", len(urls), fragmentNum)
|
||||
}
|
||||
fragments := make([]*third.GetPutFragment, fragmentNum)
|
||||
for i := 0; i < fragmentNum; i++ {
|
||||
name := path.Join(up.Path, c.fragmentName(i))
|
||||
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
if c.obj.IsNotFound(err) {
|
||||
fragments[i] = &third.GetPutFragment{Url: urls[i]}
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash, Url: urls[i]}
|
||||
}
|
||||
var validTime int64
|
||||
if up.ValidTime != nil {
|
||||
validTime = up.ValidTime.UnixMilli()
|
||||
}
|
||||
return &third.GetPutResp{
|
||||
FragmentSize: up.FragmentSize,
|
||||
Size: up.ObjectSize,
|
||||
Name: up.Name,
|
||||
Hash: up.Hash,
|
||||
Fragments: fragments,
|
||||
PutURLsHash: up.PutURLsHash,
|
||||
ContentType: up.ContentType,
|
||||
ValidTime: validTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
|
||||
put, err := c.put.Take(ctx, req.PutID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, pack := c.getFragmentNum(put.FragmentSize, put.ObjectSize)
|
||||
defer func() {
|
||||
if _err == nil {
|
||||
// 清理上传的碎片
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "deleteObject failed", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
}
|
||||
}
|
||||
}()
|
||||
now := time.Now().UnixMilli()
|
||||
if put.EffectiveTime.UnixMilli() < now {
|
||||
return nil, errs.ErrFileUploadedExpired.Wrap("put expired")
|
||||
}
|
||||
if put.ValidTime != nil && put.ValidTime.UnixMilli() < now {
|
||||
return nil, errs.ErrFileUploadedExpired.Wrap("object expired")
|
||||
}
|
||||
if hash, err := c.hash.Take(ctx, put.Hash, c.obj.Name()); err == nil {
|
||||
o := relation.ObjectInfoModel{
|
||||
Name: put.Name,
|
||||
Hash: hash.Hash,
|
||||
ValidTime: put.ValidTime,
|
||||
ContentType: put.ContentType,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, &o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: put.Path,
|
||||
})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
}
|
||||
}()
|
||||
// 服务端已存在
|
||||
return &third.ConfirmPutResp{
|
||||
Url: c.urlName(o.Name),
|
||||
}, nil
|
||||
} else if !c.isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
src := make([]obj.BucketObject, pack)
|
||||
for i := 0; i < pack; i++ {
|
||||
name := path.Join(put.Path, c.fragmentName(i))
|
||||
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i+1 == pack { // 最后一个
|
||||
size := put.ObjectSize - put.FragmentSize*int64(i)
|
||||
if size != o.Size {
|
||||
return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
|
||||
}
|
||||
} else {
|
||||
if o.Size != put.FragmentSize {
|
||||
return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, put.FragmentSize, o.Hash)
|
||||
}
|
||||
}
|
||||
src[i] = obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: name,
|
||||
}
|
||||
}
|
||||
dst := &obj.BucketObject{
|
||||
Bucket: c.obj.DataBucket(),
|
||||
Name: c.HashName(put.Hash),
|
||||
}
|
||||
if len(src) == 1 { // 未分片直接触发copy
|
||||
// 检查数据完整性,避免脏数据
|
||||
o, err := c.obj.GetObjectInfo(ctx, &src[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if put.ObjectSize != o.Size {
|
||||
return nil, fmt.Errorf("size mismatching should %d reality %d", put.ObjectSize, o.Size)
|
||||
}
|
||||
if put.Hash != o.Hash {
|
||||
return nil, fmt.Errorf("hash mismatching should %s reality %s", put.Hash, o.Hash)
|
||||
}
|
||||
if err := c.obj.CopyObject(ctx, &src[0], dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
tempBucket := &obj.BucketObject{
|
||||
Bucket: c.obj.TempBucket(),
|
||||
Name: path.Join(put.Path, "merge_"+c.UUID()),
|
||||
}
|
||||
defer func() { // 清理合成的文件
|
||||
if err := c.obj.DeleteObject(ctx, tempBucket); err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", tempBucket.Bucket, "Path", tempBucket.Name)
|
||||
}
|
||||
}()
|
||||
err := c.obj.ComposeObject(ctx, src, tempBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := c.obj.GetObjectInfo(ctx, tempBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if put.ObjectSize != info.Size {
|
||||
return nil, fmt.Errorf("size mismatch should %d reality %d", put.ObjectSize, info.Size)
|
||||
}
|
||||
if put.Hash != info.Hash {
|
||||
return nil, fmt.Errorf("hash mismatch should %s reality %s", put.Hash, info.Hash)
|
||||
}
|
||||
if err := c.obj.CopyObject(ctx, tempBucket, dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
h := &relation.ObjectHashModel{
|
||||
Hash: put.Hash,
|
||||
Engine: c.obj.Name(),
|
||||
Size: put.ObjectSize,
|
||||
Bucket: c.obj.DataBucket(),
|
||||
Name: dst.Name,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := &relation.ObjectInfoModel{
|
||||
Name: put.Name,
|
||||
Hash: put.Hash,
|
||||
ContentType: put.ContentType,
|
||||
ValidTime: put.ValidTime,
|
||||
CreateTime: time.Now(),
|
||||
}
|
||||
if err := c.info.SetObject(ctx, o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.put.DelPut(ctx, []string{put.PutID}); err != nil {
|
||||
log.ZError(ctx, "DelPut", err, "PutID", put.PutID)
|
||||
}
|
||||
return &third.ConfirmPutResp{
|
||||
Url: c.urlName(o.Name),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
|
||||
info, err := c.info.Take(ctx, req.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.ValidTime != nil && info.ValidTime.Before(time.Now()) {
|
||||
return nil, errs.ErrRecordNotFound.Wrap("object expired")
|
||||
}
|
||||
hash, err := c.hash.Take(ctx, info.Hash, c.obj.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt := obj.HeaderOption{ContentType: info.ContentType}
|
||||
if req.Attachment {
|
||||
opt.Filename = info.Name
|
||||
}
|
||||
u, err := c.obj.PresignedGetURL(ctx, hash.Bucket, hash.Name, time.Duration(req.Expires)*time.Millisecond, &opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.GetUrlResp{
|
||||
Url: u,
|
||||
Size: hash.Size,
|
||||
Hash: hash.Hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *s3Database) CleanExpirationObject(ctx context.Context, t time.Time) {
|
||||
// 清理上传产生的临时文件
|
||||
c.cleanPutTemp(ctx, t, 10)
|
||||
// 清理hash引用全过期的文件
|
||||
c.cleanExpirationObject(ctx, t)
|
||||
// 清理没有引用的hash对象
|
||||
c.clearNoCitation(ctx, c.obj.Name(), 10)
|
||||
}
|
||||
|
||||
func (c *s3Database) cleanPutTemp(ctx context.Context, t time.Time, num int) {
|
||||
for {
|
||||
puts, err := c.put.FindExpirationPut(ctx, t, num)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "FindExpirationPut", err, "Time", t, "Num", num)
|
||||
return
|
||||
}
|
||||
if len(puts) == 0 {
|
||||
return
|
||||
}
|
||||
for _, put := range puts {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
|
||||
return
|
||||
}
|
||||
}
|
||||
ids := utils.Slice(puts, func(e *relation.ObjectPutModel) string { return e.PutID })
|
||||
err = c.put.DelPut(ctx, ids)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DelPut", err, "PutID", ids)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) cleanExpirationObject(ctx context.Context, t time.Time) {
|
||||
err := c.info.DeleteExpiration(ctx, t)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteExpiration", err, "Time", t)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) clearNoCitation(ctx context.Context, engine string, limit int) {
|
||||
for {
|
||||
list, err := c.hash.DeleteNoCitation(ctx, engine, limit)
|
||||
if err != nil {
|
||||
log.ZError(ctx, "DeleteNoCitation", err, "Engine", engine, "Limit", limit)
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
var hasErr bool
|
||||
for _, h := range list {
|
||||
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: h.Bucket, Name: h.Name})
|
||||
if err != nil {
|
||||
hasErr = true
|
||||
log.ZError(ctx, "DeleteObject", err, "Bucket", h.Bucket, "Path", h.Name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if hasErr {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3Database) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
|
||||
if err := c.CheckHash(req.Hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o, err := c.hash.Take(ctx, req.Hash, c.obj.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &third.GetHashInfoResp{
|
||||
Hash: o.Hash,
|
||||
Size: o.Size,
|
||||
}, nil
|
||||
}
|
@ -1,258 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package obj
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
func NewMinioInterface() (Interface, error) {
|
||||
conf := config.Config.Object.Minio
|
||||
u, err := url.Parse(conf.Endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio endpoint parse %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid minio endpoint scheme %s", u.Scheme)
|
||||
}
|
||||
client, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio new client %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
|
||||
defer cancel()
|
||||
initBucket := func(ctx context.Context) error {
|
||||
for _, bucket := range utils.Distinct([]string{conf.TempBucket, conf.DataBucket}) {
|
||||
exists, err := client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio bucket %s exists %w", bucket, err)
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
opt := minio.MakeBucketOptions{
|
||||
Region: conf.Location,
|
||||
ObjectLocking: conf.IsDistributedMod,
|
||||
}
|
||||
if err := client.MakeBucket(ctx, bucket, opt); err != nil {
|
||||
return fmt.Errorf("minio make bucket %s %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := initBucket(ctx); err != nil {
|
||||
fmt.Println("minio init error:", err)
|
||||
}
|
||||
return &minioImpl{
|
||||
client: client,
|
||||
tempBucket: conf.TempBucket,
|
||||
dataBucket: conf.DataBucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type minioImpl struct {
|
||||
tempBucket string // 上传桶
|
||||
dataBucket string // 永久桶
|
||||
urlstr string // 访问地址
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func (m *minioImpl) Name() string {
|
||||
return "minio"
|
||||
}
|
||||
|
||||
func (m *minioImpl) MinFragmentSize() int64 {
|
||||
return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
|
||||
}
|
||||
|
||||
func (m *minioImpl) MaxFragmentNum() int {
|
||||
return 1000 // 最大分片数量 minio.maxPartsCount
|
||||
}
|
||||
|
||||
func (m *minioImpl) MinExpirationTime() time.Duration {
|
||||
return time.Hour * 24
|
||||
}
|
||||
|
||||
func (m *minioImpl) TempBucket() string {
|
||||
return m.tempBucket
|
||||
}
|
||||
|
||||
func (m *minioImpl) DataBucket() string {
|
||||
return m.dataBucket
|
||||
}
|
||||
|
||||
func (m *minioImpl) PresignedGetURL(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
name string,
|
||||
expires time.Duration,
|
||||
opt *HeaderOption,
|
||||
) (string, error) {
|
||||
var reqParams url.Values
|
||||
if opt != nil {
|
||||
reqParams = make(url.Values)
|
||||
if opt.ContentType != "" {
|
||||
reqParams.Set("response-content-type", opt.ContentType)
|
||||
}
|
||||
if opt.Filename != "" {
|
||||
reqParams.Set("response-content-disposition", "attachment;filename="+opt.Filename)
|
||||
}
|
||||
}
|
||||
u, err := m.client.PresignedGetObject(ctx, bucket, name, expires, reqParams)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
|
||||
if args.Effective <= 0 {
|
||||
return "", errors.New("EffectiveTime <= 0")
|
||||
}
|
||||
_, err := m.GetObjectInfo(ctx, &BucketObject{
|
||||
Bucket: m.tempBucket,
|
||||
Name: args.Name,
|
||||
})
|
||||
if err == nil {
|
||||
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
|
||||
} else if !m.IsNotFound(err) {
|
||||
return "", err
|
||||
}
|
||||
u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio apply error: %w", err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
|
||||
info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ObjectInfo{
|
||||
Size: info.Size,
|
||||
Hash: info.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
|
||||
_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
|
||||
Bucket: dst.Bucket,
|
||||
Object: dst.Name,
|
||||
}, minio.CopySrcOptions{
|
||||
Bucket: src.Bucket,
|
||||
Object: src.Name,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *minioImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
|
||||
return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (m *minioImpl) MoveObjectInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
|
||||
if err := m.CopyObject(ctx, src, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.DeleteObject(ctx, src)
|
||||
}
|
||||
|
||||
func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
|
||||
destOptions := minio.CopyDestOptions{
|
||||
Bucket: dst.Bucket,
|
||||
Object: dst.Name + ".temp",
|
||||
}
|
||||
sources := make([]minio.CopySrcOptions, len(src))
|
||||
for i, s := range src {
|
||||
sources[i] = minio.CopySrcOptions{
|
||||
Bucket: s.Bucket,
|
||||
Object: s.Name,
|
||||
}
|
||||
}
|
||||
_, err := m.client.ComposeObject(ctx, destOptions, sources...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.MoveObjectInfo(ctx, &BucketObject{
|
||||
Bucket: destOptions.Bucket,
|
||||
Name: destOptions.Object,
|
||||
}, &BucketObject{
|
||||
Bucket: dst.Bucket,
|
||||
Name: dst.Name,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *minioImpl) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (m *minioImpl) PutObject(
|
||||
ctx context.Context,
|
||||
info *BucketObject,
|
||||
reader io.Reader,
|
||||
size int64,
|
||||
) (*ObjectInfo, error) {
|
||||
update, err := m.client.PutObject(ctx, info.Bucket, info.Name, reader, size, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ObjectInfo{
|
||||
Size: update.Size,
|
||||
Hash: update.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
|
||||
object, err := m.client.GetObject(ctx, info.Bucket, info.Name, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := object.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewSizeReader(object, stat.Size), nil
|
||||
}
|
||||
|
||||
func (m *minioImpl) CheckName(name string) error {
|
||||
return s3utils.CheckValidObjectName(name)
|
||||
}
|
@ -1,110 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package obj
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BucketObject struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type ApplyPutArgs struct {
|
||||
Bucket string
|
||||
Name string
|
||||
Effective time.Duration // 申请有效时间
|
||||
Header http.Header // header
|
||||
MaxObjectSize int64
|
||||
}
|
||||
|
||||
type HeaderOption struct {
|
||||
ContentType string
|
||||
Filename string
|
||||
}
|
||||
|
||||
type ObjectInfo struct {
|
||||
Size int64
|
||||
Hash string
|
||||
}
|
||||
|
||||
type SizeReader interface {
|
||||
io.ReadCloser
|
||||
Size() int64
|
||||
}
|
||||
|
||||
func NewSizeReader(r io.ReadCloser, size int64) SizeReader {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
return &sizeReader{
|
||||
size: size,
|
||||
ReadCloser: r,
|
||||
}
|
||||
}
|
||||
|
||||
type sizeReader struct {
|
||||
size int64
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *sizeReader) Size() int64 {
|
||||
return r.size
|
||||
}
|
||||
|
||||
type Interface interface {
|
||||
// Name 存储名字
|
||||
Name() string
|
||||
// MinFragmentSize 最小允许的分片大小
|
||||
MinFragmentSize() int64
|
||||
// MaxFragmentNum 最大允许的分片数量
|
||||
MaxFragmentNum() int
|
||||
// MinExpirationTime 最小过期时间
|
||||
MinExpirationTime() time.Duration
|
||||
// TempBucket 临时桶名,用于上传
|
||||
TempBucket() string
|
||||
// DataBucket 永久存储的桶名
|
||||
DataBucket() string
|
||||
// PresignedGetURL 通过桶名和对象名返回URL
|
||||
PresignedGetURL(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
name string,
|
||||
expires time.Duration,
|
||||
opt *HeaderOption,
|
||||
) (string, error)
|
||||
// PresignedPutURL 申请上传,返回PUT的上传地址
|
||||
PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
|
||||
// GetObjectInfo 获取对象信息
|
||||
GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
|
||||
// CopyObject 复制对象
|
||||
CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error
|
||||
// DeleteObject 删除对象(不存在返回nil)
|
||||
DeleteObject(ctx context.Context, info *BucketObject) error
|
||||
// ComposeObject 合并对象
|
||||
ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
|
||||
// IsNotFound 判断是不是不存在导致的错误
|
||||
IsNotFound(err error) bool
|
||||
// CheckName 检查名字是否可用
|
||||
CheckName(name string) error
|
||||
// PutObject 上传文件
|
||||
PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error)
|
||||
// GetObject 下载文件
|
||||
GetObject(ctx context.Context, info *BucketObject) (SizeReader, error)
|
||||
}
|
@ -1,38 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"testing"
|
||||
)
|
||||
|
||||
//TestNewGormDB Test the retry of sporadic errors and the direct exit of wrong password.
|
||||
func TestNewGormDB(t *testing.T) {
|
||||
err := config.InitConfig("config_folder_path")
|
||||
if err != nil {
|
||||
fmt.Println("config load error")
|
||||
return
|
||||
}
|
||||
db, err := newMysqlGormDB()
|
||||
if err != nil {
|
||||
fmt.Println("password error")
|
||||
return
|
||||
}
|
||||
if db != nil {
|
||||
fmt.Println("success connect")
|
||||
}
|
||||
}
|
@ -1,66 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectHashGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
|
||||
return &ObjectHashGorm{
|
||||
NewMetaDB(db, &relation.ObjectHashModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
|
||||
return &ObjectHashGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectHashModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) Take(
|
||||
ctx context.Context,
|
||||
hash string,
|
||||
engine string,
|
||||
) (oh *relation.ObjectHashModel, err error) {
|
||||
oh = &relation.ObjectHashModel{}
|
||||
return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
|
||||
return utils.Wrap1(o.DB.Create(h).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectHashGorm) DeleteNoCitation(
|
||||
ctx context.Context,
|
||||
engine string,
|
||||
num int,
|
||||
) (list []*relation.ObjectHashModel, err error) {
|
||||
err = o.DB.Table(relation.ObjectHashModelTableName, "as h").Select("h.*").
|
||||
Joins("LEFT JOIN "+relation.ObjectInfoModelTableName+" as i ON h.hash = i.hash").
|
||||
Where("h.engine = ? AND i.hash IS NULL", engine).
|
||||
Limit(num).
|
||||
Find(&list).Error
|
||||
return list, utils.Wrap1(err)
|
||||
}
|
@ -1,69 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectInfoGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(db, &relation.ObjectInfoModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectInfoModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
|
||||
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
return errs.Wrap(o.DB.WithContext(ctx).Create(obj).Error)
|
||||
//return errs.Wrap(o.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
|
||||
// return errs.Wrap(err)
|
||||
// }
|
||||
// return errs.Wrap(tx.Create(obj).Error)
|
||||
//}))
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
|
||||
info = &relation.ObjectInfoModel{}
|
||||
return info, utils.Wrap1(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) DeleteExpiration(ctx context.Context, expiration time.Time) (err error) {
|
||||
return utils.Wrap1(
|
||||
o.DB.WithContext(ctx).
|
||||
Where("expiration_time IS NOT NULL AND expiration_time <= ?", expiration).
|
||||
Delete(&relation.ObjectInfoModel{}).
|
||||
Error,
|
||||
)
|
||||
}
|
@ -0,0 +1,36 @@
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type ObjectInfoGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(db, &relation.ObjectModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
|
||||
return &ObjectInfoGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectModel) (err error) {
|
||||
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).FirstOrCreate(obj).Error; err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectModel, err error) {
|
||||
info = &relation.ObjectModel{}
|
||||
return info, errs.Wrap(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
|
||||
}
|
@ -1,67 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package relation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
|
||||
)
|
||||
|
||||
type ObjectPutGorm struct {
|
||||
*MetaDB
|
||||
}
|
||||
|
||||
func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
|
||||
return &ObjectPutGorm{
|
||||
NewMetaDB(db, &relation.ObjectPutModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
|
||||
return &ObjectPutGorm{
|
||||
NewMetaDB(tx.(*gorm.DB), &relation.ObjectPutModel{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
|
||||
return utils.Wrap1(o.DB.Create(m).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
|
||||
put = &relation.ObjectPutModel{}
|
||||
return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
|
||||
return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) FindExpirationPut(
|
||||
ctx context.Context,
|
||||
expirationTime time.Time,
|
||||
num int,
|
||||
) (list []*relation.ObjectPutModel, err error) {
|
||||
err = o.DB.Where("effective_time <= ?", expirationTime).Limit(num).Find(&list).Error
|
||||
return list, utils.Wrap1(err)
|
||||
}
|
||||
|
||||
func (o *ObjectPutGorm) DelPut(ctx context.Context, ids []string) (err error) {
|
||||
return utils.Wrap1(o.DB.Where("put_id IN ?", ids).Delete(&relation.ObjectPutModel{}).Error)
|
||||
}
|
@ -0,0 +1,9 @@
|
||||
package cont
|
||||
|
||||
const (
|
||||
hashPath = "openim/data/hash/"
|
||||
tempPath = "openim/temp/"
|
||||
UploadTypeMultipart = 1 // 分片上传
|
||||
UploadTypePresigned = 2 // 预签名上传
|
||||
partSeparator = ","
|
||||
)
|
@ -0,0 +1,244 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
|
||||
"github.com/google/uuid"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func New(impl s3.Interface) *Controller {
|
||||
return &Controller{impl: impl}
|
||||
}
|
||||
|
||||
type Controller struct {
|
||||
impl s3.Interface
|
||||
}
|
||||
|
||||
func (c *Controller) HashPath(md5 string) string {
|
||||
return path.Join(hashPath, md5)
|
||||
}
|
||||
|
||||
func (c *Controller) NowPath() string {
|
||||
now := time.Now()
|
||||
return path.Join(
|
||||
fmt.Sprintf("%04d", now.Year()),
|
||||
fmt.Sprintf("%02d", now.Month()),
|
||||
fmt.Sprintf("%02d", now.Day()),
|
||||
fmt.Sprintf("%02d", now.Hour()),
|
||||
fmt.Sprintf("%02d", now.Minute()),
|
||||
fmt.Sprintf("%02d", now.Second()),
|
||||
)
|
||||
}
|
||||
|
||||
func (c *Controller) UUID() string {
|
||||
id := uuid.New()
|
||||
return hex.EncodeToString(id[:])
|
||||
}
|
||||
|
||||
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
return c.impl.PartSize(ctx, size)
|
||||
}
|
||||
|
||||
func (c *Controller) PartLimit() *s3.PartLimit {
|
||||
return c.impl.PartLimit()
|
||||
}
|
||||
|
||||
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
|
||||
return c.impl.StatObject(ctx, c.HashPath(hash))
|
||||
}
|
||||
|
||||
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
if size < 0 {
|
||||
return nil, errors.New("invalid size")
|
||||
}
|
||||
if hashBytes, err := hex.DecodeString(hash); err != nil {
|
||||
return nil, err
|
||||
} else if len(hashBytes) != md5.Size {
|
||||
return nil, errors.New("invalid md5")
|
||||
}
|
||||
partSize, err := c.impl.PartSize(ctx, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partNumber := int(size / partSize)
|
||||
if size%partSize > 0 {
|
||||
partNumber++
|
||||
}
|
||||
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
|
||||
return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
|
||||
}
|
||||
if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
|
||||
return nil, &HashAlreadyExistsError{Object: info}
|
||||
} else if !c.impl.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
if size <= partSize {
|
||||
// 预签名上传
|
||||
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
|
||||
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &InitiateUploadResult{
|
||||
UploadID: newMultipartUploadID(multipartUploadID{
|
||||
Type: UploadTypePresigned,
|
||||
ID: "",
|
||||
Key: key,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
}),
|
||||
PartSize: partSize,
|
||||
Sign: &s3.AuthSignResult{
|
||||
Parts: []s3.SignPart{
|
||||
{
|
||||
PartNumber: 1,
|
||||
URL: rawURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
} else {
|
||||
// 分片上传
|
||||
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if maxParts < 0 {
|
||||
maxParts = partNumber
|
||||
}
|
||||
var authSign *s3.AuthSignResult
|
||||
if maxParts > 0 {
|
||||
partNumbers := make([]int, partNumber)
|
||||
for i := 0; i < maxParts; i++ {
|
||||
partNumbers[i] = i + 1
|
||||
}
|
||||
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &InitiateUploadResult{
|
||||
UploadID: newMultipartUploadID(multipartUploadID{
|
||||
Type: UploadTypeMultipart,
|
||||
ID: upload.UploadID,
|
||||
Key: upload.Key,
|
||||
Size: size,
|
||||
Hash: hash,
|
||||
}),
|
||||
PartSize: partSize,
|
||||
Sign: authSign,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
|
||||
defer log.ZDebug(ctx, "return")
|
||||
upload, err := parseMultipartUploadID(uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
|
||||
fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
|
||||
return nil, errors.New("md5 mismatching")
|
||||
}
|
||||
if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
|
||||
return &UploadResult{
|
||||
Key: info.Key,
|
||||
Size: info.Size,
|
||||
Hash: info.ETag,
|
||||
}, nil
|
||||
} else if !c.impl.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject := make(map[string]struct{})
|
||||
defer func() {
|
||||
for key := range cleanObject {
|
||||
_ = c.impl.DeleteObject(ctx, key)
|
||||
}
|
||||
}()
|
||||
var targetKey string
|
||||
switch upload.Type {
|
||||
case UploadTypeMultipart:
|
||||
parts := make([]s3.Part, len(partHashs))
|
||||
for i, part := range partHashs {
|
||||
parts[i] = s3.Part{
|
||||
PartNumber: i + 1,
|
||||
ETag: part,
|
||||
}
|
||||
}
|
||||
// todo: 验证大小
|
||||
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetKey = result.Key
|
||||
case UploadTypePresigned:
|
||||
uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject[uploadInfo.Key] = struct{}{}
|
||||
if uploadInfo.Size != upload.Size {
|
||||
return nil, errors.New("upload size mismatching")
|
||||
}
|
||||
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
|
||||
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
|
||||
return nil, errs.ErrArgs.Wrap(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
|
||||
}
|
||||
// 防止在这个时候,并发操作,导致文件被覆盖
|
||||
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cleanObject[copyInfo.Key] = struct{}{}
|
||||
if copyInfo.ETag != uploadInfo.ETag {
|
||||
return nil, errors.New("[concurrency]copy md5 mismatching")
|
||||
}
|
||||
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
|
||||
targetKey = hashCopyInfo.Key
|
||||
default:
|
||||
return nil, errors.New("invalid upload id type")
|
||||
}
|
||||
return &UploadResult{
|
||||
Key: targetKey,
|
||||
Size: upload.Size,
|
||||
Hash: upload.Hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
upload, err := parseMultipartUploadID(uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch upload.Type {
|
||||
case UploadTypeMultipart:
|
||||
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
|
||||
case UploadTypePresigned:
|
||||
return nil, errors.New("presigned id not support auth sign")
|
||||
default:
|
||||
return nil, errors.New("invalid upload id type")
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) IsNotFound(err error) bool {
|
||||
return c.impl.IsNotFound(err)
|
||||
}
|
||||
|
||||
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
return c.impl.AccessURL(ctx, name, expire, opt)
|
||||
}
|
@ -0,0 +1,14 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
)
|
||||
|
||||
type HashAlreadyExistsError struct {
|
||||
Object *s3.ObjectInfo
|
||||
}
|
||||
|
||||
func (e *HashAlreadyExistsError) Error() string {
|
||||
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
|
||||
}
|
@ -0,0 +1,35 @@
|
||||
package cont
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type multipartUploadID struct {
|
||||
Type int `json:"a,omitempty"`
|
||||
ID string `json:"b,omitempty"`
|
||||
Key string `json:"c,omitempty"`
|
||||
Size int64 `json:"d,omitempty"`
|
||||
Hash string `json:"e,omitempty"`
|
||||
}
|
||||
|
||||
func newMultipartUploadID(id multipartUploadID) string {
|
||||
data, err := json.Marshal(id)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
|
||||
data, err := base64.StdEncoding.DecodeString(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
||||
}
|
||||
var upload multipartUploadID
|
||||
if err := json.Unmarshal(data, &upload); err != nil {
|
||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
||||
}
|
||||
return &upload, nil
|
||||
}
|
@ -0,0 +1,15 @@
|
||||
package cont
|
||||
|
||||
import "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
|
||||
type InitiateUploadResult struct {
|
||||
UploadID string `json:"uploadID"` // 上传ID
|
||||
PartSize int64 `json:"partSize"` // 分片大小
|
||||
Sign *s3.AuthSignResult `json:"sign"` // 分片信息
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
Hash string `json:"hash"`
|
||||
Size int64 `json:"size"`
|
||||
Key string `json:"key"`
|
||||
}
|
@ -0,0 +1,254 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 1 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 1000
|
||||
)
|
||||
|
||||
func NewCos() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Cos
|
||||
u, err := url.Parse(conf.BucketURL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: conf.SecretID,
|
||||
SecretKey: conf.SecretKey,
|
||||
SessionToken: conf.SessionToken,
|
||||
},
|
||||
})
|
||||
return &Cos{
|
||||
copyURL: u.Host + "/",
|
||||
client: client,
|
||||
credential: client.GetCredential(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Cos struct {
|
||||
copyURL string
|
||||
client *cos.Client
|
||||
credential *cos.Credential
|
||||
}
|
||||
|
||||
func (c *Cos) Engine() string {
|
||||
return "tencent-cos"
|
||||
}
|
||||
|
||||
func (c *Cos) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
UploadID: result.UploadID,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
opts := &cos.CompleteMultipartUploadOptions{
|
||||
Parts: make([]cos.Object, len(parts)),
|
||||
}
|
||||
for i, part := range parts {
|
||||
opts.Parts[i] = cos.Object{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
|
||||
}
|
||||
}
|
||||
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: result.Location,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
ETag: result.ETag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
result := s3.AuthSignResult{
|
||||
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Header: make(http.Header),
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
|
||||
result.Header = req.Header
|
||||
for i, partNumber := range partNumbers {
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
||||
|
||||
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
|
||||
_, err := c.client.Object.Delete(ctx, name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
if name != "" && name[0] == '/' {
|
||||
name = name[1:]
|
||||
}
|
||||
info, err := c.client.Object.Head(ctx, name, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ObjectInfo{Key: name}
|
||||
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
|
||||
return nil, errors.New("StatObject etag not found")
|
||||
}
|
||||
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
|
||||
return nil, errors.New("StatObject content-length not found")
|
||||
} else {
|
||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
|
||||
}
|
||||
if res.Size < 0 {
|
||||
return nil, errors.New("StatObject content-length must be greater than 0")
|
||||
}
|
||||
}
|
||||
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
|
||||
return nil, errors.New("StatObject last-modified not found")
|
||||
} else {
|
||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
sourceURL := c.copyURL + src
|
||||
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Cos) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *cos.ErrorResponse:
|
||||
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
|
||||
MaxParts: strconv.Itoa(maxParts),
|
||||
PartNumberMarker: strconv.Itoa(partNumberMarker),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
|
||||
}
|
||||
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
|
||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
||||
for i, part := range result.Parts {
|
||||
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: lastModified,
|
||||
ETag: part.ETag,
|
||||
Size: part.Size,
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//reqParams := make(url.Values)
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// reqParams.Set("Content-Type", opt.ContentType)
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
@ -0,0 +1,250 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
unsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 5 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 10000
|
||||
)
|
||||
|
||||
func NewMinio() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Minio
|
||||
u, err := url.Parse(conf.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
||||
Secure: u.Scheme == "https",
|
||||
}
|
||||
client, err := minio.New(u.Host, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Minio{
|
||||
bucket: conf.Bucket,
|
||||
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
|
||||
opts: opts,
|
||||
core: &minio.Core{Client: client},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Minio struct {
|
||||
bucket string
|
||||
bucketURL string
|
||||
opts *minio.Options
|
||||
core *minio.Core
|
||||
}
|
||||
|
||||
func (m *Minio) Engine() string {
|
||||
return "minio"
|
||||
}
|
||||
|
||||
func (m *Minio) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
Bucket: m.bucket,
|
||||
Key: name,
|
||||
UploadID: uploadID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
minioParts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
minioParts[i] = minio.CompletePart{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ToLower(part.ETag),
|
||||
}
|
||||
}
|
||||
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: upload.Location,
|
||||
Bucket: upload.Bucket,
|
||||
Key: upload.Key,
|
||||
ETag: strings.ToLower(upload.ETag),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
creds, err := m.opts.Creds.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := s3.AuthSignResult{
|
||||
URL: m.bucketURL + name,
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
for i, partNumber := range partNumbers {
|
||||
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
|
||||
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
URL: request.URL.String(),
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
Header: request.Header,
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rawURL.String(), nil
|
||||
}
|
||||
|
||||
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
|
||||
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.ObjectInfo{
|
||||
ETag: strings.ToLower(info.ETag),
|
||||
Key: info.Key,
|
||||
Size: info.Size,
|
||||
LastModified: info.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
|
||||
Bucket: m.bucket,
|
||||
Object: dst,
|
||||
}, minio.CopySrcOptions{
|
||||
Bucket: m.bucket,
|
||||
Object: src,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ToLower(result.ETag),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Minio) IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch e := err.(type) {
|
||||
case minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *minio.ErrorResponse:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
|
||||
}
|
||||
|
||||
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
MaxParts: result.MaxParts,
|
||||
NextPartNumberMarker: result.NextPartNumberMarker,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
|
||||
}
|
||||
for i, part := range result.ObjectParts {
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: part.LastModified,
|
||||
ETag: part.ETag,
|
||||
Size: part.Size,
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//reqParams := make(url.Values)
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// reqParams.Set("Content-Type", opt.ContentType)
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// reqParams.Set("Content-Disposition", opt.ContentDisposition)
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
@ -0,0 +1,259 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
|
||||
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
minPartSize = 1024 * 1024 * 1 // 1MB
|
||||
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
|
||||
maxNumSize = 10000
|
||||
)
|
||||
|
||||
func NewOSS() (s3.Interface, error) {
|
||||
conf := config.Config.Object.Oss
|
||||
if conf.BucketURL == "" {
|
||||
return nil, errors.New("bucket url is empty")
|
||||
}
|
||||
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, err := client.Bucket(conf.Bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
|
||||
conf.BucketURL += "/"
|
||||
}
|
||||
return &OSS{
|
||||
bucketURL: conf.BucketURL,
|
||||
bucket: bucket,
|
||||
credentials: client.Config.GetCredentials(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type OSS struct {
|
||||
bucketURL string
|
||||
bucket *oss.Bucket
|
||||
credentials oss.Credentials
|
||||
}
|
||||
|
||||
func (o *OSS) Engine() string {
|
||||
return "ali-oss"
|
||||
}
|
||||
|
||||
func (o *OSS) PartLimit() *s3.PartLimit {
|
||||
return &s3.PartLimit{
|
||||
MinPartSize: minPartSize,
|
||||
MaxPartSize: maxPartSize,
|
||||
MaxNumSize: maxNumSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
||||
result, err := o.bucket.InitiateMultipartUpload(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.InitiateMultipartUploadResult{
|
||||
UploadID: result.UploadID,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
||||
ossParts := make([]oss.UploadPart, len(parts))
|
||||
for i, part := range parts {
|
||||
ossParts[i] = oss.UploadPart{
|
||||
PartNumber: part.PartNumber,
|
||||
ETag: strings.ToUpper(part.ETag),
|
||||
}
|
||||
}
|
||||
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Bucket: o.bucket.BucketName,
|
||||
Key: name,
|
||||
}, ossParts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CompleteMultipartUploadResult{
|
||||
Location: result.Location,
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
|
||||
if size <= 0 {
|
||||
return 0, errors.New("size must be greater than 0")
|
||||
}
|
||||
if size > maxPartSize*maxNumSize {
|
||||
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
|
||||
}
|
||||
if size <= minPartSize*maxNumSize {
|
||||
return minPartSize, nil
|
||||
}
|
||||
partSize := size / maxNumSize
|
||||
if size%maxNumSize != 0 {
|
||||
partSize++
|
||||
}
|
||||
return partSize, nil
|
||||
}
|
||||
|
||||
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
||||
result := s3.AuthSignResult{
|
||||
URL: o.bucketURL + name,
|
||||
Query: url.Values{"uploadId": {uploadID}},
|
||||
Header: make(http.Header),
|
||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
||||
}
|
||||
for i, partNumber := range partNumbers {
|
||||
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o.credentials.GetSecurityToken() != "" {
|
||||
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
|
||||
}
|
||||
request.Header.Set(oss.HTTPHeaderHost, request.Host)
|
||||
request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
|
||||
authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
|
||||
request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
|
||||
result.Parts[i] = s3.SignPart{
|
||||
PartNumber: partNumber,
|
||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
||||
URL: request.URL.String(),
|
||||
Header: request.Header,
|
||||
}
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
||||
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
|
||||
}
|
||||
|
||||
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
||||
header, err := o.bucket.GetObjectMeta(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ObjectInfo{Key: name}
|
||||
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
|
||||
return nil, errors.New("StatObject etag not found")
|
||||
}
|
||||
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
|
||||
return nil, errors.New("StatObject content-length not found")
|
||||
} else {
|
||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
|
||||
}
|
||||
if res.Size < 0 {
|
||||
return nil, errors.New("StatObject content-length must be greater than 0")
|
||||
}
|
||||
}
|
||||
if lastModified := header.Get("Last-Modified"); lastModified == "" {
|
||||
return nil, errors.New("StatObject last-modified not found")
|
||||
} else {
|
||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
|
||||
return o.bucket.DeleteObject(name)
|
||||
}
|
||||
|
||||
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
||||
result, err := o.bucket.CopyObject(src, dst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3.CopyObjectInfo{
|
||||
Key: dst,
|
||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (o *OSS) IsNotFound(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case oss.ServiceError:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
case *oss.ServiceError:
|
||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
||||
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Key: name,
|
||||
Bucket: o.bucket.BucketName,
|
||||
})
|
||||
}
|
||||
|
||||
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
||||
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
|
||||
UploadID: uploadID,
|
||||
Key: name,
|
||||
Bucket: o.bucket.BucketName,
|
||||
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &s3.ListUploadedPartsResult{
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadID,
|
||||
MaxParts: result.MaxParts,
|
||||
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
|
||||
}
|
||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
||||
for i, part := range result.UploadedParts {
|
||||
res.UploadedParts[i] = s3.UploadedPart{
|
||||
PartNumber: part.PartNumber,
|
||||
LastModified: part.LastModified,
|
||||
ETag: part.ETag,
|
||||
Size: int64(part.Size),
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
||||
//var opts []oss.Option
|
||||
//if opt != nil {
|
||||
// if opt.ContentType != "" {
|
||||
// opts = append(opts, oss.ContentType(opt.ContentType))
|
||||
// }
|
||||
// if opt.ContentDisposition != "" {
|
||||
// opts = append(opts, oss.ContentDisposition(opt.ContentDisposition))
|
||||
// }
|
||||
//}
|
||||
if expire <= 0 {
|
||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
||||
} else if expire < time.Second {
|
||||
expire = time.Second
|
||||
}
|
||||
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second))
|
||||
}
|
@ -0,0 +1,81 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
|
||||
var keysList []string
|
||||
keysMap := make(map[string]string)
|
||||
srcKeys := make(map[string]string)
|
||||
|
||||
for k := range req.Header {
|
||||
srcKeys[strings.ToLower(k)] = ""
|
||||
}
|
||||
|
||||
for _, v := range o.bucket.Client.Config.AdditionalHeaders {
|
||||
if _, ok := srcKeys[strings.ToLower(v)]; ok {
|
||||
keysMap[strings.ToLower(v)] = ""
|
||||
}
|
||||
}
|
||||
|
||||
for k := range keysMap {
|
||||
keysList = append(keysList, k)
|
||||
}
|
||||
sort.Strings(keysList)
|
||||
return keysList, keysMap
|
||||
}
|
||||
|
||||
func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
|
||||
// Find out the "x-oss-"'s address in header of the request
|
||||
ossHeadersMap := make(map[string]string)
|
||||
additionalList, additionalMap := o.getAdditionalHeaderKeys(req)
|
||||
for k, v := range req.Header {
|
||||
if strings.HasPrefix(strings.ToLower(k), "x-oss-") {
|
||||
ossHeadersMap[strings.ToLower(k)] = v[0]
|
||||
} else if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
|
||||
if _, ok := additionalMap[strings.ToLower(k)]; ok {
|
||||
ossHeadersMap[strings.ToLower(k)] = v[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
hs := newHeaderSorter(ossHeadersMap)
|
||||
|
||||
// Sort the ossHeadersMap by the ascending order
|
||||
hs.Sort()
|
||||
|
||||
// Get the canonicalizedOSSHeaders
|
||||
canonicalizedOSSHeaders := ""
|
||||
for i := range hs.Keys {
|
||||
canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n"
|
||||
}
|
||||
|
||||
// Give other parameters values
|
||||
// when sign URL, date is expires
|
||||
date := req.Header.Get(oss.HTTPHeaderDate)
|
||||
contentType := req.Header.Get(oss.HTTPHeaderContentType)
|
||||
contentMd5 := req.Header.Get(oss.HTTPHeaderContentMD5)
|
||||
|
||||
// default is v1 signature
|
||||
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
|
||||
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
|
||||
|
||||
// v2 signature
|
||||
if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
|
||||
signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
|
||||
h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
|
||||
}
|
||||
_, _ = io.WriteString(h, signStr)
|
||||
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
|
||||
|
||||
return signedStr
|
||||
}
|
@ -0,0 +1,47 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// headerSorter defines the key-value structure for storing the sorted data in signHeader.
|
||||
type headerSorter struct {
|
||||
Keys []string
|
||||
Vals []string
|
||||
}
|
||||
|
||||
// newHeaderSorter is an additional function for function SignHeader.
|
||||
func newHeaderSorter(m map[string]string) *headerSorter {
|
||||
hs := &headerSorter{
|
||||
Keys: make([]string, 0, len(m)),
|
||||
Vals: make([]string, 0, len(m)),
|
||||
}
|
||||
|
||||
for k, v := range m {
|
||||
hs.Keys = append(hs.Keys, k)
|
||||
hs.Vals = append(hs.Vals, v)
|
||||
}
|
||||
return hs
|
||||
}
|
||||
|
||||
// Sort is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Sort() {
|
||||
sort.Sort(hs)
|
||||
}
|
||||
|
||||
// Len is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Len() int {
|
||||
return len(hs.Vals)
|
||||
}
|
||||
|
||||
// Less is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Less(i, j int) bool {
|
||||
return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0
|
||||
}
|
||||
|
||||
// Swap is an additional function for function SignHeader.
|
||||
func (hs *headerSorter) Swap(i, j int) {
|
||||
hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i]
|
||||
hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i]
|
||||
}
|