Merge branch 'v3dev' of github.com:OpenIMSDK/Open-IM-Server into v3dev

test-v3dev1^2
wangchuxiao 1 year ago
commit 9468589cc7

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
<mapping directory="" vcs="Git" />
</component>
</project>

@ -63,40 +63,25 @@ api:
object:
enable: minio #使用minio
apiURL: http://127.0.0.1:10002/third/object
apiURL: http://127.0.0.1:10002/object/
minio:
tempBucket: "openim" #不建议修改
dataBucket: "openim" #不建议修改
location: us-east-1 #不建议修改
bucket: "openim" #不建议修改
endpoint: http://127.0.0.1:10005 #minio对外服务的ip和端口app要能访问此ip和端口
accessKeyID: root #ID
secretAccessKey: openIM123 #秘钥
isDistributedMod: false #是否分布式多硬盘部署如果是多硬盘部署需要修改为true
tencent: #tencent cos
appID: 1314570205
region: ap-guangzhou
bucket: openim-test-1314570205
secretID: AKIDypY3K8lJk3NYVHqBXkneH5pMhsQFtKWX
secretKey: MZdXKxb9WhedR6dzAAzZyEVfgeR1NVP6
ali: #ali oss
regionID:
accessKeyID:
accessKeySecret:
stsEndpoint:
ossEndpoint:
bucket:
finalHost:
stsDurationSeconds:
OssRoleArn:
aws:
accessKeyID:
accessKeySecret:
region:
bucket:
finalHost:
roleArn:
externalId:
roleSessionName:
sessionToken: #token
cos: #tencent cos
bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
secretID: ""
secretKey: ""
sessionToken: ""
oss: #ali oss
endpoint: "https://oss-cn-chengdu.aliyuncs.com"
bucket: "demo-9999999"
bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com"
accessKeyID: ""
accessKeySecret: ""
sessionToken: ""
rpcPort: #rpc服务端口不建议修改端口由脚本读取后传入程序如启动多个程序只需要填入多个端口用逗号隔开如 [10110, 10111]
openImUserPort: [ 10110 ]

@ -17,13 +17,13 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/jinzhu/copier v0.3.5
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/minio/minio-go/v7 v7.0.22
github.com/minio/minio-go/v7 v7.0.59
github.com/mitchellh/mapstructure v1.5.0
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.13.0
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sirupsen/logrus v1.9.2 // indirect
github.com/stretchr/testify v1.8.3
github.com/tencentyun/qcloud-cos-sts-sdk v0.0.0-20210325043845-84a0811633ca
go.mongodb.org/mongo-driver v1.8.3
@ -39,6 +39,7 @@ require (
require github.com/google/uuid v1.3.0
require (
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.6.0
github.com/go-zookeeper/zk v1.0.3
@ -61,7 +62,7 @@ require (
github.com/clbanning/mxj v1.8.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
@ -85,15 +86,13 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/lithammer/shortuuid v3.0.0+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.4.0 // indirect
@ -106,7 +105,7 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@ -140,7 +139,7 @@ require (
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.10.0 // indirect
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
replace github.com/Shopify/sarama => github.com/Shopify/sarama v1.29.0

@ -61,6 +61,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis=
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -97,8 +99,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dtm-labs/rockscache v0.1.1 h1:6S1vgaHvGqrLd8Ka4hRTKeKPV7v+tT0MSkTIX81LRyA=
github.com/dtm-labs/rockscache v0.1.1/go.mod h1:c76WX0kyIibmQ2ACxUXvDvaLykoPakivMqIxt+UzE7A=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
@ -281,11 +283,10 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@ -313,14 +314,12 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.22 h1:iXhsiRyYh1ozm/+jN2qGgEIahYjEkvcpuu6NcdpSxcA=
github.com/minio/minio-go/v7 v7.0.22/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw=
github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -395,14 +394,14 @@ github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDO
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@ -797,8 +796,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

@ -109,12 +109,15 @@ func NewGinRouter(discov discoveryregistry.SvcDiscoveryRegistry, rdb redis.Unive
thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken)
thirdGroup.POST("/set_app_badge", t.SetAppBadge)
thirdGroup.POST("/apply_put", t.ApplyPut)
thirdGroup.POST("/get_put", t.GetPut)
thirdGroup.POST("/confirm_put", t.ConfirmPut)
thirdGroup.POST("/get_hash", t.GetHash)
thirdGroup.POST("/object", t.GetURL)
thirdGroup.GET("/object", t.GetURL)
objectGroup := r.Group("/object", ParseToken)
objectGroup.POST("/part_limit", t.PartLimit)
objectGroup.POST("/part_size", t.PartSize)
objectGroup.POST("/initiate_multipart_upload", t.InitiateMultipartUpload)
objectGroup.POST("/auth_sign", t.AuthSign)
objectGroup.POST("/complete_multipart_upload", t.CompleteMultipartUpload)
objectGroup.POST("/access_url", t.AccessURL)
objectGroup.GET("/object/*name", t.ObjectRedirect)
}
//Message
msgGroup := r.Group("/msg", ParseToken)

@ -1,18 +1,16 @@
package api
import (
"math/rand"
"net/http"
"strconv"
"github.com/OpenIMSDK/Open-IM-Server/pkg/a2r"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/rpcclient"
"github.com/gin-gonic/gin"
"math/rand"
"net/http"
"strconv"
)
type ThirdApi rpcclient.Third
@ -21,51 +19,48 @@ func NewThirdApi(discov discoveryregistry.SvcDiscoveryRegistry) ThirdApi {
return ThirdApi(*rpcclient.NewThird(discov))
}
func (o *ThirdApi) ApplyPut(c *gin.Context) {
a2r.Call(third.ThirdClient.ApplyPut, o.Client, c)
func (o *ThirdApi) FcmUpdateToken(c *gin.Context) {
a2r.Call(third.ThirdClient.FcmUpdateToken, o.Client, c)
}
func (o *ThirdApi) GetPut(c *gin.Context) {
a2r.Call(third.ThirdClient.GetPut, o.Client, c)
func (o *ThirdApi) SetAppBadge(c *gin.Context) {
a2r.Call(third.ThirdClient.SetAppBadge, o.Client, c)
}
func (o *ThirdApi) ConfirmPut(c *gin.Context) {
a2r.Call(third.ThirdClient.ConfirmPut, o.Client, c)
// #################### s3 ####################
func (o *ThirdApi) PartLimit(c *gin.Context) {
a2r.Call(third.ThirdClient.PartLimit, o.Client, c)
}
func (o *ThirdApi) GetHash(c *gin.Context) {
a2r.Call(third.ThirdClient.GetHashInfo, o.Client, c)
func (o *ThirdApi) PartSize(c *gin.Context) {
a2r.Call(third.ThirdClient.PartSize, o.Client, c)
}
func (o *ThirdApi) FcmUpdateToken(c *gin.Context) {
a2r.Call(third.ThirdClient.FcmUpdateToken, o.Client, c)
func (o *ThirdApi) InitiateMultipartUpload(c *gin.Context) {
a2r.Call(third.ThirdClient.InitiateMultipartUpload, o.Client, c)
}
func (o *ThirdApi) SetAppBadge(c *gin.Context) {
a2r.Call(third.ThirdClient.SetAppBadge, o.Client, c)
func (o *ThirdApi) AuthSign(c *gin.Context) {
a2r.Call(third.ThirdClient.AuthSign, o.Client, c)
}
func (o *ThirdApi) GetURL(c *gin.Context) {
if c.Request.Method == http.MethodPost {
a2r.Call(third.ThirdClient.GetUrl, o.Client, c)
return
}
name := c.Query("name")
if name == "" {
c.String(http.StatusBadRequest, "name is empty")
return
}
func (o *ThirdApi) CompleteMultipartUpload(c *gin.Context) {
a2r.Call(third.ThirdClient.CompleteMultipartUpload, o.Client, c)
}
func (o *ThirdApi) AccessURL(c *gin.Context) {
a2r.Call(third.ThirdClient.AccessURL, o.Client, c)
}
func (o *ThirdApi) ObjectRedirect(c *gin.Context) {
name := c.Param("name")
operationID := c.Query("operationID")
if operationID == "" {
operationID = "auto_" + strconv.Itoa(rand.Int())
}
expires, _ := strconv.ParseInt(c.Query("expires"), 10, 64)
if expires <= 0 {
expires = 3600 * 1000
operationID = strconv.Itoa(rand.Int())
}
attachment, _ := strconv.ParseBool(c.Query("attachment"))
c.Set(constant.OperationID, operationID)
resp, err := o.Client.GetUrl(mcontext.SetOperationID(c, operationID), &third.GetUrlReq{Name: name, Expires: expires, Attachment: attachment})
ctx := mcontext.SetOperationID(c, operationID)
resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name})
if err != nil {
if errs.ErrArgs.Is(err) {
c.String(http.StatusBadRequest, err.Error())

@ -3,11 +3,9 @@ package api
import (
"github.com/OpenIMSDK/Open-IM-Server/pkg/a2r"
"github.com/OpenIMSDK/Open-IM-Server/pkg/apiresp"
"github.com/OpenIMSDK/Open-IM-Server/pkg/apistruct"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/msggateway"
@ -51,15 +49,54 @@ func (u *UserApi) GetUsers(c *gin.Context) {
}
func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) {
params := apistruct.ManagementSendMsgReq{}
if err := c.BindJSON(&params); err != nil {
var req msggateway.GetUsersOnlineStatusReq
if err := c.BindJSON(&req); err != nil {
apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap())
return
}
if !tokenverify.IsAppManagerUid(c) {
apiresp.GinError(c, errs.ErrNoPermission.Wrap("only app manager can send message"))
conns, err := u.Discov.GetConns(c, config.Config.RpcRegisterName.OpenImMessageGatewayName)
if err != nil {
apiresp.GinError(c, err)
return
}
var wsResult []*msggateway.GetUsersOnlineStatusResp_SuccessResult
var respResult []*msggateway.GetUsersOnlineStatusResp_SuccessResult
flag := false
//Online push message
for _, v := range conns {
msgClient := msggateway.NewMsgGatewayClient(v)
reply, err := msgClient.GetUsersOnlineStatus(c, &req)
if err != nil {
log.ZWarn(c, "GetUsersOnlineStatus rpc err", err)
continue
} else {
wsResult = append(wsResult, reply.SuccessResult...)
}
}
// 遍历 api 请求体中的 userIDs
for _, v1 := range req.UserIDs {
flag = false
res := new(msggateway.GetUsersOnlineStatusResp_SuccessResult)
// 遍历从各个网关中获取的在线结果
for _, v2 := range wsResult {
// 如果匹配上说明在线,反之
if v2.UserID == v1 {
flag = true
res.UserID = v1
res.Status = constant.OnlineStatus
res.DetailPlatformStatus = append(res.DetailPlatformStatus, v2.DetailPlatformStatus...)
break
}
}
if !flag {
res.UserID = v1
res.Status = constant.OnlineStatus
}
respResult = append(respResult, res)
}
apiresp.GinSuccess(c, respResult)
}
func (u *UserApi) UserRegisterCount(c *gin.Context) {

@ -2,36 +2,144 @@ package third
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"time"
)
func (t *thirdServer) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
return t.s3dataBase.ApplyPut(ctx, req)
func (t *thirdServer) PartLimit(ctx context.Context, req *third.PartLimitReq) (*third.PartLimitResp, error) {
limit := t.s3dataBase.PartLimit()
return &third.PartLimitResp{
MinPartSize: limit.MinPartSize,
MaxPartSize: limit.MaxPartSize,
MaxNumSize: int32(limit.MaxNumSize),
}, nil
}
func (t *thirdServer) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
return t.s3dataBase.GetPut(ctx, req)
func (t *thirdServer) PartSize(ctx context.Context, req *third.PartSizeReq) (*third.PartSizeResp, error) {
size, err := t.s3dataBase.PartSize(ctx, req.Size)
if err != nil {
return nil, err
}
return &third.PartSizeResp{Size: size}, nil
}
func (t *thirdServer) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error) {
return t.s3dataBase.ConfirmPut(ctx, req)
func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
if err := checkUploadName(ctx, req.Name); err != nil {
return nil, err
}
result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, time.Hour*24, int(req.MaxParts))
if err != nil {
if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok {
obj := &relation.ObjectModel{
Name: req.Name,
UserID: mcontext.GetOpUserID(ctx),
Hash: req.Hash,
Key: haErr.Object.Key,
Size: haErr.Object.Size,
ContentType: req.ContentType,
Cause: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
return nil, err
}
return &third.InitiateMultipartUploadResp{
Url: t.apiAddress(obj.Key),
}, nil
}
return nil, err
}
var sign *third.AuthSignParts
if result.Sign != nil && len(result.Sign.Parts) > 0 {
sign = &third.AuthSignParts{
Url: result.Sign.URL,
Query: toPbMapArray(result.Sign.Query),
Header: toPbMapArray(result.Sign.Header),
Parts: make([]*third.SignPart, len(result.Sign.Parts)),
}
for i, part := range result.Sign.Parts {
sign.Parts[i] = &third.SignPart{
PartNumber: int32(part.PartNumber),
Url: part.URL,
Query: toPbMapArray(part.Query),
Header: toPbMapArray(part.Header),
}
}
}
return &third.InitiateMultipartUploadResp{
Upload: &third.UploadInfo{
UploadID: result.UploadID,
PartSize: result.PartSize,
Sign: sign,
},
}, nil
}
func (t *thirdServer) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
if req.Expires <= 0 {
if err := tokenverify.CheckAdmin(ctx); err != nil {
return nil, err
func (t *thirdServer) AuthSign(ctx context.Context, req *third.AuthSignReq) (*third.AuthSignResp, error) {
partNumbers := utils.Slice(req.PartNumbers, func(partNumber int32) int { return int(partNumber) })
result, err := t.s3dataBase.AuthSign(ctx, req.UploadID, partNumbers)
if err != nil {
return nil, err
}
resp := &third.AuthSignResp{
Url: result.URL,
Query: toPbMapArray(result.Query),
Header: toPbMapArray(result.Header),
Parts: make([]*third.SignPart, len(result.Parts)),
}
for i, part := range result.Parts {
resp.Parts[i] = &third.SignPart{
PartNumber: int32(part.PartNumber),
Url: part.URL,
Query: toPbMapArray(part.Query),
Header: toPbMapArray(part.Header),
}
}
return t.s3dataBase.GetUrl(ctx, req)
return resp, nil
}
func (t *thirdServer) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (*third.CompleteMultipartUploadResp, error) {
if err := checkUploadName(ctx, req.Name); err != nil {
return nil, err
}
result, err := t.s3dataBase.CompleteMultipartUpload(ctx, req.UploadID, req.Parts)
if err != nil {
return nil, err
}
obj := &relation.ObjectModel{
Name: req.Name,
UserID: mcontext.GetOpUserID(ctx),
Hash: result.Hash,
Key: result.Key,
Size: result.Size,
ContentType: req.ContentType,
Cause: req.Cause,
CreateTime: time.Now(),
}
if err := t.s3dataBase.SetObject(ctx, obj); err != nil {
return nil, err
}
return &third.CompleteMultipartUploadResp{
Url: t.apiAddress(obj.Key),
}, nil
}
func (t *thirdServer) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
return t.s3dataBase.GetHashInfo(ctx, req)
func (t *thirdServer) AccessURL(ctx context.Context, req *third.AccessURLReq) (*third.AccessURLResp, error) {
expireTime, rawURL, err := t.s3dataBase.AccessURL(ctx, req.Name, t.defaultExpire)
if err != nil {
return nil, err
}
return &third.AccessURLResp{
Url: rawURL,
ExpireTime: expireTime.UnixMilli(),
}, nil
}
func (t *thirdServer) CleanObject(ctx context.Context, now time.Time) {
t.s3dataBase.CleanExpirationObject(ctx, now)
func (t *thirdServer) apiAddress(name string) string {
return t.apiURL + name
}

@ -2,12 +2,17 @@ package third
import (
"context"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cos"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/minio"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/oss"
"net/url"
"time"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/relation"
relationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/discoveryregistry"
@ -17,22 +22,17 @@ import (
)
func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error {
u, err := url.Parse(config.Config.Object.ApiURL)
if err != nil {
return err
apiURL := config.Config.Object.ApiURL
if apiURL == "" {
return fmt.Errorf("api url is empty")
}
rdb, err := cache.NewRedis()
if err != nil {
if _, err := url.Parse(config.Config.Object.ApiURL); err != nil {
return err
}
// 根据配置文件策略选择 oss 方式
enable := config.Config.Object.Enable
var o obj.Interface
if enable == "minio" {
o, err = obj.NewMinioInterface()
} else if enable == "tencent" {
o, err = obj.NewCosClient()
if apiURL[len(apiURL)-1] != '/' {
apiURL += "/"
}
rdb, err := cache.NewRedis()
if err != nil {
return err
}
@ -40,21 +40,41 @@ func Start(client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) e
if err != nil {
return err
}
if err := db.AutoMigrate(&relationTb.ObjectHashModel{}, &relationTb.ObjectInfoModel{}, &relationTb.ObjectPutModel{}); err != nil {
if err := db.AutoMigrate(&relationTb.ObjectModel{}); err != nil {
return err
}
// 根据配置文件策略选择 oss 方式
enable := config.Config.Object.Enable
var o s3.Interface
switch config.Config.Object.Enable {
case "minio":
o, err = minio.NewMinio()
case "cos":
o, err = cos.NewCos()
case "oss":
o, err = oss.NewOSS()
default:
err = fmt.Errorf("invalid object enable: %s", enable)
}
if err != nil {
return err
}
third.RegisterThirdServer(server, &thirdServer{
apiURL: apiURL,
thirdDatabase: controller.NewThirdDatabase(cache.NewMsgCacheModel(rdb)),
userRpcClient: rpcclient.NewUserRpcClient(client),
s3dataBase: controller.NewS3Database(o, relation.NewObjectHash(db), relation.NewObjectInfo(db), relation.NewObjectPut(db), u),
s3dataBase: controller.NewS3Database(o, relation.NewObjectInfo(db)),
defaultExpire: time.Hour * 24 * 7,
})
return nil
}
type thirdServer struct {
apiURL string
thirdDatabase controller.ThirdDatabase
s3dataBase controller.S3Database
userRpcClient rpcclient.UserRpcClient
defaultExpire time.Duration
}
func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) {

@ -0,0 +1,62 @@
package third
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/tokenverify"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"strings"
"unicode/utf8"
)
func toPbMapArray(m map[string][]string) map[string]*third.MapValues {
res := make(map[string]*third.MapValues)
for key := range m {
res[key] = &third.MapValues{
Values: m[key],
}
}
return res
}
func checkUploadName(ctx context.Context, name string) error {
if name == "" {
return errs.ErrArgs.Wrap("name is empty")
}
if name[0] == '/' {
return errs.ErrArgs.Wrap("name cannot start with `/`")
}
if err := checkValidObjectName(name); err != nil {
return errs.ErrArgs.Wrap(err.Error())
}
opUserID := mcontext.GetOpUserID(ctx)
if opUserID == "" {
return errs.ErrNoPermission.Wrap("opUserID is empty")
}
if !tokenverify.IsManagerUserID(opUserID) {
if !strings.HasPrefix(name, opUserID+"_") {
return errs.ErrNoPermission.Wrap(fmt.Sprintf("name must start with `%s_`", opUserID))
}
}
return nil
}
func checkValidObjectNamePrefix(objectName string) error {
if len(objectName) > 1024 {
return errors.New("object name cannot be longer than 1024 characters")
}
if !utf8.ValidString(objectName) {
return errors.New("object name with non UTF-8 strings are not supported")
}
return nil
}
func checkValidObjectName(objectName string) error {
if strings.TrimSpace(objectName) == "" {
return errors.New("object name cannot be empty")
}
return checkValidObjectNamePrefix(objectName)
}

@ -103,42 +103,26 @@ type config struct {
Enable string `yaml:"enable"`
ApiURL string `yaml:"apiURL"`
Minio struct {
TempBucket string `yaml:"tempBucket"`
DataBucket string `yaml:"dataBucket"`
Location string `yaml:"location"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
IsDistributedMod bool `yaml:"isDistributedMod"`
Bucket string `yaml:"bucket"`
Endpoint string `yaml:"endpoint"`
AccessKeyID string `yaml:"accessKeyID"`
SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"minio"`
Tencent struct {
AppID string `yaml:"appID"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
Cos struct {
BucketURL string `yaml:"bucketURL"`
SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"tencent"`
Ali struct {
RegionID string `yaml:"regionID"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
StsEndpoint string `yaml:"stsEndpoint"`
OssEndpoint string `yaml:"ossEndpoint"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
StsDurationSeconds int64 `yaml:"stsDurationSeconds"`
OssRoleArn string `yaml:"OssRoleArn"`
} `yaml:"ali"`
Aws struct {
Oss struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
BucketURL string `yaml:"bucketURL"`
AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"`
Region string `yaml:"region"`
Bucket string `yaml:"bucket"`
FinalHost string `yaml:"finalHost"`
RoleArn string `yaml:"roleArn"`
ExternalId string `yaml:"externalId"`
RoleSessionName string `yaml:"roleSessionName"`
} `yaml:"aws"`
SessionToken string `yaml:"sessionToken"`
} `yaml:"ali"`
} `yaml:"object"`
RpcPort struct {

@ -0,0 +1,73 @@
package controller
import "C"
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3/cont"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"time"
)
type S3Database interface {
PartLimit() *s3.PartLimit
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error)
InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error)
SetObject(ctx context.Context, info *relation.ObjectModel) error
}
func NewS3Database(s3 s3.Interface, obj relation.ObjectInfoModelInterface) S3Database {
return &s3Database{
s3: cont.New(s3),
obj: obj,
}
}
type s3Database struct {
s3 *cont.Controller
obj relation.ObjectInfoModelInterface
}
func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) {
return s.s3.PartSize(ctx, size)
}
func (s *s3Database) PartLimit() *s3.PartLimit {
return s.s3.PartLimit()
}
func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
return s.s3.AuthSign(ctx, uploadID, partNumbers)
}
func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) {
return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts)
}
func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) {
return s.s3.CompleteUpload(ctx, uploadID, parts)
}
func (s *s3Database) SetObject(ctx context.Context, info *relation.ObjectModel) error {
return s.obj.SetObject(ctx, info)
}
func (s *s3Database) AccessURL(ctx context.Context, name string, expire time.Duration) (time.Time, string, error) {
obj, err := s.obj.Take(ctx, name)
if err != nil {
return time.Time{}, "", err
}
opt := &s3.AccessURLOption{
ContentType: obj.ContentType,
ContentDisposition: `attachment; filename=` + obj.Name,
}
expireTime := time.Now().Add(expire)
rawURL, err := s.s3.AccessURL(ctx, obj.Key, expire, opt)
if err != nil {
return time.Time{}, "", err
}
return expireTime, rawURL, nil
}

@ -1,538 +0,0 @@
package controller
import "C"
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/obj"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/log"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/google/uuid"
"io"
"net/url"
"path"
"strconv"
"time"
)
const (
hashPrefix = "hash"
tempPrefix = "temp"
fragmentPrefix = "fragment_"
urlsName = "urls.json"
)
type S3Database interface {
ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error)
GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error)
ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (*third.ConfirmPutResp, error)
GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error)
GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error)
CleanExpirationObject(ctx context.Context, t time.Time)
}
func NewS3Database(obj obj.Interface, hash relation.ObjectHashModelInterface, info relation.ObjectInfoModelInterface, put relation.ObjectPutModelInterface, url *url.URL) S3Database {
return &s3Database{
url: url,
obj: obj,
hash: hash,
info: info,
put: put,
}
}
type s3Database struct {
url *url.URL
obj obj.Interface
hash relation.ObjectHashModelInterface
info relation.ObjectInfoModelInterface
put relation.ObjectPutModelInterface
}
// today 今天的日期
func (c *s3Database) today() string {
return time.Now().Format("20060102")
}
// fragmentName 根据序号生成文件名
func (c *s3Database) fragmentName(index int) string {
return fragmentPrefix + strconv.Itoa(index+1)
}
// getFragmentNum 获取分片大小和分片数量
func (c *s3Database) getFragmentNum(fragmentSize int64, objectSize int64) (int64, int) {
if size := c.obj.MinFragmentSize(); fragmentSize < size {
fragmentSize = size
}
if fragmentSize <= 0 || objectSize <= fragmentSize {
return objectSize, 1
} else {
num := int(objectSize / fragmentSize)
if objectSize%fragmentSize > 0 {
num++
}
if n := c.obj.MaxFragmentNum(); num > n {
num = n
}
return fragmentSize, num
}
}
func (c *s3Database) CheckHash(hash string) error {
val, err := hex.DecodeString(hash)
if err != nil {
return err
}
if len(val) != md5.Size {
return errs.ErrArgs.Wrap("invalid hash")
}
return nil
}
func (c *s3Database) urlName(name string) string {
u := url.URL{
Scheme: c.url.Scheme,
Opaque: c.url.Opaque,
User: c.url.User,
Host: c.url.Host,
Path: c.url.Path,
RawPath: c.url.RawPath,
ForceQuery: c.url.ForceQuery,
RawQuery: c.url.RawQuery,
Fragment: c.url.Fragment,
RawFragment: c.url.RawFragment,
}
v := make(url.Values, 1)
v.Set("name", name)
u.RawQuery = v.Encode()
return u.String()
}
func (c *s3Database) UUID() string {
return uuid.New().String()
}
func (c *s3Database) HashName(hash string) string {
return path.Join(hashPrefix, hash+"_"+c.today()+"_"+c.UUID())
}
func (c *s3Database) isNotFound(err error) bool {
return relation.IsNotFound(err)
}
func (c *s3Database) ApplyPut(ctx context.Context, req *third.ApplyPutReq) (*third.ApplyPutResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
if err := c.obj.CheckName(req.Name); err != nil {
return nil, err
}
if req.ValidTime != 0 && req.ValidTime <= time.Now().UnixMilli() {
return nil, errors.New("invalid ValidTime")
}
var expirationTime *time.Time
if req.ValidTime != 0 {
expirationTime = utils.ToPtr(time.UnixMilli(req.ValidTime))
}
if hash, err := c.hash.Take(ctx, req.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: req.Name,
Hash: hash.Hash,
ValidTime: expirationTime,
ContentType: req.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
return &third.ApplyPutResp{Url: c.urlName(o.Name)}, nil // 服务器已存在
} else if !c.isNotFound(err) {
return nil, err
}
// 新上传
var fragmentNum int
const effective = time.Hour * 24 * 2
req.FragmentSize, fragmentNum = c.getFragmentNum(req.FragmentSize, req.Size)
put := relation.ObjectPutModel{
PutID: req.PutID,
Hash: req.Hash,
Name: req.Name,
ObjectSize: req.Size,
ContentType: req.ContentType,
FragmentSize: req.FragmentSize,
ValidTime: expirationTime,
EffectiveTime: time.Now().Add(effective),
}
if put.PutID == "" {
put.PutID = c.UUID()
}
if v, err := c.put.Take(ctx, put.PutID); err == nil {
now := time.Now().UnixMilli()
if v.EffectiveTime.UnixMilli() <= now {
if err := c.put.DelPut(ctx, []string{v.PutID}); err != nil {
return nil, err
}
} else {
return nil, errs.ErrDuplicateKey.Wrap(fmt.Sprintf("duplicate put id %s", put.PutID))
}
} else if !c.isNotFound(err) {
return nil, err
}
put.Path = path.Join(tempPrefix, c.today(), req.Hash, put.PutID)
putURLs := make([]string, 0, fragmentNum)
for i := 0; i < fragmentNum; i++ {
url, err := c.obj.PresignedPutURL(ctx, &obj.ApplyPutArgs{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, c.fragmentName(i)),
Effective: effective,
MaxObjectSize: req.FragmentSize,
})
if err != nil {
return nil, err
}
putURLs = append(putURLs, url)
}
urlsJsonData, err := json.Marshal(putURLs)
if err != nil {
return nil, err
}
t := md5.Sum(urlsJsonData)
put.PutURLsHash = hex.EncodeToString(t[:])
_, err = c.obj.PutObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(put.Path, urlsName)}, bytes.NewReader(urlsJsonData), int64(len(urlsJsonData)))
if err != nil {
return nil, err
}
put.CreateTime = time.Now()
if err := c.put.Create(ctx, []*relation.ObjectPutModel{&put}); err != nil {
return nil, err
}
return &third.ApplyPutResp{
PutID: put.PutID,
FragmentSize: put.FragmentSize,
PutURLs: putURLs,
ValidTime: put.EffectiveTime.UnixMilli(),
}, nil
}
func (c *s3Database) GetPut(ctx context.Context, req *third.GetPutReq) (*third.GetPutResp, error) {
up, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
reader, err := c.obj.GetObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: path.Join(up.Path, urlsName)})
if err != nil {
return nil, err
}
urlsData, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
t := md5.Sum(urlsData)
if h := hex.EncodeToString(t[:]); h != up.PutURLsHash {
return nil, fmt.Errorf("invalid put urls hash %s %s", h, up.PutURLsHash)
}
var urls []string
if err := json.Unmarshal(urlsData, &urls); err != nil {
return nil, err
}
_, fragmentNum := c.getFragmentNum(up.FragmentSize, up.ObjectSize)
if len(urls) != fragmentNum {
return nil, fmt.Errorf("invalid urls length %d fragment %d", len(urls), fragmentNum)
}
fragments := make([]*third.GetPutFragment, fragmentNum)
for i := 0; i < fragmentNum; i++ {
name := path.Join(up.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
if c.obj.IsNotFound(err) {
fragments[i] = &third.GetPutFragment{Url: urls[i]}
continue
}
return nil, err
}
fragments[i] = &third.GetPutFragment{Size: o.Size, Hash: o.Hash, Url: urls[i]}
}
var validTime int64
if up.ValidTime != nil {
validTime = up.ValidTime.UnixMilli()
}
return &third.GetPutResp{
FragmentSize: up.FragmentSize,
Size: up.ObjectSize,
Name: up.Name,
Hash: up.Hash,
Fragments: fragments,
PutURLsHash: up.PutURLsHash,
ContentType: up.ContentType,
ValidTime: validTime,
}, nil
}
func (c *s3Database) ConfirmPut(ctx context.Context, req *third.ConfirmPutReq) (_ *third.ConfirmPutResp, _err error) {
put, err := c.put.Take(ctx, req.PutID)
if err != nil {
return nil, err
}
_, pack := c.getFragmentNum(put.FragmentSize, put.ObjectSize)
defer func() {
if _err == nil {
// 清理上传的碎片
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "deleteObject failed", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}
}()
now := time.Now().UnixMilli()
if put.EffectiveTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("put expired")
}
if put.ValidTime != nil && put.ValidTime.UnixMilli() < now {
return nil, errs.ErrFileUploadedExpired.Wrap("object expired")
}
if hash, err := c.hash.Take(ctx, put.Hash, c.obj.Name()); err == nil {
o := relation.ObjectInfoModel{
Name: put.Name,
Hash: hash.Hash,
ValidTime: put.ValidTime,
ContentType: put.ContentType,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, &o); err != nil {
return nil, err
}
defer func() {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: put.Path,
})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
}
}()
// 服务端已存在
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
} else if !c.isNotFound(err) {
return nil, err
}
src := make([]obj.BucketObject, pack)
for i := 0; i < pack; i++ {
name := path.Join(put.Path, c.fragmentName(i))
o, err := c.obj.GetObjectInfo(ctx, &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
})
if err != nil {
return nil, err
}
if i+1 == pack { // 最后一个
size := put.ObjectSize - put.FragmentSize*int64(i)
if size != o.Size {
return nil, fmt.Errorf("last fragment %d size %d not equal to %d hash %s", i, o.Size, size, o.Hash)
}
} else {
if o.Size != put.FragmentSize {
return nil, fmt.Errorf("fragment %d size %d not equal to %d hash %s", i, o.Size, put.FragmentSize, o.Hash)
}
}
src[i] = obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: name,
}
}
dst := &obj.BucketObject{
Bucket: c.obj.DataBucket(),
Name: c.HashName(put.Hash),
}
if len(src) == 1 { // 未分片直接触发copy
// 检查数据完整性,避免脏数据
o, err := c.obj.GetObjectInfo(ctx, &src[0])
if err != nil {
return nil, err
}
if put.ObjectSize != o.Size {
return nil, fmt.Errorf("size mismatching should %d reality %d", put.ObjectSize, o.Size)
}
if put.Hash != o.Hash {
return nil, fmt.Errorf("hash mismatching should %s reality %s", put.Hash, o.Hash)
}
if err := c.obj.CopyObject(ctx, &src[0], dst); err != nil {
return nil, err
}
} else {
tempBucket := &obj.BucketObject{
Bucket: c.obj.TempBucket(),
Name: path.Join(put.Path, "merge_"+c.UUID()),
}
defer func() { // 清理合成的文件
if err := c.obj.DeleteObject(ctx, tempBucket); err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", tempBucket.Bucket, "Path", tempBucket.Name)
}
}()
err := c.obj.ComposeObject(ctx, src, tempBucket)
if err != nil {
return nil, err
}
info, err := c.obj.GetObjectInfo(ctx, tempBucket)
if err != nil {
return nil, err
}
if put.ObjectSize != info.Size {
return nil, fmt.Errorf("size mismatch should %d reality %d", put.ObjectSize, info.Size)
}
if put.Hash != info.Hash {
return nil, fmt.Errorf("hash mismatch should %s reality %s", put.Hash, info.Hash)
}
if err := c.obj.CopyObject(ctx, tempBucket, dst); err != nil {
return nil, err
}
}
h := &relation.ObjectHashModel{
Hash: put.Hash,
Engine: c.obj.Name(),
Size: put.ObjectSize,
Bucket: c.obj.DataBucket(),
Name: dst.Name,
CreateTime: time.Now(),
}
if err := c.hash.Create(ctx, []*relation.ObjectHashModel{h}); err != nil {
return nil, err
}
o := &relation.ObjectInfoModel{
Name: put.Name,
Hash: put.Hash,
ContentType: put.ContentType,
ValidTime: put.ValidTime,
CreateTime: time.Now(),
}
if err := c.info.SetObject(ctx, o); err != nil {
return nil, err
}
if err := c.put.DelPut(ctx, []string{put.PutID}); err != nil {
log.ZError(ctx, "DelPut", err, "PutID", put.PutID)
}
return &third.ConfirmPutResp{
Url: c.urlName(o.Name),
}, nil
}
func (c *s3Database) GetUrl(ctx context.Context, req *third.GetUrlReq) (*third.GetUrlResp, error) {
info, err := c.info.Take(ctx, req.Name)
if err != nil {
return nil, err
}
if info.ValidTime != nil && info.ValidTime.Before(time.Now()) {
return nil, errs.ErrRecordNotFound.Wrap("object expired")
}
hash, err := c.hash.Take(ctx, info.Hash, c.obj.Name())
if err != nil {
return nil, err
}
opt := obj.HeaderOption{ContentType: info.ContentType}
if req.Attachment {
opt.Filename = info.Name
}
u, err := c.obj.PresignedGetURL(ctx, hash.Bucket, hash.Name, time.Duration(req.Expires)*time.Millisecond, &opt)
if err != nil {
return nil, err
}
return &third.GetUrlResp{
Url: u,
Size: hash.Size,
Hash: hash.Hash,
}, nil
}
func (c *s3Database) CleanExpirationObject(ctx context.Context, t time.Time) {
// 清理上传产生的临时文件
c.cleanPutTemp(ctx, t, 10)
// 清理hash引用全过期的文件
c.cleanExpirationObject(ctx, t)
// 清理没有引用的hash对象
c.clearNoCitation(ctx, c.obj.Name(), 10)
}
func (c *s3Database) cleanPutTemp(ctx context.Context, t time.Time, num int) {
for {
puts, err := c.put.FindExpirationPut(ctx, t, num)
if err != nil {
log.ZError(ctx, "FindExpirationPut", err, "Time", t, "Num", num)
return
}
if len(puts) == 0 {
return
}
for _, put := range puts {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: c.obj.TempBucket(), Name: put.Path})
if err != nil {
log.ZError(ctx, "DeleteObject", err, "Bucket", c.obj.TempBucket(), "Path", put.Path)
return
}
}
ids := utils.Slice(puts, func(e *relation.ObjectPutModel) string { return e.PutID })
err = c.put.DelPut(ctx, ids)
if err != nil {
log.ZError(ctx, "DelPut", err, "PutID", ids)
return
}
}
}
func (c *s3Database) cleanExpirationObject(ctx context.Context, t time.Time) {
err := c.info.DeleteExpiration(ctx, t)
if err != nil {
log.ZError(ctx, "DeleteExpiration", err, "Time", t)
}
}
func (c *s3Database) clearNoCitation(ctx context.Context, engine string, limit int) {
for {
list, err := c.hash.DeleteNoCitation(ctx, engine, limit)
if err != nil {
log.ZError(ctx, "DeleteNoCitation", err, "Engine", engine, "Limit", limit)
return
}
if len(list) == 0 {
return
}
var hasErr bool
for _, h := range list {
err := c.obj.DeleteObject(ctx, &obj.BucketObject{Bucket: h.Bucket, Name: h.Name})
if err != nil {
hasErr = true
log.ZError(ctx, "DeleteObject", err, "Bucket", h.Bucket, "Path", h.Name)
continue
}
}
if hasErr {
return
}
}
}
func (c *s3Database) GetHashInfo(ctx context.Context, req *third.GetHashInfoReq) (*third.GetHashInfoResp, error) {
if err := c.CheckHash(req.Hash); err != nil {
return nil, err
}
o, err := c.hash.Take(ctx, req.Hash, c.obj.Name())
if err != nil {
return nil, err
}
return &third.GetHashInfoResp{
Hash: o.Hash,
Size: o.Size,
}, nil
}

@ -1,231 +0,0 @@
package obj
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
"io"
"net/http"
"net/url"
"time"
)
func NewMinioInterface() (Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, fmt.Errorf("minio endpoint parse %w", err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, fmt.Errorf("invalid minio endpoint scheme %s", u.Scheme)
}
client, err := minio.New(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
Secure: u.Scheme == "https",
})
if err != nil {
return nil, fmt.Errorf("minio new client %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*20)
defer cancel()
initBucket := func(ctx context.Context) error {
for _, bucket := range utils.Distinct([]string{conf.TempBucket, conf.DataBucket}) {
exists, err := client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio bucket %s exists %w", bucket, err)
}
if exists {
continue
}
opt := minio.MakeBucketOptions{
Region: conf.Location,
ObjectLocking: conf.IsDistributedMod,
}
if err := client.MakeBucket(ctx, bucket, opt); err != nil {
return fmt.Errorf("minio make bucket %s %w", bucket, err)
}
}
return nil
}
if err := initBucket(ctx); err != nil {
fmt.Println("minio init error:", err)
}
return &minioImpl{
client: client,
tempBucket: conf.TempBucket,
dataBucket: conf.DataBucket,
}, nil
}
type minioImpl struct {
tempBucket string // 上传桶
dataBucket string // 永久桶
urlstr string // 访问地址
client *minio.Client
}
func (m *minioImpl) Name() string {
return "minio"
}
func (m *minioImpl) MinFragmentSize() int64 {
return 1024 * 1024 * 5 // 每个分片最小大小 minio.absMinPartSize
}
func (m *minioImpl) MaxFragmentNum() int {
return 1000 // 最大分片数量 minio.maxPartsCount
}
func (m *minioImpl) MinExpirationTime() time.Duration {
return time.Hour * 24
}
func (m *minioImpl) TempBucket() string {
return m.tempBucket
}
func (m *minioImpl) DataBucket() string {
return m.dataBucket
}
func (m *minioImpl) PresignedGetURL(ctx context.Context, bucket string, name string, expires time.Duration, opt *HeaderOption) (string, error) {
var reqParams url.Values
if opt != nil {
reqParams = make(url.Values)
if opt.ContentType != "" {
reqParams.Set("response-content-type", opt.ContentType)
}
if opt.Filename != "" {
reqParams.Set("response-content-disposition", "attachment;filename="+opt.Filename)
}
}
u, err := m.client.PresignedGetObject(ctx, bucket, name, expires, reqParams)
if err != nil {
return "", err
}
return u.String(), nil
}
func (m *minioImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
if args.Effective <= 0 {
return "", errors.New("EffectiveTime <= 0")
}
_, err := m.GetObjectInfo(ctx, &BucketObject{
Bucket: m.tempBucket,
Name: args.Name,
})
if err == nil {
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
} else if !m.IsNotFound(err) {
return "", err
}
u, err := m.client.PresignedPutObject(ctx, m.tempBucket, args.Name, args.Effective)
if err != nil {
return "", fmt.Errorf("minio apply error: %w", err)
}
return u.String(), nil
}
func (m *minioImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
info, err := m.client.StatObject(ctx, args.Bucket, args.Name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: info.Size,
Hash: info.ETag,
}, nil
}
func (m *minioImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name,
}, minio.CopySrcOptions{
Bucket: src.Bucket,
Object: src.Name,
})
return err
}
func (m *minioImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
return m.client.RemoveObject(ctx, info.Bucket, info.Name, minio.RemoveObjectOptions{})
}
func (m *minioImpl) MoveObjectInfo(ctx context.Context, src *BucketObject, dst *BucketObject) error {
if err := m.CopyObject(ctx, src, dst); err != nil {
return err
}
return m.DeleteObject(ctx, src)
}
func (m *minioImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
destOptions := minio.CopyDestOptions{
Bucket: dst.Bucket,
Object: dst.Name + ".temp",
}
sources := make([]minio.CopySrcOptions, len(src))
for i, s := range src {
sources[i] = minio.CopySrcOptions{
Bucket: s.Bucket,
Object: s.Name,
}
}
_, err := m.client.ComposeObject(ctx, destOptions, sources...)
if err != nil {
return err
}
return m.MoveObjectInfo(ctx, &BucketObject{
Bucket: destOptions.Bucket,
Name: destOptions.Object,
}, &BucketObject{
Bucket: dst.Bucket,
Name: dst.Name,
})
}
func (m *minioImpl) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *minioImpl) PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error) {
update, err := m.client.PutObject(ctx, info.Bucket, info.Name, reader, size, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &ObjectInfo{
Size: update.Size,
Hash: update.ETag,
}, nil
}
func (m *minioImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
object, err := m.client.GetObject(ctx, info.Bucket, info.Name, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
stat, err := object.Stat()
if err != nil {
return nil, err
}
return NewSizeReader(object, stat.Size), nil
}
func (m *minioImpl) CheckName(name string) error {
return s3utils.CheckValidObjectName(name)
}

@ -1,90 +0,0 @@
package obj
import (
"context"
"io"
"net/http"
"time"
)
type BucketObject struct {
Bucket string `json:"bucket"`
Name string `json:"name"`
}
type ApplyPutArgs struct {
Bucket string
Name string
Effective time.Duration // 申请有效时间
Header http.Header // header
MaxObjectSize int64
}
type HeaderOption struct {
ContentType string
Filename string
}
type ObjectInfo struct {
Size int64
Hash string
}
type SizeReader interface {
io.ReadCloser
Size() int64
}
func NewSizeReader(r io.ReadCloser, size int64) SizeReader {
if r == nil {
return nil
}
return &sizeReader{
size: size,
ReadCloser: r,
}
}
type sizeReader struct {
size int64
io.ReadCloser
}
func (r *sizeReader) Size() int64 {
return r.size
}
type Interface interface {
// Name 存储名字
Name() string
// MinFragmentSize 最小允许的分片大小
MinFragmentSize() int64
// MaxFragmentNum 最大允许的分片数量
MaxFragmentNum() int
// MinExpirationTime 最小过期时间
MinExpirationTime() time.Duration
// TempBucket 临时桶名,用于上传
TempBucket() string
// DataBucket 永久存储的桶名
DataBucket() string
// PresignedGetURL 预签名授权 通过桶名和对象名返回URL
PresignedGetURL(ctx context.Context, bucket string, name string, expires time.Duration, opt *HeaderOption) (string, error)
// PresignedPutURL 预签名授权 申请上传,返回PUT的上传地址
PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error)
// GetObjectInfo 获取对象信息
GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error)
// CopyObject 复制对象
CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error
// DeleteObject 删除对象(不存在返回nil)
DeleteObject(ctx context.Context, info *BucketObject) error
// ComposeObject 合并对象
ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error
// IsNotFound 判断是不是不存在导致的错误
IsNotFound(err error) bool
// CheckName 检查名字是否可用
CheckName(name string) error
// PutObject 上传文件
PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error)
// GetObject 下载文件
GetObject(ctx context.Context, info *BucketObject) (SizeReader, error)
}

@ -1,212 +0,0 @@
package obj
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/tencentyun/cos-go-sdk-v5"
)
var conf = config.Config.Object.Tencent
func create_url(bucket, region, source string) string {
return fmt.Sprintf("https://%s.cos.%s.myqcloud.com/%s", bucket, region, source)
}
func NewCosClient() (Interface, error) {
u, err := url.Parse(create_url(conf.Bucket, conf.Region, ""))
if err != nil {
return nil, fmt.Errorf("tencent cos url parse %w", err)
}
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.SecretID, // 用户的 SecretId建议使用子账号密钥授权遵循最小权限指引降低使用风险。子账号密钥获取可参考 https://cloud.tencent.com/document/product/598/37140
SecretKey: conf.SecretKey, // 用户的 SecretKey建议使用子账号密钥授权遵循最小权限指引降低使用风险。子账号密钥获取可参考 https://cloud.tencent.com/document/product/598/37140
},
})
return &cosImpl{
client: c,
tempBucket: conf.Bucket,
}, err
}
type cosImpl struct {
tempBucket string // 上传桶
//dataBucket string // 永久桶
urlstr string // 访问地址
client *cos.Client
}
func (c *cosImpl) Name() string {
return "tx_oss cos"
}
func (c *cosImpl) MinFragmentSize() int64 {
return 1024 * 1024 * 5 // 每个分片最小大小 tx_oss.absMinPartSize
}
func (c *cosImpl) MaxFragmentNum() int {
return 1000 // 最大分片数量 tx_oss.maxPartsCount
}
func (c *cosImpl) MinExpirationTime() time.Duration {
return time.Hour * 24
}
func (c *cosImpl) TempBucket() string {
return c.tempBucket
}
func (c *cosImpl) DataBucket() string {
//return c.dataBucket
return ""
}
func (c *cosImpl) PresignedGetURL(ctx context.Context, bucket string, name string, expires time.Duration, opt *HeaderOption) (string, error) {
// 参考文档https://cloud.tencent.com/document/product/436/14116
// 获取对象访问 URL用于匿名下载和分发
presignedGetURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, conf.SecretID, conf.SecretKey, time.Hour, nil)
if err != nil {
return "", err
}
return presignedGetURL.String(), nil
}
func (c *cosImpl) PresignedPutURL(ctx context.Context, args *ApplyPutArgs) (string, error) {
// 参考文档https://cloud.tencent.com/document/product/436/14114
if args.Effective <= 0 {
return "", errors.New("EffectiveTime <= 0")
}
_, err := c.GetObjectInfo(ctx, &BucketObject{
Bucket: c.tempBucket,
Name: args.Name,
})
if err == nil {
return "", fmt.Errorf("minio bucket %s name %s already exists", args.Bucket, args.Name)
} else if !c.IsNotFound(err) {
return "", err
}
// 获取预签名 URL
presignedPutURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, args.Name, conf.SecretID, conf.SecretKey, time.Hour, nil)
if err != nil {
return "", fmt.Errorf("minio apply error: %w", err)
}
return presignedPutURL.String(), nil
}
func (c *cosImpl) GetObjectInfo(ctx context.Context, args *BucketObject) (*ObjectInfo, error) {
// https://cloud.tencent.com/document/product/436/7745
// 新增参数 id 代表指定版本,如果不指定,默认查询对象最新版本
head, err := c.client.Object.Head(ctx, args.Name, nil, "")
if err != nil {
return nil, err
}
size, _ := strconv.ParseInt(head.Header.Get("Content-Length"), 10, 64)
return &ObjectInfo{
Size: size,
Hash: head.Header.Get("ETag"),
}, nil
}
func (c *cosImpl) CopyObject(ctx context.Context, src *BucketObject, dst *BucketObject) error {
srcURL := create_url(src.Bucket, conf.Region, src.Name)
_, _, err := c.client.Object.Copy(ctx, dst.Name, srcURL, nil)
if err == nil {
_, err = c.client.Object.Delete(ctx, srcURL, nil)
if err != nil {
// Error
}
}
return err
}
func (c *cosImpl) DeleteObject(ctx context.Context, info *BucketObject) error {
_, err := c.client.Object.Delete(ctx, info.Name)
return err
}
func (c *cosImpl) ComposeObject(ctx context.Context, src []BucketObject, dst *BucketObject) error {
//TODO implement me
panic("implement me")
}
func (c *cosImpl) IsNotFound(err error) bool {
ok, err := c.client.Object.IsExist(context.Background(), c.tempBucket)
if err == nil && ok {
fmt.Printf("object exists\n")
return true
} else if err != nil {
fmt.Printf("head object failed: %v\n", err)
return false
} else {
fmt.Printf("object does not exist\n")
return false
}
}
func (c *cosImpl) CheckName(name string) error {
return s3utils.CheckValidObjectName(name)
}
func (c *cosImpl) PutObject(ctx context.Context, info *BucketObject, reader io.Reader, size int64) (*ObjectInfo, error) {
/*// 采用高级接口, 无需用户指定 size
update, _, err := c.client.Object.Upload(
ctx, info.Name, info.Bucket, nil,
)
if err != nil {
return nil, err
}
return &ObjectInfo{
Hash: update.ETag,
}, nil*/
// Case1 使用 Put 上传对象
opt := &cos.ObjectPutOptions{
ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
ContentType: "text/html",
},
ACLHeaderOptions: &cos.ACLHeaderOptions{
// 如果不是必要操作,建议上传文件时不要给单个文件设置权限,避免达到限制。若不设置默认继承桶的权限。
XCosACL: "private",
},
}
resp, err := c.client.Object.Put(ctx, info.Name, reader, opt)
if err != nil {
return nil, err
}
return &ObjectInfo{
Hash: resp.Header.Get("ETag"),
}, nil
}
func (c *cosImpl) GetObject(ctx context.Context, info *BucketObject) (SizeReader, error) {
opt := &cos.MultiDownloadOptions{
ThreadPoolSize: 5,
}
update, err := c.client.Object.Download(
ctx, info.Name, info.Bucket, opt,
)
if err != nil {
return nil, err
}
size, _ := strconv.ParseInt(update.Header.Get("Content-Length"), 10, 64)
body := update.Body
if body == nil {
return nil, errors.New("response body is nil")
}
readCloser, ok := body.(io.ReadCloser)
if !ok {
return nil, errors.New("failed to convert response to ReadCloser")
}
return NewSizeReader(readCloser, size), nil
}

@ -1,42 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
)
type ObjectHashGorm struct {
*MetaDB
}
func NewObjectHash(db *gorm.DB) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(db, &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) NewTx(tx any) relation.ObjectHashModelInterface {
return &ObjectHashGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectHashModel{}),
}
}
func (o *ObjectHashGorm) Take(ctx context.Context, hash string, engine string) (oh *relation.ObjectHashModel, err error) {
oh = &relation.ObjectHashModel{}
return oh, utils.Wrap1(o.DB.Where("hash = ? and engine = ?", hash, engine).Take(oh).Error)
}
func (o *ObjectHashGorm) Create(ctx context.Context, h []*relation.ObjectHashModel) (err error) {
return utils.Wrap1(o.DB.Create(h).Error)
}
func (o *ObjectHashGorm) DeleteNoCitation(ctx context.Context, engine string, num int) (list []*relation.ObjectHashModel, err error) {
err = o.DB.Table(relation.ObjectHashModelTableName, "as h").Select("h.*").
Joins("LEFT JOIN "+relation.ObjectInfoModelTableName+" as i ON h.hash = i.hash").
Where("h.engine = ? AND i.hash IS NULL", engine).
Limit(num).
Find(&list).Error
return list, utils.Wrap1(err)
}

@ -1,48 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectInfoModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectInfoModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
return errs.Wrap(err)
}
return errs.Wrap(o.DB.WithContext(ctx).Create(obj).Error)
//return errs.Wrap(o.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// if err := tx.Where("name = ?", obj.Name).Delete(&relation.ObjectInfoModel{}).Error; err != nil {
// return errs.Wrap(err)
// }
// return errs.Wrap(tx.Create(obj).Error)
//}))
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectInfoModel, err error) {
info = &relation.ObjectInfoModel{}
return info, utils.Wrap1(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}
func (o *ObjectInfoGorm) DeleteExpiration(ctx context.Context, expiration time.Time) (err error) {
return utils.Wrap1(o.DB.WithContext(ctx).Where("expiration_time IS NOT NULL AND expiration_time <= ?", expiration).Delete(&relation.ObjectInfoModel{}).Error)
}

@ -0,0 +1,36 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
"gorm.io/gorm"
)
type ObjectInfoGorm struct {
*MetaDB
}
func NewObjectInfo(db *gorm.DB) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(db, &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) NewTx(tx any) relation.ObjectInfoModelInterface {
return &ObjectInfoGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectModel{}),
}
}
func (o *ObjectInfoGorm) SetObject(ctx context.Context, obj *relation.ObjectModel) (err error) {
if err := o.DB.WithContext(ctx).Where("name = ?", obj.Name).FirstOrCreate(obj).Error; err != nil {
return errs.Wrap(err)
}
return nil
}
func (o *ObjectInfoGorm) Take(ctx context.Context, name string) (info *relation.ObjectModel, err error) {
info = &relation.ObjectModel{}
return info, errs.Wrap(o.DB.WithContext(ctx).Where("name = ?", name).Take(info).Error)
}

@ -1,47 +0,0 @@
package relation
import (
"context"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/relation"
"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
"gorm.io/gorm"
"time"
)
type ObjectPutGorm struct {
*MetaDB
}
func NewObjectPut(db *gorm.DB) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(db, &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) NewTx(tx any) relation.ObjectPutModelInterface {
return &ObjectPutGorm{
NewMetaDB(tx.(*gorm.DB), &relation.ObjectPutModel{}),
}
}
func (o *ObjectPutGorm) Create(ctx context.Context, m []*relation.ObjectPutModel) (err error) {
return utils.Wrap1(o.DB.Create(m).Error)
}
func (o *ObjectPutGorm) Take(ctx context.Context, putID string) (put *relation.ObjectPutModel, err error) {
put = &relation.ObjectPutModel{}
return put, utils.Wrap1(o.DB.Where("put_id = ?", putID).Take(put).Error)
}
func (o *ObjectPutGorm) SetCompleted(ctx context.Context, putID string) (err error) {
return utils.Wrap1(o.DB.Model(&relation.ObjectPutModel{}).Where("put_id = ?", putID).Update("complete", true).Error)
}
func (o *ObjectPutGorm) FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) (list []*relation.ObjectPutModel, err error) {
err = o.DB.Where("effective_time <= ?", expirationTime).Limit(num).Find(&list).Error
return list, utils.Wrap1(err)
}
func (o *ObjectPutGorm) DelPut(ctx context.Context, ids []string) (err error) {
return utils.Wrap1(o.DB.Where("put_id IN ?", ids).Delete(&relation.ObjectPutModel{}).Error)
}

@ -0,0 +1,8 @@
package cont
const (
hashPath = "openim/data/hash/"
tempPath = "openim/temp/"
UploadTypeMultipart = 1 // 分片上传
UploadTypePresigned = 2 // 预签名上传
)

@ -0,0 +1,237 @@
package cont
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/google/uuid"
"path"
"strings"
"time"
)
func New(impl s3.Interface) *Controller {
return &Controller{impl: impl}
}
type Controller struct {
impl s3.Interface
}
func (c *Controller) HashPath(md5 string) string {
return path.Join(hashPath, md5)
}
func (c *Controller) NowPath() string {
now := time.Now()
return path.Join(
fmt.Sprintf("%04d", now.Year()),
fmt.Sprintf("%02d", now.Month()),
fmt.Sprintf("%02d", now.Day()),
fmt.Sprintf("%02d", now.Hour()),
fmt.Sprintf("%02d", now.Minute()),
fmt.Sprintf("%02d", now.Second()),
)
}
func (c *Controller) UUID() string {
id := uuid.New()
return hex.EncodeToString(id[:])
}
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
return c.impl.PartSize(ctx, size)
}
func (c *Controller) PartLimit() *s3.PartLimit {
return c.impl.PartLimit()
}
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
return c.impl.StatObject(ctx, c.HashPath(hash))
}
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
if size < 0 {
return nil, errors.New("invalid size")
}
if hashBytes, err := hex.DecodeString(hash); err != nil {
return nil, err
} else if len(hashBytes) != md5.Size {
return nil, errors.New("invalid md5")
}
partSize, err := c.impl.PartSize(ctx, size)
if err != nil {
return nil, err
}
partNumber := int(size / partSize)
if size%partSize > 0 {
partNumber++
}
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
return nil, errors.New(fmt.Sprintf("too many parts: %d", partNumber))
}
if info, err := c.impl.StatObject(ctx, c.HashPath(hash)); err == nil {
return nil, &HashAlreadyExistsError{Object: info}
} else if !c.impl.IsNotFound(err) {
return nil, err
}
if size <= partSize {
// 预签名上传
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
if err != nil {
return nil, err
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypePresigned,
ID: "",
Key: key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: &s3.AuthSignResult{
Parts: []s3.SignPart{
{
PartNumber: 1,
URL: rawURL,
},
},
},
}, nil
} else {
// 分片上传
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
if err != nil {
return nil, err
}
if maxParts < 0 {
maxParts = partNumber
}
var authSign *s3.AuthSignResult
if maxParts > 0 {
partNumbers := make([]int, partNumber)
for i := 0; i < maxParts; i++ {
partNumbers[i] = i + 1
}
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
if err != nil {
return nil, err
}
}
return &InitiateUploadResult{
UploadID: newMultipartUploadID(multipartUploadID{
Type: UploadTypeMultipart,
ID: upload.UploadID,
Key: upload.Key,
Size: size,
Hash: hash,
}),
PartSize: partSize,
Sign: authSign,
}, nil
}
}
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, ","))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
fmt.Println("CompleteUpload sum:", hex.EncodeToString(md5Sum[:]), "upload hash:", upload.Hash)
return nil, errors.New("md5 mismatching")
}
if info, err := c.impl.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
return &UploadResult{
Key: info.Key,
Size: info.Size,
Hash: info.ETag,
}, nil
} else if !c.impl.IsNotFound(err) {
return nil, err
}
cleanObject := make(map[string]struct{})
defer func() {
for key := range cleanObject {
_ = c.impl.DeleteObject(ctx, key)
}
}()
var targetKey string
switch upload.Type {
case UploadTypeMultipart:
parts := make([]s3.Part, len(partHashs))
for i, part := range partHashs {
parts[i] = s3.Part{
PartNumber: i + 1,
ETag: part,
}
}
// todo: 验证大小
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
if err != nil {
return nil, err
}
targetKey = result.Key
case UploadTypePresigned:
uploadInfo, err := c.impl.StatObject(ctx, upload.Key)
if err != nil {
return nil, err
}
cleanObject[uploadInfo.Key] = struct{}{}
if uploadInfo.Size != upload.Size {
return nil, errors.New("upload size mismatching")
}
if uploadInfo.ETag != upload.Hash {
return nil, errors.New("upload md5 mismatching")
}
// 防止在这个时候,并发操作,导致文件被覆盖
copyInfo, err := c.impl.CopyObject(ctx, targetKey, upload.Key+"."+c.UUID())
if err != nil {
return nil, err
}
cleanObject[copyInfo.Key] = struct{}{}
if copyInfo.ETag != upload.Hash {
return nil, errors.New("copy md5 mismatching")
}
if _, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash)); err != nil {
return nil, err
}
targetKey = copyInfo.Key
default:
return nil, errors.New("invalid upload id type")
}
return &UploadResult{
Key: targetKey,
Size: upload.Size,
Hash: upload.Hash,
}, nil
}
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
upload, err := parseMultipartUploadID(uploadID)
if err != nil {
return nil, err
}
switch upload.Type {
case UploadTypeMultipart:
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
case UploadTypePresigned:
return nil, errors.New("presigned id not support auth sign")
default:
return nil, errors.New("invalid upload id type")
}
}
func (c *Controller) IsNotFound(err error) bool {
return c.impl.IsNotFound(err)
}
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
return c.impl.AccessURL(ctx, name, expire, opt)
}

@ -0,0 +1,14 @@
package cont
import (
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
)
type HashAlreadyExistsError struct {
Object *s3.ObjectInfo
}
func (e *HashAlreadyExistsError) Error() string {
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
}

@ -0,0 +1,35 @@
package cont
import (
"encoding/base64"
"encoding/json"
"fmt"
)
type multipartUploadID struct {
Type int `json:"a,omitempty"`
ID string `json:"b,omitempty"`
Key string `json:"c,omitempty"`
Size int64 `json:"d,omitempty"`
Hash string `json:"e,omitempty"`
}
func newMultipartUploadID(id multipartUploadID) string {
data, err := json.Marshal(id)
if err != nil {
panic(err)
}
return base64.StdEncoding.EncodeToString(data)
}
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
data, err := base64.StdEncoding.DecodeString(id)
if err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
var upload multipartUploadID
if err := json.Unmarshal(data, &upload); err != nil {
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
}
return &upload, nil
}

@ -0,0 +1,15 @@
package cont
import "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
type InitiateUploadResult struct {
UploadID string `json:"uploadID"` // 上传ID
PartSize int64 `json:"partSize"` // 分片大小
Sign *s3.AuthSignResult `json:"sign"` // 分片信息
}
type UploadResult struct {
Hash string `json:"hash"`
Size int64 `json:"size"`
Key string `json:"key"`
}

@ -0,0 +1,271 @@
package cos
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 1000
)
func NewCos() (s3.Interface, error) {
conf := config.Config.Object.Cos
u, err := url.Parse(conf.BucketURL)
if err != nil {
panic(err)
}
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.SecretID,
SecretKey: conf.SecretKey,
SessionToken: conf.SessionToken,
},
})
return &Cos{
copyURL: u.Host + "/",
client: client,
credential: client.GetCredential(),
}, nil
}
type Cos struct {
copyURL string
client *cos.Client
credential *cos.Credential
}
func (c *Cos) Engine() string {
return "tencent-cos"
}
func (c *Cos) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
opts := &cos.CompleteMultipartUploadOptions{
Parts: make([]cos.Object, len(parts)),
}
for i, part := range parts {
opts.Parts[i] = cos.Object{
PartNumber: part.PartNumber,
ETag: strings.ToLower(part.ETag),
}
}
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: result.ETag,
}, nil
}
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (c *Cos) ClientUploadPart(ctx context.Context, partSize int64, expire time.Duration, upload *s3.InitiateMultipartUploadResult) (*s3.MultipartUploadRequest, error) {
uri := c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(upload.Key)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, nil)
if err != nil {
return nil, err
}
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
return &s3.MultipartUploadRequest{
UploadID: upload.UploadID,
Bucket: upload.Bucket,
Key: upload.Key,
Method: req.Method,
URL: uri,
Query: url.Values{"uploadId": {upload.UploadID}},
Header: req.Header,
PartKey: "partNumber",
PartSize: partSize,
FirstPart: 1,
}, nil
}
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
if err != nil {
return nil, err
}
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
result.Header = req.Header
for i, partNumber := range partNumbers {
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
}
}
return &result, nil
}
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
_, err := c.client.Object.Delete(ctx, name)
return err
}
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
info, err := c.client.Object.Head(ctx, name, nil)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, _, err := c.client.Object.Copy(ctx, src, c.copyURL+dst, &cos.ObjectCopyOptions{})
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: result.ETag,
}, nil
}
func (c *Cos) IsNotFound(err error) bool {
switch e := err.(type) {
case *cos.ErrorResponse:
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
return err
}
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
MaxParts: strconv.Itoa(maxParts),
PartNumberMarker: strconv.Itoa(partNumberMarker),
})
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
}
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.Parts {
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: lastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
reqParams := make(url.Values)
if opt != nil {
if opt.ContentType != "" {
reqParams.Set("Content-Type", opt.ContentType)
}
if opt.ContentDisposition != "" {
reqParams.Set("Content-Disposition", opt.ContentDisposition)
}
}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, reqParams)
if err != nil {
return "", err
}
return rawURL.String(), nil
}

@ -0,0 +1,250 @@
package minio
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/signer"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
unsignedPayload = "UNSIGNED-PAYLOAD"
)
const (
minPartSize = 1024 * 1024 * 5 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewMinio() (s3.Interface, error) {
conf := config.Config.Object.Minio
u, err := url.Parse(conf.Endpoint)
if err != nil {
return nil, err
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
Secure: u.Scheme == "https",
}
client, err := minio.New(u.Host, opts)
if err != nil {
return nil, err
}
return &Minio{
bucket: conf.Bucket,
bucketURL: conf.Endpoint + "/" + conf.Bucket + "/",
opts: opts,
core: &minio.Core{Client: client},
}, nil
}
type Minio struct {
bucket string
bucketURL string
opts *minio.Options
core *minio.Core
}
func (m *Minio) Engine() string {
return "minio"
}
func (m *Minio) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
Bucket: m.bucket,
Key: name,
UploadID: uploadID,
}, nil
}
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
minioParts := make([]minio.CompletePart, len(parts))
for i, part := range parts {
minioParts[i] = minio.CompletePart{
PartNumber: part.PartNumber,
ETag: strings.ToLower(part.ETag),
}
}
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: upload.Location,
Bucket: upload.Bucket,
Key: upload.Key,
ETag: strings.ToLower(upload.ETag),
}, nil
}
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
creds, err := m.opts.Creds.Get()
if err != nil {
return nil, err
}
result := s3.AuthSignResult{
URL: m.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
URL: request.URL.String(),
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
Header: request.Header,
}
}
return &result, nil
}
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire)
if err != nil {
return "", err
}
return rawURL.String(), nil
}
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
}
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &s3.ObjectInfo{
ETag: strings.ToLower(info.ETag),
Key: info.Key,
Size: info.Size,
LastModified: info.LastModified,
}, nil
}
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
Bucket: m.bucket,
Object: dst,
}, minio.CopySrcOptions{
Bucket: m.bucket,
Object: src,
})
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(result.ETag),
}, nil
}
func (m *Minio) IsNotFound(err error) bool {
if err == nil {
return false
}
switch e := err.(type) {
case minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *minio.ErrorResponse:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
}
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
NextPartNumberMarker: result.NextPartNumberMarker,
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
}
for i, part := range result.ObjectParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size,
}
}
return res, nil
}
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
reqParams := make(url.Values)
if opt != nil {
if opt.ContentType != "" {
reqParams.Set("Content-Type", opt.ContentType)
}
if opt.ContentDisposition != "" {
reqParams.Set("Content-Disposition", opt.ContentDisposition)
}
}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams)
if err != nil {
return "", err
}
return u.String(), nil
}

@ -0,0 +1,259 @@
package oss
import (
"context"
"errors"
"fmt"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
"github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
minPartSize = 1024 * 1024 * 1 // 1MB
maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
maxNumSize = 10000
)
func NewOSS() (s3.Interface, error) {
conf := config.Config.Object.Oss
if conf.BucketURL == "" {
return nil, errors.New("bucket url is empty")
}
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(conf.Bucket)
if err != nil {
return nil, err
}
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
conf.BucketURL += "/"
}
return &OSS{
bucketURL: conf.BucketURL,
bucket: bucket,
credentials: client.Config.GetCredentials(),
}, nil
}
type OSS struct {
bucketURL string
bucket *oss.Bucket
credentials oss.Credentials
}
func (o *OSS) Engine() string {
return "ali-oss"
}
func (o *OSS) PartLimit() *s3.PartLimit {
return &s3.PartLimit{
MinPartSize: minPartSize,
MaxPartSize: maxPartSize,
MaxNumSize: maxNumSize,
}
}
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
result, err := o.bucket.InitiateMultipartUpload(name)
if err != nil {
return nil, err
}
return &s3.InitiateMultipartUploadResult{
UploadID: result.UploadID,
Bucket: result.Bucket,
Key: result.Key,
}, nil
}
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
ossParts := make([]oss.UploadPart, len(parts))
for i, part := range parts {
ossParts[i] = oss.UploadPart{
PartNumber: part.PartNumber,
ETag: strings.ToUpper(part.ETag),
}
}
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Bucket: o.bucket.BucketName,
Key: name,
}, ossParts)
if err != nil {
return nil, err
}
return &s3.CompleteMultipartUploadResult{
Location: result.Location,
Bucket: result.Bucket,
Key: result.Key,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > maxPartSize*maxNumSize {
return 0, fmt.Errorf("size must be less than %db", maxPartSize*maxNumSize)
}
if size <= minPartSize*maxNumSize {
return minPartSize, nil
}
partSize := size / maxNumSize
if size%maxNumSize != 0 {
partSize++
}
return partSize, nil
}
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
result := s3.AuthSignResult{
URL: o.bucketURL + name,
Query: url.Values{"uploadId": {uploadID}},
Header: make(http.Header),
Parts: make([]s3.SignPart, len(partNumbers)),
}
for i, partNumber := range partNumbers {
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
if err != nil {
return nil, err
}
if o.credentials.GetSecurityToken() != "" {
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
}
request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, time.Now().UTC().Format(http.TimeFormat))
authorization := fmt.Sprintf(`OSS %s:%s`, o.credentials.GetAccessKeyID(), o.getSignedStr(request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID), o.credentials.GetAccessKeySecret()))
request.Header.Set(oss.HTTPHeaderAuthorization, authorization)
result.Parts[i] = s3.SignPart{
PartNumber: partNumber,
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
URL: request.URL.String(),
Header: request.Header,
}
}
return &result, nil
}
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
}
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
header, err := o.bucket.GetObjectMeta(name)
if err != nil {
return nil, err
}
res := &s3.ObjectInfo{Key: name}
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
return nil, errors.New("StatObject etag not found")
}
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
return nil, errors.New("StatObject content-length not found")
} else {
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
}
if res.Size < 0 {
return nil, errors.New("StatObject content-length must be greater than 0")
}
}
if lastModified := header.Get("Last-Modified"); lastModified == "" {
return nil, errors.New("StatObject last-modified not found")
} else {
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
if err != nil {
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
}
}
return res, nil
}
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
return o.bucket.DeleteObject(name)
}
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
result, err := o.bucket.CopyObject(src, dst)
if err != nil {
return nil, err
}
return &s3.CopyObjectInfo{
Key: dst,
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
}, nil
}
func (o *OSS) IsNotFound(err error) bool {
switch e := err.(type) {
case oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
case *oss.ServiceError:
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
default:
return false
}
}
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
})
}
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
UploadID: uploadID,
Key: name,
Bucket: o.bucket.BucketName,
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
if err != nil {
return nil, err
}
res := &s3.ListUploadedPartsResult{
Key: result.Key,
UploadID: result.UploadID,
MaxParts: result.MaxParts,
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
}
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
for i, part := range result.UploadedParts {
res.UploadedParts[i] = s3.UploadedPart{
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: int64(part.Size),
}
}
return res, nil
}
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
var opts []oss.Option
if opt != nil {
if opt.ContentType != "" {
opts = append(opts, oss.ContentType(opt.ContentType))
}
if opt.ContentDisposition != "" {
opts = append(opts, oss.ContentDisposition(opt.ContentDisposition))
}
}
if expire <= 0 {
expire = time.Hour * 24 * 365 * 99 // 99 years
} else if expire < time.Second {
expire = time.Second
}
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
}

@ -0,0 +1,81 @@
package oss
import (
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"hash"
"io"
"net/http"
"sort"
"strings"
)
func (o *OSS) getAdditionalHeaderKeys(req *http.Request) ([]string, map[string]string) {
var keysList []string
keysMap := make(map[string]string)
srcKeys := make(map[string]string)
for k := range req.Header {
srcKeys[strings.ToLower(k)] = ""
}
for _, v := range o.bucket.Client.Config.AdditionalHeaders {
if _, ok := srcKeys[strings.ToLower(v)]; ok {
keysMap[strings.ToLower(v)] = ""
}
}
for k := range keysMap {
keysList = append(keysList, k)
}
sort.Strings(keysList)
return keysList, keysMap
}
func (o *OSS) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
// Find out the "x-oss-"'s address in header of the request
ossHeadersMap := make(map[string]string)
additionalList, additionalMap := o.getAdditionalHeaderKeys(req)
for k, v := range req.Header {
if strings.HasPrefix(strings.ToLower(k), "x-oss-") {
ossHeadersMap[strings.ToLower(k)] = v[0]
} else if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
if _, ok := additionalMap[strings.ToLower(k)]; ok {
ossHeadersMap[strings.ToLower(k)] = v[0]
}
}
}
hs := newHeaderSorter(ossHeadersMap)
// Sort the ossHeadersMap by the ascending order
hs.Sort()
// Get the canonicalizedOSSHeaders
canonicalizedOSSHeaders := ""
for i := range hs.Keys {
canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n"
}
// Give other parameters values
// when sign URL, date is expires
date := req.Header.Get(oss.HTTPHeaderDate)
contentType := req.Header.Get(oss.HTTPHeaderContentType)
contentMd5 := req.Header.Get(oss.HTTPHeaderContentMD5)
// default is v1 signature
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
// v2 signature
if o.bucket.Client.Config.AuthVersion == oss.AuthV2 {
signStr = req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + strings.Join(additionalList, ";") + "\n" + canonicalizedResource
h = hmac.New(func() hash.Hash { return sha256.New() }, []byte(keySecret))
}
_, _ = io.WriteString(h, signStr)
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
return signedStr
}

@ -0,0 +1,47 @@
package oss
import (
"bytes"
"sort"
)
// headerSorter defines the key-value structure for storing the sorted data in signHeader.
type headerSorter struct {
Keys []string
Vals []string
}
// newHeaderSorter is an additional function for function SignHeader.
func newHeaderSorter(m map[string]string) *headerSorter {
hs := &headerSorter{
Keys: make([]string, 0, len(m)),
Vals: make([]string, 0, len(m)),
}
for k, v := range m {
hs.Keys = append(hs.Keys, k)
hs.Vals = append(hs.Vals, v)
}
return hs
}
// Sort is an additional function for function SignHeader.
func (hs *headerSorter) Sort() {
sort.Sort(hs)
}
// Len is an additional function for function SignHeader.
func (hs *headerSorter) Len() int {
return len(hs.Vals)
}
// Less is an additional function for function SignHeader.
func (hs *headerSorter) Less(i, j int) bool {
return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0
}
// Swap is an additional function for function SignHeader.
func (hs *headerSorter) Swap(i, j int) {
hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i]
hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i]
}

@ -0,0 +1,134 @@
package s3
import (
"context"
"net/http"
"net/url"
"time"
)
type PartLimit struct {
MinPartSize int64 `json:"minPartSize"`
MaxPartSize int64 `json:"maxPartSize"`
MaxNumSize int `json:"maxNumSize"`
}
type InitiateMultipartUploadResult struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadID"`
}
type MultipartUploadRequest struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type Part struct {
PartNumber int `json:"partNumber"`
ETag string `json:"etag"`
}
type CompleteMultipartUploadResult struct {
Location string `json:"location"`
Bucket string `json:"bucket"`
Key string `json:"key"`
ETag string `json:"etag"`
}
type SignResult struct {
Parts []SignPart `json:"parts"`
}
type ObjectInfo struct {
ETag string `json:"etag"`
Key string `json:"name"`
Size int64 `json:"size"`
LastModified time.Time `json:"lastModified"`
}
type CopyObjectInfo struct {
Key string `json:"name"`
ETag string `json:"etag"`
}
type SignPart struct {
PartNumber int `json:"partNumber"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
}
type AuthSignResult struct {
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
Parts []SignPart `json:"parts"`
}
type InitiateUpload struct {
UploadID string `json:"uploadId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Method string `json:"method"`
URL string `json:"url"`
Query url.Values `json:"query"`
Header http.Header `json:"header"`
PartKey string `json:"partKey"`
PartSize int64 `json:"partSize"`
FirstPart int `json:"firstPart"`
}
type UploadedPart struct {
PartNumber int `json:"partNumber"`
LastModified time.Time `json:"lastModified"`
ETag string `json:"etag"`
Size int64 `json:"size"`
}
type ListUploadedPartsResult struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
UploadedParts []UploadedPart `xml:"Part"`
}
type AccessURLOption struct {
ContentType string `json:"contentType"`
ContentDisposition string `json:"contentDisposition"`
}
type Interface interface {
Engine() string
PartLimit() *PartLimit
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
PartSize(ctx context.Context, size int64) (int64, error)
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
DeleteObject(ctx context.Context, name string) error
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
IsNotFound(err error) bool
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
}

@ -0,0 +1,31 @@
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object"
)
type ObjectModel struct {
Name string `gorm:"column:name;primary_key"`
UserID string `gorm:"column:user_id"`
Hash string `gorm:"column:hash"`
Key string `gorm:"column:key"`
Size int64 `gorm:"column:size"`
ContentType string `gorm:"column:content_type"`
Cause string `gorm:"column:cause"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectModel) error
Take(ctx context.Context, name string) (*ObjectModel, error)
}

@ -1,30 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectHashModelTableName = "object_hash"
)
type ObjectHashModel struct {
Hash string `gorm:"column:hash;primary_key;size:32"`
Engine string `gorm:"column:engine;primary_key;size:16"`
Size int64 `gorm:"column:size"`
Bucket string `gorm:"column:bucket"`
Name string `gorm:"column:name"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectHashModel) TableName() string {
return ObjectHashModelTableName
}
type ObjectHashModelInterface interface {
NewTx(tx any) ObjectHashModelInterface
Take(ctx context.Context, hash string, engine string) (*ObjectHashModel, error)
Create(ctx context.Context, h []*ObjectHashModel) error
DeleteNoCitation(ctx context.Context, engine string, num int) (list []*ObjectHashModel, err error)
}

@ -1,29 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectInfoModelTableName = "object_info"
)
type ObjectInfoModel struct {
Name string `gorm:"column:name;primary_key"`
Hash string `gorm:"column:hash"`
ContentType string `gorm:"column:content_type"`
ValidTime *time.Time `gorm:"column:valid_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectInfoModel) TableName() string {
return ObjectInfoModelTableName
}
type ObjectInfoModelInterface interface {
NewTx(tx any) ObjectInfoModelInterface
SetObject(ctx context.Context, obj *ObjectInfoModel) error
Take(ctx context.Context, name string) (*ObjectInfoModel, error)
DeleteExpiration(ctx context.Context, expiration time.Time) error
}

@ -1,37 +0,0 @@
package relation
import (
"context"
"time"
)
const (
ObjectPutModelTableName = "object_put"
)
type ObjectPutModel struct {
PutID string `gorm:"column:put_id;primary_key"`
Hash string `gorm:"column:hash"`
Path string `gorm:"column:path"`
Name string `gorm:"column:name"`
ContentType string `gorm:"column:content_type"`
ObjectSize int64 `gorm:"column:object_size"`
FragmentSize int64 `gorm:"column:fragment_size"`
PutURLsHash string `gorm:"column:put_urls_hash"`
ValidTime *time.Time `gorm:"column:valid_time"`
EffectiveTime time.Time `gorm:"column:effective_time"`
CreateTime time.Time `gorm:"column:create_time"`
}
func (ObjectPutModel) TableName() string {
return ObjectPutModelTableName
}
type ObjectPutModelInterface interface {
NewTx(tx any) ObjectPutModelInterface
Create(ctx context.Context, m []*ObjectPutModel) error
Take(ctx context.Context, putID string) (*ObjectPutModel, error)
SetCompleted(ctx context.Context, putID string) error
FindExpirationPut(ctx context.Context, expirationTime time.Time, num int) ([]*ObjectPutModel, error)
DelPut(ctx context.Context, ids []string) error
}

File diff suppressed because it is too large Load Diff

@ -2,80 +2,92 @@ syntax = "proto3";
package OpenIMServer.third;
option go_package = "github.com/OpenIMSDK/Open-IM-Server/pkg/proto/third";
message ApplyPutReq {
string putID = 1;
string name = 2;
int64 size = 3;
string hash = 4;
string contentType = 5;
int64 fragmentSize = 6;
int64 validTime = 7; //
message MapValues {
repeated string values = 1;
}
message ApplyPutResp {
message SignPart {
int32 partNumber = 1;
string url = 2;
map<string, MapValues> query = 3;
map<string, MapValues> header = 4;
}
message AuthSignParts {
string url = 1;
string putID = 2;
int64 fragmentSize = 3;
int64 validTime = 4;//
string putURLsHash = 5;
repeated string putURLs = 6;
map<string, MapValues> query = 2;
map<string, MapValues> header = 3;
repeated SignPart parts = 4;
}
message PartLimitReq {
}
message ConfirmPutReq {
string putID = 1;
message PartLimitResp {
int64 minPartSize = 1;
int64 maxPartSize = 2;
int32 maxNumSize = 3;
}
message PartSizeReq {
int64 size = 1;
}
message ConfirmPutResp {
message PartSizeResp {
int64 size = 2;
}
message InitiateMultipartUploadReq {
string hash = 1;
int64 size = 2;
int64 maxParts = 3;
string cause = 4;
string name = 5;
string contentType = 6;
}
message UploadInfo {
string uploadID = 1;
int64 partSize = 2;
AuthSignParts sign = 3;
}
message InitiateMultipartUploadResp {
string url = 1;
UploadInfo upload = 2;
}
message GetUrlReq {
string name = 1; //
int64 expires = 2; // url
bool attachment = 3;//
message AuthSignReq {
string uploadID = 1;
repeated int32 partNumbers = 2;
}
message GetUrlResp {
message AuthSignResp {
string url = 1;
int64 size = 2;
string hash = 3;
map<string, MapValues> query = 2;
map<string, MapValues> header = 3;
repeated SignPart parts = 4;
}
message GetPutReq {
string putID = 1;
message CompleteMultipartUploadReq {
string uploadID = 1;
repeated string parts = 2;
string name = 3;
string contentType = 4;
string cause = 5;
}
message GetPutFragment{
int64 size = 1;
string hash = 2;
string url = 3;
message CompleteMultipartUploadResp {
string url = 1;
}
message GetPutResp {
message AccessURLReq {
string name = 1;
int64 size = 2;
string hash = 3;
int64 fragmentSize = 4;
string contentType = 5;
int64 validTime = 6; //
// repeated GetPutFragment fragments = 7;
// string putURLsHash = 8;
// string putID = 2;
// int64 fragmentSize = 3;
// int64 validTime = 4;//
string putURLsHash = 7;
repeated GetPutFragment fragments = 8;
// repeated string putURLs = 6;
// repeated GetPutFragment fragments = 7;
}
message GetHashInfoReq {
string hash = 1;
}
message GetHashInfoResp {
string hash = 1;
int64 size = 2;
message AccessURLResp {
string url = 1;
int64 expireTime = 2;
}
message FcmUpdateTokenReq {
@ -97,11 +109,13 @@ message SetAppBadgeResp {
}
service third {
rpc ApplyPut(ApplyPutReq) returns(ApplyPutResp);
rpc GetPut(GetPutReq) returns(GetPutResp);
rpc ConfirmPut(ConfirmPutReq) returns(ConfirmPutResp);
rpc GetUrl(GetUrlReq) returns(GetUrlResp);
rpc GetHashInfo(GetHashInfoReq) returns(GetHashInfoResp);
rpc PartLimit(PartLimitReq) returns(PartLimitResp);
rpc PartSize(PartSizeReq) returns(PartSizeResp);
rpc InitiateMultipartUpload(InitiateMultipartUploadReq) returns(InitiateMultipartUploadResp);
rpc AuthSign(AuthSignReq) returns(AuthSignResp);
rpc CompleteMultipartUpload(CompleteMultipartUploadReq) returns(CompleteMultipartUploadResp);
rpc AccessURL(AccessURLReq) returns(AccessURLResp);
rpc FcmUpdateToken(FcmUpdateTokenReq) returns(FcmUpdateTokenResp);
rpc SetAppBadge(SetAppBadgeReq) returns(SetAppBadgeResp);
}

Loading…
Cancel
Save