withchao
2 years ago
parent
809e3786c1
commit
53b64ae7c8
@ -1,38 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont
|
|
||||||
|
|
||||||
const (
|
|
||||||
// HashPath defines the storage path for hash data within the 'openim' directory.
|
|
||||||
hashPath = "openim/data/hash/"
|
|
||||||
|
|
||||||
// TempPath specifies the directory for temporary files in the 'openim' structure.
|
|
||||||
tempPath = "openim/temp/"
|
|
||||||
|
|
||||||
// DirectPath indicates the directory for direct uploads or access within the 'openim' structure.
|
|
||||||
DirectPath = "openim/direct"
|
|
||||||
|
|
||||||
// UploadTypeMultipart represents the identifier for multipart uploads,
|
|
||||||
// allowing large files to be uploaded in chunks.
|
|
||||||
UploadTypeMultipart = 1
|
|
||||||
|
|
||||||
// UploadTypePresigned signifies the use of presigned URLs for uploads,
|
|
||||||
// facilitating secure, authorized file transfers without requiring direct access to the storage credentials.
|
|
||||||
UploadTypePresigned = 2
|
|
||||||
|
|
||||||
// PartSeparator is used as a delimiter in multipart upload processes,
|
|
||||||
// separating individual file parts.
|
|
||||||
partSeparator = ","
|
|
||||||
)
|
|
||||||
@ -1,282 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
"github.com/openimsdk/tools/errs"
|
|
||||||
"github.com/openimsdk/tools/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
func New(cache cache.S3Cache, impl s3.Interface) *Controller {
|
|
||||||
return &Controller{
|
|
||||||
cache: cache,
|
|
||||||
impl: impl,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type Controller struct {
|
|
||||||
cache cache.S3Cache
|
|
||||||
impl s3.Interface
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) Engine() string {
|
|
||||||
return c.impl.Engine()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) HashPath(md5 string) string {
|
|
||||||
return path.Join(hashPath, md5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) NowPath() string {
|
|
||||||
now := time.Now()
|
|
||||||
return path.Join(
|
|
||||||
fmt.Sprintf("%04d", now.Year()),
|
|
||||||
fmt.Sprintf("%02d", now.Month()),
|
|
||||||
fmt.Sprintf("%02d", now.Day()),
|
|
||||||
fmt.Sprintf("%02d", now.Hour()),
|
|
||||||
fmt.Sprintf("%02d", now.Minute()),
|
|
||||||
fmt.Sprintf("%02d", now.Second()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) UUID() string {
|
|
||||||
id := uuid.New()
|
|
||||||
return hex.EncodeToString(id[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) PartSize(ctx context.Context, size int64) (int64, error) {
|
|
||||||
return c.impl.PartSize(ctx, size)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) PartLimit() *s3.PartLimit {
|
|
||||||
return c.impl.PartLimit()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
|
||||||
return c.cache.GetKey(ctx, c.impl.Engine(), name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) GetHashObject(ctx context.Context, hash string) (*s3.ObjectInfo, error) {
|
|
||||||
return c.StatObject(ctx, c.HashPath(hash))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) InitiateUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*InitiateUploadResult, error) {
|
|
||||||
defer log.ZDebug(ctx, "return")
|
|
||||||
if size < 0 {
|
|
||||||
return nil, errors.New("invalid size")
|
|
||||||
}
|
|
||||||
if hashBytes, err := hex.DecodeString(hash); err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else if len(hashBytes) != md5.Size {
|
|
||||||
return nil, errors.New("invalid md5")
|
|
||||||
}
|
|
||||||
partSize, err := c.impl.PartSize(ctx, size)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
partNumber := int(size / partSize)
|
|
||||||
if size%partSize > 0 {
|
|
||||||
partNumber++
|
|
||||||
}
|
|
||||||
if maxParts > 0 && partNumber > 0 && partNumber < maxParts {
|
|
||||||
return nil, fmt.Errorf("too many parts: %d", partNumber)
|
|
||||||
}
|
|
||||||
if info, err := c.StatObject(ctx, c.HashPath(hash)); err == nil {
|
|
||||||
return nil, &HashAlreadyExistsError{Object: info}
|
|
||||||
} else if !c.impl.IsNotFound(err) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if size <= partSize {
|
|
||||||
// Pre-signed upload
|
|
||||||
key := path.Join(tempPath, c.NowPath(), fmt.Sprintf("%s_%d_%s.presigned", hash, size, c.UUID()))
|
|
||||||
rawURL, err := c.impl.PresignedPutObject(ctx, key, expire)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &InitiateUploadResult{
|
|
||||||
UploadID: newMultipartUploadID(multipartUploadID{
|
|
||||||
Type: UploadTypePresigned,
|
|
||||||
ID: "",
|
|
||||||
Key: key,
|
|
||||||
Size: size,
|
|
||||||
Hash: hash,
|
|
||||||
}),
|
|
||||||
PartSize: partSize,
|
|
||||||
Sign: &s3.AuthSignResult{
|
|
||||||
Parts: []s3.SignPart{
|
|
||||||
{
|
|
||||||
PartNumber: 1,
|
|
||||||
URL: rawURL,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
} else {
|
|
||||||
// Fragment upload
|
|
||||||
upload, err := c.impl.InitiateMultipartUpload(ctx, c.HashPath(hash))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if maxParts < 0 {
|
|
||||||
maxParts = partNumber
|
|
||||||
}
|
|
||||||
var authSign *s3.AuthSignResult
|
|
||||||
if maxParts > 0 {
|
|
||||||
partNumbers := make([]int, maxParts)
|
|
||||||
for i := 0; i < maxParts; i++ {
|
|
||||||
partNumbers[i] = i + 1
|
|
||||||
}
|
|
||||||
authSign, err = c.impl.AuthSign(ctx, upload.UploadID, upload.Key, time.Hour*24, partNumbers)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &InitiateUploadResult{
|
|
||||||
UploadID: newMultipartUploadID(multipartUploadID{
|
|
||||||
Type: UploadTypeMultipart,
|
|
||||||
ID: upload.UploadID,
|
|
||||||
Key: upload.Key,
|
|
||||||
Size: size,
|
|
||||||
Hash: hash,
|
|
||||||
}),
|
|
||||||
PartSize: partSize,
|
|
||||||
Sign: authSign,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) CompleteUpload(ctx context.Context, uploadID string, partHashs []string) (*UploadResult, error) {
|
|
||||||
defer log.ZDebug(ctx, "return")
|
|
||||||
upload, err := parseMultipartUploadID(uploadID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if md5Sum := md5.Sum([]byte(strings.Join(partHashs, partSeparator))); hex.EncodeToString(md5Sum[:]) != upload.Hash {
|
|
||||||
return nil, errors.New("md5 mismatching")
|
|
||||||
}
|
|
||||||
if info, err := c.StatObject(ctx, c.HashPath(upload.Hash)); err == nil {
|
|
||||||
return &UploadResult{
|
|
||||||
Key: info.Key,
|
|
||||||
Size: info.Size,
|
|
||||||
Hash: info.ETag,
|
|
||||||
}, nil
|
|
||||||
} else if !c.IsNotFound(err) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
cleanObject := make(map[string]struct{})
|
|
||||||
defer func() {
|
|
||||||
for key := range cleanObject {
|
|
||||||
_ = c.impl.DeleteObject(ctx, key)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
var targetKey string
|
|
||||||
switch upload.Type {
|
|
||||||
case UploadTypeMultipart:
|
|
||||||
parts := make([]s3.Part, len(partHashs))
|
|
||||||
for i, part := range partHashs {
|
|
||||||
parts[i] = s3.Part{
|
|
||||||
PartNumber: i + 1,
|
|
||||||
ETag: part,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// todo: Validation size
|
|
||||||
result, err := c.impl.CompleteMultipartUpload(ctx, upload.ID, upload.Key, parts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
targetKey = result.Key
|
|
||||||
case UploadTypePresigned:
|
|
||||||
uploadInfo, err := c.StatObject(ctx, upload.Key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
cleanObject[uploadInfo.Key] = struct{}{}
|
|
||||||
if uploadInfo.Size != upload.Size {
|
|
||||||
return nil, errors.New("upload size mismatching")
|
|
||||||
}
|
|
||||||
md5Sum := md5.Sum([]byte(strings.Join([]string{uploadInfo.ETag}, partSeparator)))
|
|
||||||
if md5val := hex.EncodeToString(md5Sum[:]); md5val != upload.Hash {
|
|
||||||
return nil, errs.ErrArgs.WrapMsg(fmt.Sprintf("md5 mismatching %s != %s", md5val, upload.Hash))
|
|
||||||
}
|
|
||||||
// Prevents concurrent operations at this time that cause files to be overwritten
|
|
||||||
copyInfo, err := c.impl.CopyObject(ctx, uploadInfo.Key, upload.Key+"."+c.UUID())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
cleanObject[copyInfo.Key] = struct{}{}
|
|
||||||
if copyInfo.ETag != uploadInfo.ETag {
|
|
||||||
return nil, errors.New("[concurrency]copy md5 mismatching")
|
|
||||||
}
|
|
||||||
hashCopyInfo, err := c.impl.CopyObject(ctx, copyInfo.Key, c.HashPath(upload.Hash))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
log.ZInfo(ctx, "hashCopyInfo", "value", fmt.Sprintf("%+v", hashCopyInfo))
|
|
||||||
targetKey = hashCopyInfo.Key
|
|
||||||
default:
|
|
||||||
return nil, errors.New("invalid upload id type")
|
|
||||||
}
|
|
||||||
if err := c.cache.DelS3Key(c.impl.Engine(), targetKey).ExecDel(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &UploadResult{
|
|
||||||
Key: targetKey,
|
|
||||||
Size: upload.Size,
|
|
||||||
Hash: upload.Hash,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) {
|
|
||||||
upload, err := parseMultipartUploadID(uploadID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
switch upload.Type {
|
|
||||||
case UploadTypeMultipart:
|
|
||||||
return c.impl.AuthSign(ctx, upload.ID, upload.Key, time.Hour*24, partNumbers)
|
|
||||||
case UploadTypePresigned:
|
|
||||||
return nil, errors.New("presigned id not support auth sign")
|
|
||||||
default:
|
|
||||||
return nil, errors.New("invalid upload id type")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) IsNotFound(err error) bool {
|
|
||||||
return c.impl.IsNotFound(err) || errs.ErrRecordNotFound.Is(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
|
||||||
if opt.Image != nil {
|
|
||||||
opt.Filename = ""
|
|
||||||
opt.ContentType = ""
|
|
||||||
}
|
|
||||||
return c.impl.AccessURL(ctx, name, expire, opt)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Controller) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
|
|
||||||
return c.impl.FormData(ctx, name, size, contentType, duration)
|
|
||||||
}
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
// Copyright © 2024 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cont"
|
|
||||||
@ -1,29 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
)
|
|
||||||
|
|
||||||
type HashAlreadyExistsError struct {
|
|
||||||
Object *s3.ObjectInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *HashAlreadyExistsError) Error() string {
|
|
||||||
return fmt.Sprintf("hash already exists: %s", e.Object.Key)
|
|
||||||
}
|
|
||||||
@ -1,49 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
type multipartUploadID struct {
|
|
||||||
Type int `json:"a,omitempty"`
|
|
||||||
ID string `json:"b,omitempty"`
|
|
||||||
Key string `json:"c,omitempty"`
|
|
||||||
Size int64 `json:"d,omitempty"`
|
|
||||||
Hash string `json:"e,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMultipartUploadID(id multipartUploadID) string {
|
|
||||||
data, err := json.Marshal(id)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return base64.StdEncoding.EncodeToString(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseMultipartUploadID(id string) (*multipartUploadID, error) {
|
|
||||||
data, err := base64.StdEncoding.DecodeString(id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
|
||||||
}
|
|
||||||
var upload multipartUploadID
|
|
||||||
if err := json.Unmarshal(data, &upload); err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid multipart upload id: %w", err)
|
|
||||||
}
|
|
||||||
return &upload, nil
|
|
||||||
}
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cont
|
|
||||||
|
|
||||||
import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
|
|
||||||
type InitiateUploadResult struct {
|
|
||||||
// UploadID uniquely identifies the upload session for tracking and management purposes.
|
|
||||||
UploadID string `json:"uploadID"`
|
|
||||||
|
|
||||||
// PartSize specifies the size of each part in a multipart upload. This is relevant for breaking down large uploads into manageable pieces.
|
|
||||||
PartSize int64 `json:"partSize"`
|
|
||||||
|
|
||||||
// Sign contains the authentication and signature information necessary for securely uploading each part. This could include signed URLs or tokens.
|
|
||||||
Sign *s3.AuthSignResult `json:"sign"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type UploadResult struct {
|
|
||||||
Hash string `json:"hash"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
Key string `json:"key"`
|
|
||||||
}
|
|
||||||
@ -1,400 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cos
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/sha1"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
"github.com/openimsdk/tools/errs"
|
|
||||||
"github.com/tencentyun/cos-go-sdk-v5"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
minPartSize int64 = 1024 * 1024 * 1 // 1MB
|
|
||||||
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
|
|
||||||
maxNumSize int64 = 1000
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
imagePng = "png"
|
|
||||||
imageJpg = "jpg"
|
|
||||||
imageJpeg = "jpeg"
|
|
||||||
imageGif = "gif"
|
|
||||||
imageWebp = "webp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const successCode = http.StatusOK
|
|
||||||
|
|
||||||
type Config struct {
|
|
||||||
BucketURL string
|
|
||||||
SecretID string
|
|
||||||
SecretKey string
|
|
||||||
SessionToken string
|
|
||||||
PublicRead bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCos(conf Config) (s3.Interface, error) {
|
|
||||||
u, err := url.Parse(conf.BucketURL)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, &http.Client{
|
|
||||||
Transport: &cos.AuthorizationTransport{
|
|
||||||
SecretID: conf.SecretID,
|
|
||||||
SecretKey: conf.SecretKey,
|
|
||||||
SessionToken: conf.SessionToken,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
return &Cos{
|
|
||||||
publicRead: conf.PublicRead,
|
|
||||||
copyURL: u.Host + "/",
|
|
||||||
client: client,
|
|
||||||
credential: client.GetCredential(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type Cos struct {
|
|
||||||
publicRead bool
|
|
||||||
copyURL string
|
|
||||||
client *cos.Client
|
|
||||||
credential *cos.Credential
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) Engine() string {
|
|
||||||
return "tencent-cos"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) PartLimit() *s3.PartLimit {
|
|
||||||
return &s3.PartLimit{
|
|
||||||
MinPartSize: minPartSize,
|
|
||||||
MaxPartSize: maxPartSize,
|
|
||||||
MaxNumSize: maxNumSize,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
|
||||||
result, _, err := c.client.Object.InitiateMultipartUpload(ctx, name, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.InitiateMultipartUploadResult{
|
|
||||||
UploadID: result.UploadID,
|
|
||||||
Bucket: result.Bucket,
|
|
||||||
Key: result.Key,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
|
||||||
opts := &cos.CompleteMultipartUploadOptions{
|
|
||||||
Parts: make([]cos.Object, len(parts)),
|
|
||||||
}
|
|
||||||
for i, part := range parts {
|
|
||||||
opts.Parts[i] = cos.Object{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
ETag: strings.ReplaceAll(part.ETag, `"`, ``),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result, _, err := c.client.Object.CompleteMultipartUpload(ctx, name, uploadID, opts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.CompleteMultipartUploadResult{
|
|
||||||
Location: result.Location,
|
|
||||||
Bucket: result.Bucket,
|
|
||||||
Key: result.Key,
|
|
||||||
ETag: result.ETag,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) PartSize(ctx context.Context, size int64) (int64, error) {
|
|
||||||
if size <= 0 {
|
|
||||||
return 0, errors.New("size must be greater than 0")
|
|
||||||
}
|
|
||||||
if size > maxPartSize*maxNumSize {
|
|
||||||
return 0, fmt.Errorf("COS size must be less than the maximum allowed limit")
|
|
||||||
}
|
|
||||||
if size <= minPartSize*maxNumSize {
|
|
||||||
return minPartSize, nil
|
|
||||||
}
|
|
||||||
partSize := size / maxNumSize
|
|
||||||
if size%maxNumSize != 0 {
|
|
||||||
partSize++
|
|
||||||
}
|
|
||||||
return partSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
|
||||||
result := s3.AuthSignResult{
|
|
||||||
URL: c.client.BaseURL.BucketURL.String() + "/" + cos.EncodeURIComponent(name),
|
|
||||||
Query: url.Values{"uploadId": {uploadID}},
|
|
||||||
Header: make(http.Header),
|
|
||||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
|
||||||
}
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, result.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
cos.AddAuthorizationHeader(c.credential.SecretID, c.credential.SecretKey, c.credential.SessionToken, req, cos.NewAuthTime(expire))
|
|
||||||
result.Header = req.Header
|
|
||||||
for i, partNumber := range partNumbers {
|
|
||||||
result.Parts[i] = s3.SignPart{
|
|
||||||
PartNumber: partNumber,
|
|
||||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
|
||||||
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodPut, name, c.credential.SecretID, c.credential.SecretKey, expire, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return rawURL.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) DeleteObject(ctx context.Context, name string) error {
|
|
||||||
_, err := c.client.Object.Delete(ctx, name)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
|
||||||
if name != "" && name[0] == '/' {
|
|
||||||
name = name[1:]
|
|
||||||
}
|
|
||||||
info, err := c.client.Object.Head(ctx, name, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res := &s3.ObjectInfo{Key: name}
|
|
||||||
if res.ETag = strings.ToLower(strings.ReplaceAll(info.Header.Get("ETag"), `"`, "")); res.ETag == "" {
|
|
||||||
return nil, errors.New("StatObject etag not found")
|
|
||||||
}
|
|
||||||
if contentLengthStr := info.Header.Get("Content-Length"); contentLengthStr == "" {
|
|
||||||
return nil, errors.New("StatObject content-length not found")
|
|
||||||
} else {
|
|
||||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("StatObject content-length parse error: %w", err)
|
|
||||||
}
|
|
||||||
if res.Size < 0 {
|
|
||||||
return nil, errors.New("StatObject content-length must be greater than 0")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if lastModified := info.Header.Get("Last-Modified"); lastModified == "" {
|
|
||||||
return nil, errors.New("StatObject last-modified not found")
|
|
||||||
} else {
|
|
||||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("StatObject last-modified parse error: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
|
||||||
sourceURL := c.copyURL + src
|
|
||||||
result, _, err := c.client.Object.Copy(ctx, dst, sourceURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.CopyObjectInfo{
|
|
||||||
Key: dst,
|
|
||||||
ETag: strings.ReplaceAll(result.ETag, `"`, ``),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) IsNotFound(err error) bool {
|
|
||||||
switch e := errs.Unwrap(err).(type) {
|
|
||||||
case *cos.ErrorResponse:
|
|
||||||
return e.Response.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
|
||||||
_, err := c.client.Object.AbortMultipartUpload(ctx, name, uploadID)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
|
||||||
result, _, err := c.client.Object.ListParts(ctx, name, uploadID, &cos.ObjectListPartsOptions{
|
|
||||||
MaxParts: strconv.Itoa(maxParts),
|
|
||||||
PartNumberMarker: strconv.Itoa(partNumberMarker),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res := &s3.ListUploadedPartsResult{
|
|
||||||
Key: result.Key,
|
|
||||||
UploadID: result.UploadID,
|
|
||||||
UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
|
|
||||||
}
|
|
||||||
res.MaxParts, _ = strconv.Atoi(result.MaxParts)
|
|
||||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
|
||||||
for i, part := range result.Parts {
|
|
||||||
lastModified, _ := time.Parse(http.TimeFormat, part.LastModified)
|
|
||||||
res.UploadedParts[i] = s3.UploadedPart{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
LastModified: lastModified,
|
|
||||||
ETag: part.ETag,
|
|
||||||
Size: part.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
|
||||||
var imageMogr string
|
|
||||||
var option cos.PresignedURLOptions
|
|
||||||
if opt != nil {
|
|
||||||
query := make(url.Values)
|
|
||||||
if opt.Image != nil {
|
|
||||||
// https://cloud.tencent.com/document/product/436/44880
|
|
||||||
style := make([]string, 0, 2)
|
|
||||||
wh := make([]string, 2)
|
|
||||||
if opt.Image.Width > 0 {
|
|
||||||
wh[0] = strconv.Itoa(opt.Image.Width)
|
|
||||||
}
|
|
||||||
if opt.Image.Height > 0 {
|
|
||||||
wh[1] = strconv.Itoa(opt.Image.Height)
|
|
||||||
}
|
|
||||||
if opt.Image.Width > 0 || opt.Image.Height > 0 {
|
|
||||||
style = append(style, strings.Join(wh, "x"))
|
|
||||||
}
|
|
||||||
switch opt.Image.Format {
|
|
||||||
case
|
|
||||||
imagePng,
|
|
||||||
imageJpg,
|
|
||||||
imageJpeg,
|
|
||||||
imageGif,
|
|
||||||
imageWebp:
|
|
||||||
style = append(style, "format/"+opt.Image.Format)
|
|
||||||
}
|
|
||||||
if len(style) > 0 {
|
|
||||||
imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if opt.ContentType != "" {
|
|
||||||
query.Set("response-content-type", opt.ContentType)
|
|
||||||
}
|
|
||||||
if opt.Filename != "" {
|
|
||||||
query.Set("response-content-disposition", `attachment; filename=`+strconv.Quote(opt.Filename))
|
|
||||||
}
|
|
||||||
if len(query) > 0 {
|
|
||||||
option.Query = &query
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if expire <= 0 {
|
|
||||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
|
||||||
} else if expire < time.Second {
|
|
||||||
expire = time.Second
|
|
||||||
}
|
|
||||||
rawURL, err := c.getPresignedURL(ctx, name, expire, &option)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if imageMogr != "" {
|
|
||||||
if rawURL.RawQuery == "" {
|
|
||||||
rawURL.RawQuery = imageMogr
|
|
||||||
} else {
|
|
||||||
rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return rawURL.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
|
|
||||||
if !c.publicRead {
|
|
||||||
return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
|
|
||||||
}
|
|
||||||
return c.client.Object.GetObjectURL(name), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Cos) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
|
|
||||||
// https://cloud.tencent.com/document/product/436/14690
|
|
||||||
now := time.Now()
|
|
||||||
expiration := now.Add(duration)
|
|
||||||
keyTime := fmt.Sprintf("%d;%d", now.Unix(), expiration.Unix())
|
|
||||||
conditions := []any{
|
|
||||||
map[string]string{"q-sign-algorithm": "sha1"},
|
|
||||||
map[string]string{"q-ak": c.credential.SecretID},
|
|
||||||
map[string]string{"q-sign-time": keyTime},
|
|
||||||
map[string]string{"key": name},
|
|
||||||
}
|
|
||||||
if contentType != "" {
|
|
||||||
conditions = append(conditions, map[string]string{"Content-Type": contentType})
|
|
||||||
}
|
|
||||||
policy := map[string]any{
|
|
||||||
"expiration": expiration.Format("2006-01-02T15:04:05.000Z"),
|
|
||||||
"conditions": conditions,
|
|
||||||
}
|
|
||||||
policyJson, err := json.Marshal(policy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
signKey := hmacSha1val(c.credential.SecretKey, keyTime)
|
|
||||||
strToSign := sha1val(string(policyJson))
|
|
||||||
signature := hmacSha1val(signKey, strToSign)
|
|
||||||
|
|
||||||
fd := &s3.FormData{
|
|
||||||
URL: c.client.BaseURL.BucketURL.String(),
|
|
||||||
File: "file",
|
|
||||||
Expires: expiration,
|
|
||||||
FormData: map[string]string{
|
|
||||||
"policy": base64.StdEncoding.EncodeToString(policyJson),
|
|
||||||
"q-sign-algorithm": "sha1",
|
|
||||||
"q-ak": c.credential.SecretID,
|
|
||||||
"q-key-time": keyTime,
|
|
||||||
"q-signature": signature,
|
|
||||||
"key": name,
|
|
||||||
"success_action_status": strconv.Itoa(successCode),
|
|
||||||
},
|
|
||||||
SuccessCodes: []int{successCode},
|
|
||||||
}
|
|
||||||
if contentType != "" {
|
|
||||||
fd.FormData["Content-Type"] = contentType
|
|
||||||
}
|
|
||||||
if c.credential.SessionToken != "" {
|
|
||||||
fd.FormData["x-cos-security-token"] = c.credential.SessionToken
|
|
||||||
}
|
|
||||||
return fd, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func hmacSha1val(key, msg string) string {
|
|
||||||
v := hmac.New(sha1.New, []byte(key))
|
|
||||||
v.Write([]byte(msg))
|
|
||||||
return hex.EncodeToString(v.Sum(nil))
|
|
||||||
}
|
|
||||||
|
|
||||||
func sha1val(msg string) string {
|
|
||||||
sha1Hash := sha1.New()
|
|
||||||
sha1Hash.Write([]byte(msg))
|
|
||||||
return hex.EncodeToString(sha1Hash.Sum(nil))
|
|
||||||
}
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
// Copyright © 2024 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package cos
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
_ "unsafe"
|
|
||||||
|
|
||||||
"github.com/tencentyun/cos-go-sdk-v5"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
|
|
||||||
func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body any, optQuery any, optHeader any) (req *http.Request, err error)
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
// Copyright © 2024 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package s3 // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
// Copyright © 2024 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"
|
|
||||||
@ -1,120 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package minio
|
|
||||||
|
|
||||||
import (
|
|
||||||
"image"
|
|
||||||
_ "image/gif"
|
|
||||||
_ "image/jpeg"
|
|
||||||
_ "image/png"
|
|
||||||
"io"
|
|
||||||
|
|
||||||
_ "golang.org/x/image/bmp"
|
|
||||||
_ "golang.org/x/image/tiff"
|
|
||||||
_ "golang.org/x/image/webp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
formatPng = "png"
|
|
||||||
formatJpeg = "jpeg"
|
|
||||||
formatJpg = "jpg"
|
|
||||||
formatGif = "gif"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ImageStat(reader io.Reader) (image.Image, string, error) {
|
|
||||||
return image.Decode(reader)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ImageWidthHeight(img image.Image) (int, int) {
|
|
||||||
bounds := img.Bounds().Max
|
|
||||||
return bounds.X, bounds.Y
|
|
||||||
}
|
|
||||||
|
|
||||||
func resizeImage(img image.Image, maxWidth, maxHeight int) image.Image {
|
|
||||||
bounds := img.Bounds()
|
|
||||||
imgWidth := bounds.Max.X
|
|
||||||
imgHeight := bounds.Max.Y
|
|
||||||
|
|
||||||
// Calculating scaling
|
|
||||||
scaleWidth := float64(maxWidth) / float64(imgWidth)
|
|
||||||
scaleHeight := float64(maxHeight) / float64(imgHeight)
|
|
||||||
|
|
||||||
// If both are 0, then no scaling is done and the original image is returned
|
|
||||||
if maxWidth == 0 && maxHeight == 0 {
|
|
||||||
return img
|
|
||||||
}
|
|
||||||
|
|
||||||
// If both width and height are greater than 0, select a smaller zoom ratio to maintain the aspect ratio
|
|
||||||
if maxWidth > 0 && maxHeight > 0 {
|
|
||||||
scale := scaleWidth
|
|
||||||
if scaleHeight < scaleWidth {
|
|
||||||
scale = scaleHeight
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate Thumbnail Size
|
|
||||||
thumbnailWidth := int(float64(imgWidth) * scale)
|
|
||||||
thumbnailHeight := int(float64(imgHeight) * scale)
|
|
||||||
|
|
||||||
// Thumbnails are generated using the Resample method of the "image" library.
|
|
||||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
|
||||||
for y := 0; y < thumbnailHeight; y++ {
|
|
||||||
for x := 0; x < thumbnailWidth; x++ {
|
|
||||||
srcX := int(float64(x) / scale)
|
|
||||||
srcY := int(float64(y) / scale)
|
|
||||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return thumbnail
|
|
||||||
}
|
|
||||||
|
|
||||||
// If only width or height is specified, thumbnails are generated based on the maximum not to exceed rule
|
|
||||||
if maxWidth > 0 {
|
|
||||||
thumbnailWidth := maxWidth
|
|
||||||
thumbnailHeight := int(float64(imgHeight) * scaleWidth)
|
|
||||||
|
|
||||||
// Thumbnails are generated using the Resample method of the "image" library.
|
|
||||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
|
||||||
for y := 0; y < thumbnailHeight; y++ {
|
|
||||||
for x := 0; x < thumbnailWidth; x++ {
|
|
||||||
srcX := int(float64(x) / scaleWidth)
|
|
||||||
srcY := int(float64(y) / scaleWidth)
|
|
||||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return thumbnail
|
|
||||||
}
|
|
||||||
|
|
||||||
if maxHeight > 0 {
|
|
||||||
thumbnailWidth := int(float64(imgWidth) * scaleHeight)
|
|
||||||
thumbnailHeight := maxHeight
|
|
||||||
|
|
||||||
// Thumbnails are generated using the Resample method of the "image" library.
|
|
||||||
thumbnail := image.NewRGBA(image.Rect(0, 0, thumbnailWidth, thumbnailHeight))
|
|
||||||
for y := 0; y < thumbnailHeight; y++ {
|
|
||||||
for x := 0; x < thumbnailWidth; x++ {
|
|
||||||
srcX := int(float64(x) / scaleHeight)
|
|
||||||
srcY := int(float64(y) / scaleHeight)
|
|
||||||
thumbnail.Set(x, y, img.At(srcX, srcY))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return thumbnail
|
|
||||||
}
|
|
||||||
|
|
||||||
// By default, the original image is returned
|
|
||||||
return img
|
|
||||||
}
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package minio
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/url"
|
|
||||||
_ "unsafe"
|
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
|
|
||||||
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)
|
|
||||||
@ -1,499 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package minio
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"path"
|
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
|
||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
||||||
"github.com/minio/minio-go/v7/pkg/signer"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
"github.com/openimsdk/tools/errs"
|
|
||||||
"github.com/openimsdk/tools/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
unsignedPayload = "UNSIGNED-PAYLOAD"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
minPartSize int64 = 1024 * 1024 * 5 // 5MB
|
|
||||||
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
|
|
||||||
maxNumSize int64 = 10000
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
maxImageWidth = 1024
|
|
||||||
maxImageHeight = 1024
|
|
||||||
maxImageSize = 1024 * 1024 * 50
|
|
||||||
imageThumbnailPath = "openim/thumbnail"
|
|
||||||
)
|
|
||||||
|
|
||||||
const successCode = http.StatusOK
|
|
||||||
|
|
||||||
type Config struct {
|
|
||||||
Bucket string
|
|
||||||
Endpoint string
|
|
||||||
AccessKeyID string
|
|
||||||
SecretAccessKey string
|
|
||||||
SessionToken string
|
|
||||||
SignEndpoint string
|
|
||||||
PublicRead bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewMinio(cache cache.MinioCache, conf Config) (s3.Interface, error) {
|
|
||||||
u, err := url.Parse(conf.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
opts := &minio.Options{
|
|
||||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
|
||||||
Secure: u.Scheme == "https",
|
|
||||||
}
|
|
||||||
client, err := minio.New(u.Host, opts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m := &Minio{
|
|
||||||
conf: conf,
|
|
||||||
bucket: conf.Bucket,
|
|
||||||
core: &minio.Core{Client: client},
|
|
||||||
lock: &sync.Mutex{},
|
|
||||||
init: false,
|
|
||||||
cache: cache,
|
|
||||||
}
|
|
||||||
if conf.SignEndpoint == "" || conf.SignEndpoint == conf.Endpoint {
|
|
||||||
m.opts = opts
|
|
||||||
m.sign = m.core.Client
|
|
||||||
m.prefix = u.Path
|
|
||||||
u.Path = ""
|
|
||||||
conf.Endpoint = u.String()
|
|
||||||
m.signEndpoint = conf.Endpoint
|
|
||||||
} else {
|
|
||||||
su, err := url.Parse(conf.SignEndpoint)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m.opts = &minio.Options{
|
|
||||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken),
|
|
||||||
Secure: su.Scheme == "https",
|
|
||||||
}
|
|
||||||
m.sign, err = minio.New(su.Host, m.opts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m.prefix = su.Path
|
|
||||||
su.Path = ""
|
|
||||||
conf.SignEndpoint = su.String()
|
|
||||||
m.signEndpoint = conf.SignEndpoint
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
fmt.Println("init minio error:", err)
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type Minio struct {
|
|
||||||
conf Config
|
|
||||||
bucket string
|
|
||||||
signEndpoint string
|
|
||||||
location string
|
|
||||||
opts *minio.Options
|
|
||||||
core *minio.Core
|
|
||||||
sign *minio.Client
|
|
||||||
lock sync.Locker
|
|
||||||
init bool
|
|
||||||
prefix string
|
|
||||||
cache cache.MinioCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) initMinio(ctx context.Context) error {
|
|
||||||
if m.init {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
m.lock.Lock()
|
|
||||||
defer m.lock.Unlock()
|
|
||||||
if m.init {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
exists, err := m.core.Client.BucketExists(ctx, m.conf.Bucket)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("check bucket exists error: %w", err)
|
|
||||||
}
|
|
||||||
if !exists {
|
|
||||||
if err = m.core.Client.MakeBucket(ctx, m.conf.Bucket, minio.MakeBucketOptions{}); err != nil {
|
|
||||||
return fmt.Errorf("make bucket error: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m.conf.PublicRead {
|
|
||||||
policy := fmt.Sprintf(
|
|
||||||
`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
|
|
||||||
m.conf.Bucket,
|
|
||||||
)
|
|
||||||
if err = m.core.Client.SetBucketPolicy(ctx, m.conf.Bucket, policy); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
m.location, err = m.core.Client.GetBucketLocation(ctx, m.conf.Bucket)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
func() {
|
|
||||||
if m.conf.SignEndpoint == "" || m.conf.SignEndpoint == m.conf.Endpoint {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if r := recover(); r != nil {
|
|
||||||
m.sign = m.core.Client
|
|
||||||
log.ZWarn(
|
|
||||||
context.Background(),
|
|
||||||
"set sign bucket location cache panic",
|
|
||||||
errors.New("failed to get private field value"),
|
|
||||||
"recover",
|
|
||||||
fmt.Sprintf("%+v", r),
|
|
||||||
"development version",
|
|
||||||
"github.com/minio/minio-go/v7 v7.0.61",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
blc := reflect.ValueOf(m.sign).Elem().FieldByName("bucketLocCache")
|
|
||||||
vblc := reflect.New(reflect.PtrTo(blc.Type()))
|
|
||||||
*(*unsafe.Pointer)(vblc.UnsafePointer()) = unsafe.Pointer(blc.UnsafeAddr())
|
|
||||||
vblc.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(m.conf.Bucket, m.location)
|
|
||||||
}()
|
|
||||||
m.init = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) Engine() string {
|
|
||||||
return "minio"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) PartLimit() *s3.PartLimit {
|
|
||||||
return &s3.PartLimit{
|
|
||||||
MinPartSize: minPartSize,
|
|
||||||
MaxPartSize: maxPartSize,
|
|
||||||
MaxNumSize: maxNumSize,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
uploadID, err := m.core.NewMultipartUpload(ctx, m.bucket, name, minio.PutObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.InitiateMultipartUploadResult{
|
|
||||||
Bucket: m.bucket,
|
|
||||||
Key: name,
|
|
||||||
UploadID: uploadID,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
minioParts := make([]minio.CompletePart, len(parts))
|
|
||||||
for i, part := range parts {
|
|
||||||
minioParts[i] = minio.CompletePart{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
ETag: strings.ToLower(part.ETag),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
upload, err := m.core.CompleteMultipartUpload(ctx, m.bucket, name, uploadID, minioParts, minio.PutObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m.delObjectImageInfoKey(ctx, name, upload.Size)
|
|
||||||
return &s3.CompleteMultipartUploadResult{
|
|
||||||
Location: upload.Location,
|
|
||||||
Bucket: upload.Bucket,
|
|
||||||
Key: upload.Key,
|
|
||||||
ETag: strings.ToLower(upload.ETag),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) PartSize(ctx context.Context, size int64) (int64, error) {
|
|
||||||
if size <= 0 {
|
|
||||||
return 0, errors.New("size must be greater than 0")
|
|
||||||
}
|
|
||||||
if size > maxPartSize*maxNumSize {
|
|
||||||
return 0, fmt.Errorf("MINIO size must be less than the maximum allowed limit")
|
|
||||||
}
|
|
||||||
if size <= minPartSize*maxNumSize {
|
|
||||||
return minPartSize, nil
|
|
||||||
}
|
|
||||||
partSize := size / maxNumSize
|
|
||||||
if size%maxNumSize != 0 {
|
|
||||||
partSize++
|
|
||||||
}
|
|
||||||
return partSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
creds, err := m.opts.Creds.Get()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result := s3.AuthSignResult{
|
|
||||||
URL: m.signEndpoint + "/" + m.bucket + "/" + name,
|
|
||||||
Query: url.Values{"uploadId": {uploadID}},
|
|
||||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
|
||||||
}
|
|
||||||
for i, partNumber := range partNumbers {
|
|
||||||
rawURL := result.URL + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID
|
|
||||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
request.Header.Set("X-Amz-Content-Sha256", unsignedPayload)
|
|
||||||
request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, m.location, nil)
|
|
||||||
result.Parts[i] = s3.SignPart{
|
|
||||||
PartNumber: partNumber,
|
|
||||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
|
||||||
Header: request.Header,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m.prefix != "" {
|
|
||||||
result.URL = m.signEndpoint + m.prefix + "/" + m.bucket + "/" + name
|
|
||||||
}
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
rawURL, err := m.sign.PresignedPutObject(ctx, m.bucket, name, expire)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if m.prefix != "" {
|
|
||||||
rawURL.Path = path.Join(m.prefix, rawURL.Path)
|
|
||||||
}
|
|
||||||
return rawURL.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) DeleteObject(ctx context.Context, name string) error {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return m.core.Client.RemoveObject(ctx, m.bucket, name, minio.RemoveObjectOptions{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
info, err := m.core.Client.StatObject(ctx, m.bucket, name, minio.StatObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.ObjectInfo{
|
|
||||||
ETag: strings.ToLower(info.ETag),
|
|
||||||
Key: info.Key,
|
|
||||||
Size: info.Size,
|
|
||||||
LastModified: info.LastModified,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result, err := m.core.Client.CopyObject(ctx, minio.CopyDestOptions{
|
|
||||||
Bucket: m.bucket,
|
|
||||||
Object: dst,
|
|
||||||
}, minio.CopySrcOptions{
|
|
||||||
Bucket: m.bucket,
|
|
||||||
Object: src,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.CopyObjectInfo{
|
|
||||||
Key: dst,
|
|
||||||
ETag: strings.ToLower(result.ETag),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) IsNotFound(err error) bool {
|
|
||||||
switch e := errs.Unwrap(err).(type) {
|
|
||||||
case minio.ErrorResponse:
|
|
||||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
|
||||||
case *minio.ErrorResponse:
|
|
||||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return m.core.AbortMultipartUpload(ctx, m.bucket, name, uploadID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result, err := m.core.ListObjectParts(ctx, m.bucket, name, uploadID, partNumberMarker, maxParts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res := &s3.ListUploadedPartsResult{
|
|
||||||
Key: result.Key,
|
|
||||||
UploadID: result.UploadID,
|
|
||||||
MaxParts: result.MaxParts,
|
|
||||||
NextPartNumberMarker: result.NextPartNumberMarker,
|
|
||||||
UploadedParts: make([]s3.UploadedPart, len(result.ObjectParts)),
|
|
||||||
}
|
|
||||||
for i, part := range result.ObjectParts {
|
|
||||||
res.UploadedParts[i] = s3.UploadedPart{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
LastModified: part.LastModified,
|
|
||||||
ETag: part.ETag,
|
|
||||||
Size: part.Size,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) PresignedGetObject(ctx context.Context, name string, expire time.Duration, query url.Values) (string, error) {
|
|
||||||
if expire <= 0 {
|
|
||||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
|
||||||
} else if expire < time.Second {
|
|
||||||
expire = time.Second
|
|
||||||
}
|
|
||||||
var (
|
|
||||||
rawURL *url.URL
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if m.conf.PublicRead {
|
|
||||||
rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
|
|
||||||
} else {
|
|
||||||
rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if m.prefix != "" {
|
|
||||||
rawURL.Path = path.Join(m.prefix, rawURL.Path)
|
|
||||||
}
|
|
||||||
return rawURL.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
reqParams := make(url.Values)
|
|
||||||
if opt != nil {
|
|
||||||
if opt.ContentType != "" {
|
|
||||||
reqParams.Set("response-content-type", opt.ContentType)
|
|
||||||
}
|
|
||||||
if opt.Filename != "" {
|
|
||||||
reqParams.Set("response-content-disposition", `attachment; filename=`+strconv.Quote(opt.Filename))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if opt.Image == nil || (opt.Image.Width < 0 && opt.Image.Height < 0 && opt.Image.Format == "") || (opt.Image.Width > maxImageWidth || opt.Image.Height > maxImageHeight) {
|
|
||||||
return m.PresignedGetObject(ctx, name, expire, reqParams)
|
|
||||||
}
|
|
||||||
return m.getImageThumbnailURL(ctx, name, expire, opt.Image)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) getObjectData(ctx context.Context, name string, limit int64) ([]byte, error) {
|
|
||||||
object, err := m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer object.Close()
|
|
||||||
if limit < 0 {
|
|
||||||
return io.ReadAll(object)
|
|
||||||
}
|
|
||||||
return io.ReadAll(io.LimitReader(object, limit))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
|
|
||||||
if err := m.initMinio(ctx); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
policy := minio.NewPostPolicy()
|
|
||||||
if err := policy.SetKey(name); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
expires := time.Now().Add(duration)
|
|
||||||
if err := policy.SetExpires(expires); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if size > 0 {
|
|
||||||
if err := policy.SetContentLengthRange(0, size); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := policy.SetSuccessStatusAction(strconv.Itoa(successCode)); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if contentType != "" {
|
|
||||||
if err := policy.SetContentType(contentType); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := policy.SetBucket(m.bucket); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
u, fd, err := m.core.PresignedPostPolicy(ctx, policy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
sign, err := url.Parse(m.signEndpoint)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
u.Scheme = sign.Scheme
|
|
||||||
u.Host = sign.Host
|
|
||||||
return &s3.FormData{
|
|
||||||
URL: u.String(),
|
|
||||||
File: "file",
|
|
||||||
Header: nil,
|
|
||||||
FormData: fd,
|
|
||||||
Expires: expires,
|
|
||||||
SuccessCodes: []int{successCode},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@ -1,150 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package minio
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"image"
|
|
||||||
"image/gif"
|
|
||||||
"image/jpeg"
|
|
||||||
"image/png"
|
|
||||||
"net/url"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/servererrs"
|
|
||||||
"github.com/openimsdk/tools/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (m *Minio) getImageThumbnailURL(ctx context.Context, name string, expire time.Duration, opt *s3.Image) (string, error) {
|
|
||||||
var img image.Image
|
|
||||||
info, err := m.cache.GetImageObjectKeyInfo(ctx, name, func(ctx context.Context) (info *cache.MinioImageInfo, err error) {
|
|
||||||
info, img, err = m.getObjectImageInfo(ctx, name)
|
|
||||||
return
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !info.IsImg {
|
|
||||||
return "", servererrs.ErrData.WrapMsg("object not image")
|
|
||||||
}
|
|
||||||
if opt.Width > info.Width || opt.Width <= 0 {
|
|
||||||
opt.Width = info.Width
|
|
||||||
}
|
|
||||||
if opt.Height > info.Height || opt.Height <= 0 {
|
|
||||||
opt.Height = info.Height
|
|
||||||
}
|
|
||||||
opt.Format = strings.ToLower(opt.Format)
|
|
||||||
if opt.Format == formatJpg {
|
|
||||||
opt.Format = formatJpeg
|
|
||||||
}
|
|
||||||
switch opt.Format {
|
|
||||||
case formatPng, formatJpeg, formatGif:
|
|
||||||
default:
|
|
||||||
opt.Format = ""
|
|
||||||
}
|
|
||||||
reqParams := make(url.Values)
|
|
||||||
if opt.Width == info.Width && opt.Height == info.Height && (opt.Format == info.Format || opt.Format == "") {
|
|
||||||
reqParams.Set("response-content-type", "image/"+info.Format)
|
|
||||||
return m.PresignedGetObject(ctx, name, expire, reqParams)
|
|
||||||
}
|
|
||||||
if opt.Format == "" {
|
|
||||||
switch opt.Format {
|
|
||||||
case formatGif:
|
|
||||||
opt.Format = formatGif
|
|
||||||
case formatJpeg:
|
|
||||||
opt.Format = formatJpeg
|
|
||||||
case formatPng:
|
|
||||||
opt.Format = formatPng
|
|
||||||
default:
|
|
||||||
opt.Format = formatPng
|
|
||||||
}
|
|
||||||
}
|
|
||||||
key, err := m.cache.GetThumbnailKey(ctx, name, opt.Format, opt.Width, opt.Height, func(ctx context.Context) (string, error) {
|
|
||||||
if img == nil {
|
|
||||||
var reader *minio.Object
|
|
||||||
reader, err = m.core.Client.GetObject(ctx, m.bucket, name, minio.GetObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer reader.Close()
|
|
||||||
img, _, err = ImageStat(reader)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
thumbnail := resizeImage(img, opt.Width, opt.Height)
|
|
||||||
buf := bytes.NewBuffer(nil)
|
|
||||||
switch opt.Format {
|
|
||||||
case formatPng:
|
|
||||||
err = png.Encode(buf, thumbnail)
|
|
||||||
case formatJpeg:
|
|
||||||
err = jpeg.Encode(buf, thumbnail, nil)
|
|
||||||
case formatGif:
|
|
||||||
err = gif.Encode(buf, thumbnail, nil)
|
|
||||||
}
|
|
||||||
cacheKey := filepath.Join(imageThumbnailPath, info.Etag, fmt.Sprintf("image_w%d_h%d.%s", opt.Width, opt.Height, opt.Format))
|
|
||||||
if _, err = m.core.Client.PutObject(ctx, m.bucket, cacheKey, buf, int64(buf.Len()), minio.PutObjectOptions{}); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return cacheKey, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
reqParams.Set("response-content-type", "image/"+opt.Format)
|
|
||||||
return m.PresignedGetObject(ctx, key, expire, reqParams)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) getObjectImageInfo(ctx context.Context, name string) (*cache.MinioImageInfo, image.Image, error) {
|
|
||||||
fileInfo, err := m.StatObject(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
if fileInfo.Size > maxImageSize {
|
|
||||||
return nil, nil, errors.New("file size too large")
|
|
||||||
}
|
|
||||||
imageData, err := m.getObjectData(ctx, name, fileInfo.Size)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
var info cache.MinioImageInfo
|
|
||||||
imageInfo, format, err := ImageStat(bytes.NewReader(imageData))
|
|
||||||
if err == nil {
|
|
||||||
info.IsImg = true
|
|
||||||
info.Format = format
|
|
||||||
info.Width, info.Height = ImageWidthHeight(imageInfo)
|
|
||||||
} else {
|
|
||||||
info.IsImg = false
|
|
||||||
}
|
|
||||||
info.Etag = fileInfo.ETag
|
|
||||||
return &info, imageInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Minio) delObjectImageInfoKey(ctx context.Context, key string, size int64) {
|
|
||||||
if size > 0 && size > maxImageSize {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := m.cache.DelObjectImageInfoKey(key).ExecDel(ctx); err != nil {
|
|
||||||
log.ZError(ctx, "DelObjectImageInfoKey failed", err, "key", key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
// Copyright © 2024 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package oss
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
_ "unsafe"
|
|
||||||
|
|
||||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader
|
|
||||||
func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
|
|
||||||
|
|
||||||
//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
|
|
||||||
func getURLParams(c oss.Conn, params map[string]any) string
|
|
||||||
|
|
||||||
//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
|
|
||||||
func getURL(um urlMaker, bucket, object, params string) *url.URL
|
|
||||||
|
|
||||||
type urlMaker struct {
|
|
||||||
Scheme string
|
|
||||||
NetLoc string
|
|
||||||
Type int
|
|
||||||
IsProxy bool
|
|
||||||
}
|
|
||||||
@ -1,382 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package oss
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/sha1"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
|
||||||
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
|
|
||||||
"github.com/openimsdk/tools/errs"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
minPartSize int64 = 1024 * 1024 * 1 // 1MB
|
|
||||||
maxPartSize int64 = 1024 * 1024 * 1024 * 5 // 5GB
|
|
||||||
maxNumSize int64 = 10000
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
imagePng = "png"
|
|
||||||
imageJpg = "jpg"
|
|
||||||
imageJpeg = "jpeg"
|
|
||||||
imageGif = "gif"
|
|
||||||
imageWebp = "webp"
|
|
||||||
)
|
|
||||||
|
|
||||||
const successCode = http.StatusOK
|
|
||||||
|
|
||||||
type Config struct {
|
|
||||||
Endpoint string
|
|
||||||
Bucket string
|
|
||||||
BucketURL string
|
|
||||||
AccessKeyID string
|
|
||||||
AccessKeySecret string
|
|
||||||
SessionToken string
|
|
||||||
PublicRead bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOSS(conf Config) (s3.Interface, error) {
|
|
||||||
if conf.BucketURL == "" {
|
|
||||||
return nil, errs.Wrap(errors.New("bucket url is empty"))
|
|
||||||
}
|
|
||||||
client, err := oss.New(conf.Endpoint, conf.AccessKeyID, conf.AccessKeySecret)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
bucket, err := client.Bucket(conf.Bucket)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "ali-oss bucket error")
|
|
||||||
}
|
|
||||||
if conf.BucketURL[len(conf.BucketURL)-1] != '/' {
|
|
||||||
conf.BucketURL += "/"
|
|
||||||
}
|
|
||||||
return &OSS{
|
|
||||||
bucketURL: conf.BucketURL,
|
|
||||||
bucket: bucket,
|
|
||||||
credentials: client.Config.GetCredentials(),
|
|
||||||
um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
|
|
||||||
publicRead: conf.PublicRead,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type OSS struct {
|
|
||||||
bucketURL string
|
|
||||||
bucket *oss.Bucket
|
|
||||||
credentials oss.Credentials
|
|
||||||
um urlMaker
|
|
||||||
publicRead bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) Engine() string {
|
|
||||||
return "ali-oss"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) PartLimit() *s3.PartLimit {
|
|
||||||
return &s3.PartLimit{
|
|
||||||
MinPartSize: minPartSize,
|
|
||||||
MaxPartSize: maxPartSize,
|
|
||||||
MaxNumSize: maxNumSize,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
|
|
||||||
result, err := o.bucket.InitiateMultipartUpload(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.InitiateMultipartUploadResult{
|
|
||||||
UploadID: result.UploadID,
|
|
||||||
Bucket: result.Bucket,
|
|
||||||
Key: result.Key,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
|
|
||||||
ossParts := make([]oss.UploadPart, len(parts))
|
|
||||||
for i, part := range parts {
|
|
||||||
ossParts[i] = oss.UploadPart{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
ETag: strings.ToUpper(part.ETag),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result, err := o.bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{
|
|
||||||
UploadID: uploadID,
|
|
||||||
Bucket: o.bucket.BucketName,
|
|
||||||
Key: name,
|
|
||||||
}, ossParts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &s3.CompleteMultipartUploadResult{
|
|
||||||
Location: result.Location,
|
|
||||||
Bucket: result.Bucket,
|
|
||||||
Key: result.Key,
|
|
||||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) PartSize(ctx context.Context, size int64) (int64, error) {
|
|
||||||
if size <= 0 {
|
|
||||||
return 0, errs.Wrap(errors.New("size must be greater than 0"))
|
|
||||||
}
|
|
||||||
if size > maxPartSize*maxNumSize {
|
|
||||||
return 0, errs.Wrap(errors.New("size must be less than the maximum allowed limit"))
|
|
||||||
}
|
|
||||||
if size <= minPartSize*maxNumSize {
|
|
||||||
return minPartSize, nil
|
|
||||||
}
|
|
||||||
partSize := size / maxNumSize
|
|
||||||
if size%maxNumSize != 0 {
|
|
||||||
partSize++
|
|
||||||
}
|
|
||||||
return partSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
|
|
||||||
result := s3.AuthSignResult{
|
|
||||||
URL: o.bucketURL + name,
|
|
||||||
Query: url.Values{"uploadId": {uploadID}},
|
|
||||||
Header: make(http.Header),
|
|
||||||
Parts: make([]s3.SignPart, len(partNumbers)),
|
|
||||||
}
|
|
||||||
for i, partNumber := range partNumbers {
|
|
||||||
rawURL := fmt.Sprintf(`%s%s?partNumber=%d&uploadId=%s`, o.bucketURL, name, partNumber, uploadID)
|
|
||||||
request, err := http.NewRequest(http.MethodPut, rawURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if o.credentials.GetSecurityToken() != "" {
|
|
||||||
request.Header.Set(oss.HTTPHeaderOssSecurityToken, o.credentials.GetSecurityToken())
|
|
||||||
}
|
|
||||||
now := time.Now().UTC().Format(http.TimeFormat)
|
|
||||||
request.Header.Set(oss.HTTPHeaderHost, request.Host)
|
|
||||||
request.Header.Set(oss.HTTPHeaderDate, now)
|
|
||||||
request.Header.Set(oss.HttpHeaderOssDate, now)
|
|
||||||
signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
|
|
||||||
delete(request.Header, oss.HTTPHeaderDate)
|
|
||||||
result.Parts[i] = s3.SignPart{
|
|
||||||
PartNumber: partNumber,
|
|
||||||
Query: url.Values{"partNumber": {strconv.Itoa(partNumber)}},
|
|
||||||
URL: request.URL.String(),
|
|
||||||
Header: request.Header,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
|
|
||||||
return o.bucket.SignURL(name, http.MethodPut, int64(expire/time.Second))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
|
|
||||||
header, err := o.bucket.GetObjectMeta(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
res := &s3.ObjectInfo{Key: name}
|
|
||||||
if res.ETag = strings.ToLower(strings.ReplaceAll(header.Get("ETag"), `"`, ``)); res.ETag == "" {
|
|
||||||
return nil, errs.Wrap(errors.New("StatObject etag not found"))
|
|
||||||
}
|
|
||||||
if contentLengthStr := header.Get("Content-Length"); contentLengthStr == "" {
|
|
||||||
return nil, errors.New("StatObject content-length not found")
|
|
||||||
} else {
|
|
||||||
res.Size, err = strconv.ParseInt(contentLengthStr, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "StatObject content-length parse error")
|
|
||||||
}
|
|
||||||
if res.Size < 0 {
|
|
||||||
return nil, errs.Wrap(errors.New("StatObject content-length must be greater than 0"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if lastModified := header.Get("Last-Modified"); lastModified == "" {
|
|
||||||
return nil, errs.Wrap(errors.New("StatObject last-modified not found"))
|
|
||||||
} else {
|
|
||||||
res.LastModified, err = time.Parse(http.TimeFormat, lastModified)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "StatObject last-modified parse error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) DeleteObject(ctx context.Context, name string) error {
|
|
||||||
return o.bucket.DeleteObject(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
|
|
||||||
result, err := o.bucket.CopyObject(src, dst)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "CopyObject error")
|
|
||||||
}
|
|
||||||
return &s3.CopyObjectInfo{
|
|
||||||
Key: dst,
|
|
||||||
ETag: strings.ToLower(strings.ReplaceAll(result.ETag, `"`, ``)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) IsNotFound(err error) bool {
|
|
||||||
switch e := errs.Unwrap(err).(type) {
|
|
||||||
case oss.ServiceError:
|
|
||||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
|
||||||
case *oss.ServiceError:
|
|
||||||
return e.StatusCode == http.StatusNotFound || e.Code == "NoSuchKey"
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
|
|
||||||
return o.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{
|
|
||||||
UploadID: uploadID,
|
|
||||||
Key: name,
|
|
||||||
Bucket: o.bucket.BucketName,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
|
|
||||||
result, err := o.bucket.ListUploadedParts(oss.InitiateMultipartUploadResult{
|
|
||||||
UploadID: uploadID,
|
|
||||||
Key: name,
|
|
||||||
Bucket: o.bucket.BucketName,
|
|
||||||
}, oss.MaxUploads(100), oss.MaxParts(maxParts), oss.PartNumberMarker(partNumberMarker))
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "ListUploadedParts error")
|
|
||||||
}
|
|
||||||
res := &s3.ListUploadedPartsResult{
|
|
||||||
Key: result.Key,
|
|
||||||
UploadID: result.UploadID,
|
|
||||||
MaxParts: result.MaxParts,
|
|
||||||
UploadedParts: make([]s3.UploadedPart, len(result.UploadedParts)),
|
|
||||||
}
|
|
||||||
res.NextPartNumberMarker, _ = strconv.Atoi(result.NextPartNumberMarker)
|
|
||||||
for i, part := range result.UploadedParts {
|
|
||||||
res.UploadedParts[i] = s3.UploadedPart{
|
|
||||||
PartNumber: part.PartNumber,
|
|
||||||
LastModified: part.LastModified,
|
|
||||||
ETag: part.ETag,
|
|
||||||
Size: int64(part.Size),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
|
|
||||||
var opts []oss.Option
|
|
||||||
if opt != nil {
|
|
||||||
if opt.Image != nil {
|
|
||||||
// Docs Address: https://help.aliyun.com/zh/oss/user-guide/resize-images-4?spm=a2c4g.11186623.0.0.4b3b1e4fWW6yji
|
|
||||||
var format string
|
|
||||||
switch opt.Image.Format {
|
|
||||||
case
|
|
||||||
imagePng,
|
|
||||||
imageJpg,
|
|
||||||
imageJpeg,
|
|
||||||
imageGif,
|
|
||||||
imageWebp:
|
|
||||||
format = opt.Image.Format
|
|
||||||
default:
|
|
||||||
opt.Image.Format = imageJpg
|
|
||||||
}
|
|
||||||
// https://oss-console-img-demo-cn-hangzhou.oss-cn-hangzhou.aliyuncs.com/example.jpg?x-oss-process=image/resize,h_100,m_lfit
|
|
||||||
process := "image/resize,m_lfit"
|
|
||||||
if opt.Image.Width > 0 {
|
|
||||||
process += ",w_" + strconv.Itoa(opt.Image.Width)
|
|
||||||
}
|
|
||||||
if opt.Image.Height > 0 {
|
|
||||||
process += ",h_" + strconv.Itoa(opt.Image.Height)
|
|
||||||
}
|
|
||||||
process += ",format," + format
|
|
||||||
opts = append(opts, oss.Process(process))
|
|
||||||
}
|
|
||||||
if !o.publicRead {
|
|
||||||
if opt.ContentType != "" {
|
|
||||||
opts = append(opts, oss.ResponseContentType(opt.ContentType))
|
|
||||||
}
|
|
||||||
if opt.Filename != "" {
|
|
||||||
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if expire <= 0 {
|
|
||||||
expire = time.Hour * 24 * 365 * 99 // 99 years
|
|
||||||
} else if expire < time.Second {
|
|
||||||
expire = time.Second
|
|
||||||
}
|
|
||||||
if !o.publicRead {
|
|
||||||
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
|
|
||||||
}
|
|
||||||
rawParams, err := oss.GetRawParams(opts)
|
|
||||||
if err != nil {
|
|
||||||
return "", errs.WrapMsg(err, "AccessURL error")
|
|
||||||
}
|
|
||||||
params := getURLParams(*o.bucket.Client.Conn, rawParams)
|
|
||||||
return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (o *OSS) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
|
|
||||||
// https://help.aliyun.com/zh/oss/developer-reference/postobject?spm=a2c4g.11186623.0.0.1cb83cebkP55nn
|
|
||||||
expires := time.Now().Add(duration)
|
|
||||||
conditions := []any{
|
|
||||||
map[string]string{"bucket": o.bucket.BucketName},
|
|
||||||
map[string]string{"key": name},
|
|
||||||
}
|
|
||||||
if size > 0 {
|
|
||||||
conditions = append(conditions, []any{"content-length-range", 0, size})
|
|
||||||
}
|
|
||||||
policy := map[string]any{
|
|
||||||
"expiration": expires.Format("2006-01-02T15:04:05.000Z"),
|
|
||||||
"conditions": conditions,
|
|
||||||
}
|
|
||||||
policyJson, err := json.Marshal(policy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "Marshal json error")
|
|
||||||
}
|
|
||||||
policyStr := base64.StdEncoding.EncodeToString(policyJson)
|
|
||||||
h := hmac.New(sha1.New, []byte(o.credentials.GetAccessKeySecret()))
|
|
||||||
if _, err := io.WriteString(h, policyStr); err != nil {
|
|
||||||
return nil, errs.WrapMsg(err, "WriteString error")
|
|
||||||
}
|
|
||||||
fd := &s3.FormData{
|
|
||||||
URL: o.bucketURL,
|
|
||||||
File: "file",
|
|
||||||
Expires: expires,
|
|
||||||
FormData: map[string]string{
|
|
||||||
"key": name,
|
|
||||||
"policy": policyStr,
|
|
||||||
"OSSAccessKeyId": o.credentials.GetAccessKeyID(),
|
|
||||||
"success_action_status": strconv.Itoa(successCode),
|
|
||||||
"signature": base64.StdEncoding.EncodeToString(h.Sum(nil)),
|
|
||||||
},
|
|
||||||
SuccessCodes: []int{successCode},
|
|
||||||
}
|
|
||||||
if contentType != "" {
|
|
||||||
fd.FormData["x-oss-content-type"] = contentType
|
|
||||||
}
|
|
||||||
return fd, nil
|
|
||||||
}
|
|
||||||
@ -1,166 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package s3
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PartLimit struct {
|
|
||||||
MinPartSize int64 `json:"minPartSize"`
|
|
||||||
MaxPartSize int64 `json:"maxPartSize"`
|
|
||||||
MaxNumSize int64 `json:"maxNumSize"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type InitiateMultipartUploadResult struct {
|
|
||||||
Bucket string `json:"bucket"`
|
|
||||||
Key string `json:"key"`
|
|
||||||
UploadID string `json:"uploadID"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type MultipartUploadRequest struct {
|
|
||||||
UploadID string `json:"uploadId"`
|
|
||||||
Bucket string `json:"bucket"`
|
|
||||||
Key string `json:"key"`
|
|
||||||
Method string `json:"method"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Query url.Values `json:"query"`
|
|
||||||
Header http.Header `json:"header"`
|
|
||||||
PartKey string `json:"partKey"`
|
|
||||||
PartSize int64 `json:"partSize"`
|
|
||||||
FirstPart int `json:"firstPart"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Part struct {
|
|
||||||
PartNumber int `json:"partNumber"`
|
|
||||||
ETag string `json:"etag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type CompleteMultipartUploadResult struct {
|
|
||||||
Location string `json:"location"`
|
|
||||||
Bucket string `json:"bucket"`
|
|
||||||
Key string `json:"key"`
|
|
||||||
ETag string `json:"etag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SignResult struct {
|
|
||||||
Parts []SignPart `json:"parts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ObjectInfo struct {
|
|
||||||
ETag string `json:"etag"`
|
|
||||||
Key string `json:"name"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
LastModified time.Time `json:"lastModified"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type CopyObjectInfo struct {
|
|
||||||
Key string `json:"name"`
|
|
||||||
ETag string `json:"etag"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type FormData struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
File string `json:"file"`
|
|
||||||
Header http.Header `json:"header"`
|
|
||||||
FormData map[string]string `json:"form"`
|
|
||||||
Expires time.Time `json:"expires"`
|
|
||||||
SuccessCodes []int `json:"successActionStatus"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SignPart struct {
|
|
||||||
PartNumber int `json:"partNumber"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Query url.Values `json:"query"`
|
|
||||||
Header http.Header `json:"header"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuthSignResult struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
Query url.Values `json:"query"`
|
|
||||||
Header http.Header `json:"header"`
|
|
||||||
Parts []SignPart `json:"parts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type InitiateUpload struct {
|
|
||||||
UploadID string `json:"uploadId"`
|
|
||||||
Bucket string `json:"bucket"`
|
|
||||||
Key string `json:"key"`
|
|
||||||
Method string `json:"method"`
|
|
||||||
URL string `json:"url"`
|
|
||||||
Query url.Values `json:"query"`
|
|
||||||
Header http.Header `json:"header"`
|
|
||||||
PartKey string `json:"partKey"`
|
|
||||||
PartSize int64 `json:"partSize"`
|
|
||||||
FirstPart int `json:"firstPart"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type UploadedPart struct {
|
|
||||||
PartNumber int `json:"partNumber"`
|
|
||||||
LastModified time.Time `json:"lastModified"`
|
|
||||||
ETag string `json:"etag"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ListUploadedPartsResult struct {
|
|
||||||
Key string `xml:"Key"`
|
|
||||||
UploadID string `xml:"UploadId"`
|
|
||||||
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
|
|
||||||
MaxParts int `xml:"MaxParts"`
|
|
||||||
UploadedParts []UploadedPart `xml:"Part"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Image struct {
|
|
||||||
Format string `json:"format"`
|
|
||||||
Width int `json:"width"`
|
|
||||||
Height int `json:"height"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AccessURLOption struct {
|
|
||||||
ContentType string `json:"contentType"`
|
|
||||||
Filename string `json:"filename"`
|
|
||||||
Image *Image `json:"image"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Interface interface {
|
|
||||||
Engine() string
|
|
||||||
PartLimit() *PartLimit
|
|
||||||
|
|
||||||
InitiateMultipartUpload(ctx context.Context, name string) (*InitiateMultipartUploadResult, error)
|
|
||||||
CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []Part) (*CompleteMultipartUploadResult, error)
|
|
||||||
|
|
||||||
PartSize(ctx context.Context, size int64) (int64, error)
|
|
||||||
AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*AuthSignResult, error)
|
|
||||||
|
|
||||||
PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error)
|
|
||||||
|
|
||||||
DeleteObject(ctx context.Context, name string) error
|
|
||||||
|
|
||||||
CopyObject(ctx context.Context, src string, dst string) (*CopyObjectInfo, error)
|
|
||||||
|
|
||||||
StatObject(ctx context.Context, name string) (*ObjectInfo, error)
|
|
||||||
|
|
||||||
IsNotFound(err error) bool
|
|
||||||
|
|
||||||
AbortMultipartUpload(ctx context.Context, uploadID string, name string) error
|
|
||||||
ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*ListUploadedPartsResult, error)
|
|
||||||
|
|
||||||
AccessURL(ctx context.Context, name string, expire time.Duration, opt *AccessURLOption) (string, error)
|
|
||||||
|
|
||||||
FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*FormData, error)
|
|
||||||
}
|
|
||||||
Loading…
Reference in new issue