msb-public
/
cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Code Issues Projects Releases Wiki Activity

support lambda storage policy

Browse Source
pull/443/head
LittleAnt 5 years ago
parent 27bf8ca9b2
commit 832b6b3dda
6 changed files with 371 additions and 7 deletions
Show Stats Download Patch File Download Diff File

11
go.mod
Unescape View File

@ -5,6 +5,7 @@ go 1.13
require (
github.com/DATA-DOG/go-sqlmock v1.3.3
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible
github.com/aws/aws-sdk-go v1.31.9
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4
github.com/fatih/color v1.7.0
@ -15,7 +16,7 @@ require (
github.com/gin-gonic/gin v1.5.0
github.com/go-ini/ini v1.50.0
github.com/go-mail/mail v2.3.1+incompatible
github.com/gofrs/uuid v3.2.0+incompatible
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-querystring v1.0.0
github.com/gorilla/websocket v1.4.1
@ -25,17 +26,17 @@ require (
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/pkg/errors v0.8.0
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.2.0
github.com/qingwg/payjs v0.0.0-20190928033402-c53dbe16b371
github.com/qingwg/payjs v0.0.0-20190928033402-c53dbe16b371 // indirect
github.com/qiniu/api.v7/v7 v7.4.0
github.com/rafaeljusto/redigomock v0.0.0-20191117212112-00b2509252a1
github.com/rakyll/statik v0.1.7
github.com/robfig/cron/v3 v3.0.1
github.com/smartwalle/alipay/v3 v3.0.13
github.com/smartwalle/alipay/v3 v3.0.13 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/speps/go-hashids v2.0.0+incompatible
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.5.1
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible

11
go.sum
Unescape View File

@ -15,6 +15,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible h1:A3oZlWPD/Poa19FvNbw+Zu4yKAurDBTjlRDilYGBiS4=
github.com/aliyun/aliyun-oss-go-sdk v2.0.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.31.9 h1:n+b34ydVfgC30j0Qm69yaapmjejQPW2BoDBX7Uy/tLI=
github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -75,6 +77,8 @@ github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rm
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@ -128,6 +132,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -186,6 +192,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok=
@ -231,6 +239,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VBT7PCRBcBxkzlDOiTk7Td8ATiia1b1GuE=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
@ -268,6 +278,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

5
models/policy.go
Unescape View File

@ -58,6 +58,7 @@ var thumbSuffix = map[string][]string{
"upyun": {".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"remote": {},
"onedrive": {"*"},
"bos": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
}
func init() {
@ -203,7 +204,7 @@ func (policy *Policy) IsThumbExist(name string) bool {
// IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转
func (policy *Policy) IsTransitUpload(size uint64) bool {
if policy.Type == "local" {
if policy.Type == "local" || policy.Type == "bos" {
return true
}
if policy.Type == "onedrive" && size < 4*1024*1024 {
@ -236,7 +237,7 @@ func (policy *Policy) GetUploadURL() string {
var controller *url.URL
switch policy.Type {
case "local", "onedrive":
case "local", "onedrive", "bos":
return "/api/v3/file/upload"
case "remote":
controller, _ = url.Parse("/api/v3/slave/upload")

256
pkg/filesystem/driver/bos/handler.go
Unescape View File

@ -0,0 +1,256 @@
package bos
import (
"context"
"errors"
"fmt"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"io"
"net/url"
"os"
"path/filepath"
)
var UserPathPrefix = "user"
// Driver 策略适配器
type Driver struct {
Policy *model.Policy
ss *session.Session
}
// InitSess 初始化会话
func (handler *Driver) connect() {
if nil == handler.Policy {
panic(errors.New("存储策略为空"))
}
handler.ss = session.Must(session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""),
Endpoint: aws.String(handler.Policy.Server),
Region: aws.String(endpoints.CnNorth1RegionID),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true), // 这个必须设置为TRUE，坑了老半天
}))
}
// 刷新 S3 Gateway 服务的订单信息
func (handler *Driver) init() error {
if nil == handler.ss {
handler.connect()
}
svc := s3.New(handler.ss)
_, err := svc.ListBuckets(nil)
return err
}
func prefix(path string) string {
return UserPathPrefix + path
}
// List 列出BOS上存储的文件
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
// 刷新Buckets
if err := handler.init(); nil != err {
return nil, err
}
svc := s3.New(handler.ss)
resp, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(handler.Policy.BucketName),
Prefix: aws.String(prefix(path)),
})
if nil != err {
return nil, err
}
for _, item := range resp.Contents {
res = append(res, response.Object{
Name: filepath.Base(aws.StringValue(item.Key)),
Size: uint64(aws.Int64Value(item.Size)),
LastModify: aws.TimeValue(item.LastModified),
RelativePath: filepath.ToSlash(aws.StringValue(item.Key)),
})
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 刷新Buckets
if err := handler.init(); nil != err {
return nil, err
}
file, err := os.Create(filepath.Base(path))
if nil != err {
return nil, err
}
downloader := s3manager.NewDownloader(handler.ss)
_, err = downloader.Download(file, &s3.GetObjectInput{
Bucket: aws.String(handler.Policy.BucketName),
Key: aws.String(prefix(path)),
})
if nil != err {
return nil, err
}
return file, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
// 刷新Buckets
if err := handler.init(); nil != err {
return err
}
uploader := s3manager.NewUploader(handler.ss)
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(handler.Policy.BucketName),
Key: aws.String(prefix(dst)),
Body: file,
})
return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return files, errors.New("Lambda Chain 暂不支持删除操作")
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
//if nil == handler.ss {
// handler.connect()
//}
//
//svc := s3.New(handler.ss)
//result, err := svc.ListBuckets(nil)
//if nil != err {
// return nil, err
//}
//
//for _, b := range result.Buckets {
// log.Println(aws.StringValue(b.Name))
//
// resp, err := svc.ListObjects(&s3.ListObjectsInput{
// Bucket: b.Name,
// })
//
// if nil != err {
// log.Println("Error:", err)
// continue
// }
//
// for _, item := range resp.Contents {
// log.Println("Key:", aws.StringValue(item.Key))
// log.Println("Size:", aws.Int64Value(item.Size))
// log.Println("Class:", aws.StringValue(item.StorageClass))
// log.Println("LastTime:", aws.TimeValue(item.LastModified))
// }
//}
return &response.ContentResponse{
Redirect: true,
URL: "",
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(ctx context.Context, path string, baseURL url.URL, ttl int64, isDownload bool, speed int) (string, error) {
file, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return "", errors.New("无法获取文件记录上下文")
}
var (
signedURI *url.URL
err error
)
if isDownload {
// 创建下载会话，将文件信息写入缓存
downloadSessionID := util.RandStringRunes(16)
err = cache.Set("download_"+downloadSessionID, file, int(ttl))
if err != nil {
return "", serializer.NewError(serializer.CodeCacheOperation, "无法创建下载会话", err)
}
// 签名生成文件记录
signedURI, err = auth.SignURI(
auth.General,
fmt.Sprintf("/api/v3/file/download/%s", downloadSessionID),
ttl,
)
}
if nil != err {
return "", serializer.NewError(serializer.CodeEncryptError, "无法对URL进行签名", err)
}
finalURL := baseURL.ResolveReference(signedURI).String()
return finalURL, nil
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, TTL int64, key string) (serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/bos/" + key)
apiURL := siteURL.ResolveReference(apiBaseURI).String()
return serializer.UploadCredential{
Callback: apiURL,
Key: key,
}, nil
}
// Meta 获取文件信息用于回调验证
// Cloudreve 本身不允许同名文件存在，所以通过Cloudreve上传到
// Lambda BOS存储的文件原则上也不会有重名，所以用文件名称和大
// 小做验证信息，比较粗劣，后续优化
//func (handler Driver) Meta(path string) (uint64, error) {
// // 刷新Buckets
// if err := handler.init(); nil != err {
// return 0, err
// }
//
// dir, _ := filepath.Split(path)
// svc := s3.New(handler.ss)
// resp, err := svc.ListObjects(&s3.ListObjectsInput{
// Bucket: aws.String(handler.Policy.BucketName),
// Prefix: aws.String(prefix(dir)),
// })
// if nil != err {
// return 0, err
// }
//
// for _, item := range resp.Contents {
// if path == aws.StringValue(item.Key) {
// return uint64(aws.Int64Value(item.Size)), nil
// }
// }
//
// return 0, errors.New("文件验证异常")
//}

90
pkg/filesystem/driver/bos/handler_test.go
Unescape View File

@ -0,0 +1,90 @@
package bos
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/stretchr/testify/assert"
"io/ioutil"
"log"
"strings"
"testing"
)
func TestDriver_Get(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "accesskey",
SecretKey: "secretkey",
BucketName: "file",
Server: "http://39.105.48.70:9002",
},
}
path := "TestFile.txt_44c873c0fe0a4733a28e701c4e24cf6d"
resp, err := handler.Get(context.Background(), path)
asserts.Error(err)
asserts.Nil(resp)
}
func TestDriver_Put(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "accesskey",
SecretKey: "secretkey",
BucketName: "file",
Server: "http://39.105.48.70:9002",
},
}
dst := "TestFile.txt"
err := handler.Put(context.Background(), ioutil.NopCloser(strings.NewReader("666")), dst, 3)
asserts.Error(err)
}
func TestDriver_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "accesskey",
SecretKey: "secretkey",
Server: "http://39.105.48.70:9002",
},
}
resp, err := handler.Thumb(context.Background(), "./999.txt")
asserts.Error(err)
asserts.Nil(resp)
}
func TestDriver_Delete(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "accesskey",
SecretKey: "secretkey",
BucketName: "file",
Server: "http://39.105.48.70:9002",
},
}
resp, err := handler.Delete(context.Background(), []string{"TestFile.txt_3e46f40e0e774376979add820f142653"})
asserts.Error(err)
log.Println(resp)
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "accesskey",
SecretKey: "secretkey",
BucketName: "439C4A6DC3F38B825D03F0357729A1E22B39FFCE",
Server: "http://123.56.171.188:9002",
},
}
resp, err := handler.List(context.Background(), "1", false)
asserts.Error(err)
log.Println(resp)
}

5
pkg/filesystem/filesystem.go
Unescape View File

@ -6,6 +6,7 @@ import (
"github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/bos"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/cos"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/onedrive"
@ -219,6 +220,10 @@ func (fs *FileSystem) DispatchHandler() error {
HTTPClient: request.HTTPClient{},
}
return nil
case "bos":
fs.Handler = bos.Driver{
Policy: currentPolicy,
}
default:
return ErrUnknownPolicyType
}

Write Preview
Loading…
Cancel
Save