From 752ce5ce622a31a1947a313a969798fe09204c09 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Tue, 21 Jan 2020 15:36:04 +0800 Subject: [PATCH] Feat: resume upload in server side --- models/migration.go | 1 + pkg/filesystem/driver/onedrive/api.go | 112 +++++++++++++++++++-- pkg/filesystem/driver/onedrive/handller.go | 48 +++++++-- pkg/filesystem/driver/onedrive/types.go | 16 +++ pkg/filesystem/filesystem.go | 5 +- pkg/request/request.go | 14 ++- pkg/webdav/prop.go | 2 +- service/callback/upload.go | 2 +- 8 files changed, 178 insertions(+), 22 deletions(-) diff --git a/models/migration.go b/models/migration.go index c1c3620..5ca26a4 100644 --- a/models/migration.go +++ b/models/migration.go @@ -109,6 +109,7 @@ solid #e9e9e9;"bgcolor="#fff">= 300 { decodeErr = json.Unmarshal([]byte(respBody), &errResp) if decodeErr != nil { return "", sysError(decodeErr) @@ -388,7 +478,7 @@ func (client *Client) request(ctx context.Context, method string, url string, bo func (client *Client) requestWithStr(ctx context.Context, method string, url string, body string, expectedCode int) (string, *RespError) { // 发送请求 bodyReader := ioutil.NopCloser(strings.NewReader(body)) - return client.request(ctx, method, url, bodyReader, expectedCode, + return client.request(ctx, method, url, bodyReader, request.WithContentLength(int64(len(body))), ) } diff --git a/pkg/filesystem/driver/onedrive/handller.go b/pkg/filesystem/driver/onedrive/handller.go index 27f0525..f0c0516 100644 --- a/pkg/filesystem/driver/onedrive/handller.go +++ b/pkg/filesystem/driver/onedrive/handller.go @@ -6,6 +6,7 @@ import ( model "github.com/HFO4/cloudreve/models" "github.com/HFO4/cloudreve/pkg/filesystem/fsctx" "github.com/HFO4/cloudreve/pkg/filesystem/response" + "github.com/HFO4/cloudreve/pkg/request" "github.com/HFO4/cloudreve/pkg/serializer" "io" "net/url" @@ -13,20 +14,51 @@ import ( // Driver OneDrive 适配器 type Driver struct { - Policy *model.Policy - Client *Client + Policy *model.Policy + Client *Client + HTTPClient request.Client } // Get 获取文件 func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { - return nil, errors.New("未实现") + // 获取文件源地址 + downloadURL, err := handler.Source( + ctx, + path, + url.URL{}, + int64(model.GetIntSetting("preview_timeout", 60)), + false, + 0, + ) + if err != nil { + return nil, err + } + + // 获取文件数据流 + resp, err := handler.HTTPClient.Request( + "GET", + downloadURL, + nil, + request.WithContext(ctx), + ).CheckHTTPResponse(200).GetRSCloser() + if err != nil { + return nil, err + } + + resp.SetFirstFakeChunk() + + // 尝试自主获取文件大小 + if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { + resp.SetContentLength(int64(file.Size)) + } + + return resp, nil } // Put 将文件流保存到指定目录 func (handler Driver) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error { defer file.Close() - _, err := handler.Client.PutFile(ctx, dst, file) - return err + return handler.Client.Upload(ctx, dst, int(size), file) } // Delete 删除一个或多个文件， @@ -67,7 +99,11 @@ func (handler Driver) Source( isDownload bool, speed int, ) (string, error) { - return "", errors.New("未实现") + res, err := handler.Client.Meta(ctx, "", path) + if err == nil { + return res.DownloadURL, nil + } + return "", err } // Token 获取上传会话URL diff --git a/pkg/filesystem/driver/onedrive/types.go b/pkg/filesystem/driver/onedrive/types.go index b7b0970..5386c93 100644 --- a/pkg/filesystem/driver/onedrive/types.go +++ b/pkg/filesystem/driver/onedrive/types.go @@ -1,6 +1,7 @@ package onedrive import ( + "io" "sync" ) @@ -29,6 +30,7 @@ type FileInfo struct { Size uint64 `json:"size"` Image imageInfo `json:"image"` ParentReference parentReference `json:"parentReference"` + DownloadURL string `json:"@microsoft.graph.downloadUrl"` } type imageInfo struct { @@ -79,4 +81,18 @@ type ThumbResponse struct { Value []map[string]interface{} `json:"value"` } +// Chunk 文件分片 +type Chunk struct { + Offset int + ChunkSize int + Total int + Retried int + Reader io.Reader +} + +// IsLast 返回是否为最后一个分片 +func (chunk *Chunk) IsLast() bool { + return chunk.Total-chunk.Offset == chunk.ChunkSize +} + var callbackSignal sync.Map diff --git a/pkg/filesystem/filesystem.go b/pkg/filesystem/filesystem.go index 4c32883..a2599c3 100644 --- a/pkg/filesystem/filesystem.go +++ b/pkg/filesystem/filesystem.go @@ -186,8 +186,9 @@ func (fs *FileSystem) DispatchHandler() error { case "onedrive": client, err := onedrive.NewClient(currentPolicy) fs.Handler = onedrive.Driver{ - Policy: currentPolicy, - Client: client, + Policy: currentPolicy, + Client: client, + HTTPClient: request.HTTPClient{}, } return err default: diff --git a/pkg/request/request.go b/pkg/request/request.go index a1dfc34..98a523d 100644 --- a/pkg/request/request.go +++ b/pkg/request/request.go @@ -85,7 +85,19 @@ func WithCredential(instance auth.Auth, ttl int64) Option { // WithHeader 设置请求Header func WithHeader(header http.Header) Option { return optionFunc(func(o *options) { - o.header = header + for k, v := range header { + o.header[k] = v + } + }) +} + +// WithoutHeader 设置清除请求Header +func WithoutHeader(header []string) Option { + return optionFunc(func(o *options) { + for _, v := range header { + delete(o.header, v) + } + }) } diff --git a/pkg/webdav/prop.go b/pkg/webdav/prop.go index 7921d9a..636745f 100644 --- a/pkg/webdav/prop.go +++ b/pkg/webdav/prop.go @@ -280,7 +280,7 @@ loop: // The file doesn't implement the optional DeadPropsHolder interface, so // all patches are forbidden. - pstat := Propstat{Status: http.StatusForbidden} + pstat := Propstat{Status: http.StatusOK} for _, patch := range patches { for _, p := range patch.Props { pstat.Props = append(pstat.Props, Property{XMLName: p.XMLName}) diff --git a/service/callback/upload.go b/service/callback/upload.go index 3db4d55..226b79a 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -160,7 +160,7 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response callbackSession := callbackSessionRaw.(*serializer.UploadSession) // 获取文件信息 - info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), service.ID) + info, err := fs.Handler.(onedrive.Driver).Client.Meta(context.Background(), service.ID, "") if err != nil { return serializer.Err(serializer.CodeUploadFailed, "文件元信息查询失败", err) }