|
|
|
@ -7,8 +7,11 @@ import (
|
|
|
|
|
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
|
|
|
|
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
|
|
|
|
"io"
|
|
|
|
|
"os"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const bufferTempPattern = "cdChunk.*.tmp"
|
|
|
|
|
|
|
|
|
|
// ChunkProcessFunc callback function for processing a chunk
|
|
|
|
|
type ChunkProcessFunc func(c *ChunkGroup, chunk io.Reader) error
|
|
|
|
|
|
|
|
|
@ -17,19 +20,22 @@ type ChunkGroup struct {
|
|
|
|
|
file fsctx.FileHeader
|
|
|
|
|
chunkSize uint64
|
|
|
|
|
backoff backoff.Backoff
|
|
|
|
|
enableRetryBuffer bool
|
|
|
|
|
|
|
|
|
|
fileInfo *fsctx.UploadTaskInfo
|
|
|
|
|
currentIndex int
|
|
|
|
|
chunkNum uint64
|
|
|
|
|
bufferTemp *os.File
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff) *ChunkGroup {
|
|
|
|
|
func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Backoff, useBuffer bool) *ChunkGroup {
|
|
|
|
|
c := &ChunkGroup{
|
|
|
|
|
file: file,
|
|
|
|
|
chunkSize: chunkSize,
|
|
|
|
|
backoff: backoff,
|
|
|
|
|
fileInfo: file.Info(),
|
|
|
|
|
currentIndex: -1,
|
|
|
|
|
enableRetryBuffer: useBuffer,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if c.chunkSize == 0 {
|
|
|
|
@ -44,14 +50,54 @@ func NewChunkGroup(file fsctx.FileHeader, chunkSize uint64, backoff backoff.Back
|
|
|
|
|
return c
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TempAvailable returns if current chunk temp file is available to be read
|
|
|
|
|
func (c *ChunkGroup) TempAvailable() bool {
|
|
|
|
|
if c.bufferTemp != nil {
|
|
|
|
|
state, _ := c.bufferTemp.Stat()
|
|
|
|
|
return state != nil && state.Size() == c.Length()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Process a chunk with retry logic
|
|
|
|
|
func (c *ChunkGroup) Process(processor ChunkProcessFunc) error {
|
|
|
|
|
err := processor(c, io.LimitReader(c.file, int64(c.chunkSize)))
|
|
|
|
|
reader := io.LimitReader(c.file, int64(c.chunkSize))
|
|
|
|
|
|
|
|
|
|
// If useBuffer is enabled, tee the reader to a temp file
|
|
|
|
|
if c.enableRetryBuffer && c.bufferTemp == nil && !c.file.Seekable() {
|
|
|
|
|
c.bufferTemp, _ = os.CreateTemp("", bufferTempPattern)
|
|
|
|
|
reader = io.TeeReader(reader, c.bufferTemp)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if c.bufferTemp != nil {
|
|
|
|
|
defer func() {
|
|
|
|
|
if c.bufferTemp != nil {
|
|
|
|
|
c.bufferTemp.Close()
|
|
|
|
|
os.Remove(c.bufferTemp.Name())
|
|
|
|
|
c.bufferTemp = nil
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
// if temp buffer file is available, use it
|
|
|
|
|
if c.TempAvailable() {
|
|
|
|
|
if _, err := c.bufferTemp.Seek(0, io.SeekStart); err != nil {
|
|
|
|
|
return fmt.Errorf("failed to seek temp file back to chunk start: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
util.Log().Debug("Chunk %d will be read from temp file %q.", c.Index(), c.bufferTemp.Name())
|
|
|
|
|
reader = c.bufferTemp
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err := processor(c, reader)
|
|
|
|
|
if err != nil {
|
|
|
|
|
if err != context.Canceled && c.file.Seekable() && c.backoff.Next() {
|
|
|
|
|
if err != context.Canceled && (c.file.Seekable() || c.TempAvailable()) && c.backoff.Next() {
|
|
|
|
|
if c.file.Seekable() {
|
|
|
|
|
if _, seekErr := c.file.Seek(c.Start(), io.SeekStart); seekErr != nil {
|
|
|
|
|
return fmt.Errorf("failed to seek back to chunk start: %w, last error: %w", seekErr, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
util.Log().Debug("Retrying chunk %d, last error: %s", c.currentIndex, err)
|
|
|
|
|
return c.Process(processor)
|
|
|
|
|