From 0c821347484bac874146657dcd765e258eece933 Mon Sep 17 00:00:00 2001 From: Sam <100821827+01101sam@users.noreply.github.com> Date: Tue, 15 Aug 2023 01:29:46 +0800 Subject: [PATCH] fix(thumb - speedup): Consistent with author's request changes --- pkg/filesystem/image.go | 44 +++++++++++++++++----------------------- pkg/thumb/builtin.go | 4 ++-- pkg/thumb/ffmpeg.go | 5 ++++- pkg/thumb/libreoffice.go | 2 +- pkg/thumb/pipeline.go | 18 ++++++++++------ pkg/thumb/vips.go | 2 +- 6 files changed, 39 insertions(+), 36 deletions(-) diff --git a/pkg/filesystem/image.go b/pkg/filesystem/image.go index 1ee23b2d..8202d172 100644 --- a/pkg/filesystem/image.go +++ b/pkg/filesystem/image.go @@ -16,7 +16,7 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" "github.com/cloudreve/Cloudreve/v3/pkg/thumb" "github.com/cloudreve/Cloudreve/v3/pkg/util" -) + ) /* ================ 图像处理相关 @@ -117,41 +117,35 @@ func (fs *FileSystem) generateThumbnail(ctx context.Context, file *model.File) e defer cancel() // TODO: check file size - getThumbWorker().addWorker() - defer getThumbWorker().releaseWorker() - - // 获取文件数据 - source, err := fs.Handler.Get(newCtx, file.SourceName) - if err != nil { - return fmt.Errorf("faield to fetch original file %q: %w", file.SourceName, err) - } - defer source.Close() - // Provide file source path for local policy files - src := "" + var err error + url, src := "", "" if conf.SystemConfig.Mode == "slave" || file.GetPolicy().Type == "local" { src = file.SourceName } else { - ffmpegFormats := strings.Split(model.GetSettingByName("thumb_ffmpeg_exts"), ",") - vipsFormats := strings.Split(model.GetSettingByName("thumb_vips_exts"), ",") - // 如果文件格式支持 ffmpeg 或 vips 生成缩略图，则使用源文件链接 - if (util.IsInExtensionList(ffmpegFormats, file.Name) && - model.IsTrueVal(model.GetSettingByName("thumb_ffmpeg_enabled"))) || - (util.IsInExtensionList(vipsFormats, file.Name) && - model.IsTrueVal(model.GetSettingByName("thumb_vips_enabled"))) { - src, err = fs.Handler.Source(ctx, file.SourceName, 300, false, 0) - if err != nil { - util.Log().Warning("failed to get source url, fallback to source download method: %w", err) - } + url, err = fs.Handler.Source(ctx, file.SourceName, 300, false, 0) + if err != nil { + util.Log().Warning("failed to get slave file download url: %w", err) } } - if src == "" && file.Size > uint64(model.GetIntSetting("thumb_max_src_size", 31457280)) { + // Only check max size for file that will touch local fs + if url == "" && file.Size > uint64(model.GetIntSetting("thumb_max_src_size", 31457280)) { _ = updateThumbStatus(file, model.ThumbStatusNotAvailable) return errors.New("file too large") } - thumbRes, err := thumb.Generators.Generate(ctx, source, src, file.Name, model.GetSettingByNames( + getThumbWorker().addWorker() + defer getThumbWorker().releaseWorker() + + // 获取文件数据 + source, err := fs.Handler.Get(newCtx, file.SourceName) + if err != nil { + return fmt.Errorf("failed to fetch original file %q: %w", file.SourceName, err) + } + defer source.Close() + + thumbRes, err := thumb.Generators.Generate(ctx, source, src, url, file, model.GetSettingByNames( "thumb_width", "thumb_height", "thumb_builtin_enabled", diff --git a/pkg/thumb/builtin.go b/pkg/thumb/builtin.go index 206d0465..e657688e 100644 --- a/pkg/thumb/builtin.go +++ b/pkg/thumb/builtin.go @@ -3,7 +3,7 @@ package thumb import ( "context" "fmt" - "image" + "image" "image/gif" "image/jpeg" "image/png" @@ -157,7 +157,7 @@ func (image *Thumb) CreateAvatar(uid uint) error { type Builtin struct{} -func (b Builtin) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) { +func (b Builtin) Generate(ctx context.Context, file io.Reader, src, url, name string, options map[string]string) (*Result, error) { img, err := NewThumbFromFile(file, name) if err != nil { return nil, err diff --git a/pkg/thumb/ffmpeg.go b/pkg/thumb/ffmpeg.go index d98f107b..3b7fc0f0 100644 --- a/pkg/thumb/ffmpeg.go +++ b/pkg/thumb/ffmpeg.go @@ -23,7 +23,7 @@ type FfmpegGenerator struct { lastRawExts string } -func (f *FfmpegGenerator) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) { +func (f *FfmpegGenerator) Generate(ctx context.Context, file io.Reader, src, url, name string, options map[string]string) (*Result, error) { ffmpegOpts := model.GetSettingByNames("thumb_ffmpeg_path", "thumb_ffmpeg_exts", "thumb_ffmpeg_seek", "thumb_encode_method", "temp_path") if f.lastRawExts != ffmpegOpts["thumb_ffmpeg_exts"] { @@ -41,6 +41,9 @@ func (f *FfmpegGenerator) Generate(ctx context.Context, file io.Reader, src, nam ) tempInputPath := src + if url != "" { + tempInputPath = url + } if tempInputPath == "" { // If not local policy files, download to temp folder tempInputPath = filepath.Join( diff --git a/pkg/thumb/libreoffice.go b/pkg/thumb/libreoffice.go index 75e871bd..b79ddbd9 100644 --- a/pkg/thumb/libreoffice.go +++ b/pkg/thumb/libreoffice.go @@ -23,7 +23,7 @@ type LibreOfficeGenerator struct { lastRawExts string } -func (l *LibreOfficeGenerator) Generate(ctx context.Context, file io.Reader, src string, name string, options map[string]string) (*Result, error) { +func (l *LibreOfficeGenerator) Generate(ctx context.Context, file io.Reader, src, url, name string, options map[string]string) (*Result, error) { sofficeOpts := model.GetSettingByNames("thumb_libreoffice_path", "thumb_libreoffice_exts", "thumb_encode_method", "temp_path") if l.lastRawExts != sofficeOpts["thumb_libreoffice_exts"] { diff --git a/pkg/thumb/pipeline.go b/pkg/thumb/pipeline.go index 8ea1fd5d..89f534ed 100644 --- a/pkg/thumb/pipeline.go +++ b/pkg/thumb/pipeline.go @@ -18,7 +18,7 @@ import ( type Generator interface { // Generate generates a thumbnail for a given reader. Src is the original file path, only provided // for local policy files. - Generate(ctx context.Context, file io.Reader, src string, name string, options map[string]string) (*Result, error) + Generate(ctx context.Context, file io.Reader, src, url, name string, options map[string]string) (*Result, error) // Priority of execution order, smaller value means higher priority. Priority() int @@ -63,18 +63,24 @@ func RegisterGenerator(generator Generator) { sort.Sort(Generators) } -func (p GeneratorList) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) { - inputFile, inputSrc, inputName := file, src, name +func (p GeneratorList) Generate(ctx context.Context, fileReader io.Reader, src, url string, file *model.File, options map[string]string) (*Result, error) { + inputFile, inputSrc, inputUrl, inputName := fileReader, src, url, file.Name for _, generator := range p { if model.IsTrueVal(options[generator.EnableFlag()]) { - res, err := generator.Generate(ctx, inputFile, inputSrc, inputName, options) + if generator.EnableFlag() == "thumb_ffmpeg_enabled" && + file.Size > uint64(model.GetIntSetting("thumb_max_src_size", 31457280)) { + util.Log().Debug("Generator %s failed to generate thumbnail for %s since file too large, passing through to next generator.", reflect.TypeOf(generator).String(), inputName) + continue + } + + res, err := generator.Generate(ctx, inputFile, inputSrc, inputUrl, inputName, options) if errors.Is(err, ErrPassThrough) { - util.Log().Debug("Failed to generate thumbnail using %s for %s: %s, passing through to next generator.", reflect.TypeOf(generator).String(), name, err) + util.Log().Debug("Failed to generate thumbnail using %s for %s: %s, passing through to next generator.", reflect.TypeOf(generator).String(), inputName, err) continue } if res != nil && res.Continue { - util.Log().Debug("Generator %s for %s returned continue, passing through to next generator.", reflect.TypeOf(generator).String(), name) + util.Log().Debug("Generator %s for %s returned continue, passing through to next generator.", reflect.TypeOf(generator).String(), inputName) // defer cleanup funcs for _, cleanup := range res.Cleanup { diff --git a/pkg/thumb/vips.go b/pkg/thumb/vips.go index ac43535e..e678e1c0 100644 --- a/pkg/thumb/vips.go +++ b/pkg/thumb/vips.go @@ -22,7 +22,7 @@ type VipsGenerator struct { lastRawExts string } -func (v *VipsGenerator) Generate(ctx context.Context, file io.Reader, src, name string, options map[string]string) (*Result, error) { +func (v *VipsGenerator) Generate(ctx context.Context, file io.Reader, src, url, name string, options map[string]string) (*Result, error) { vipsOpts := model.GetSettingByNames("thumb_vips_path", "thumb_vips_exts", "thumb_encode_quality", "thumb_encode_method", "temp_path") if v.lastRawExts != vipsOpts["thumb_vips_exts"] {