From 62b73b577b3b5ef43618a687e7fad615d82c2ec2 Mon Sep 17 00:00:00 2001 From: Aaron Liu <912394456@qq.com> Date: Fri, 7 Apr 2023 19:26:39 +0800 Subject: [PATCH] feat(thumb): generate and return sidecar thumb --- bootstrap/init.go | 14 +++-- go.mod | 6 +- go.sum | 6 ++ models/defaults.go | 2 + models/file.go | 5 ++ models/policy.go | 12 ++++ pkg/cache/driver.go | 12 ++-- pkg/filesystem/driver/local/handler.go | 2 +- pkg/filesystem/driver/remote/handler.go | 2 +- pkg/filesystem/image.go | 81 +++++++++++++++++-------- pkg/thumb/builtin.go | 24 ++++++-- pkg/thumb/pipeline.go | 10 +-- 12 files changed, 127 insertions(+), 49 deletions(-) diff --git a/bootstrap/init.go b/bootstrap/init.go index a9e7c21..4ee57d5 100644 --- a/bootstrap/init.go +++ b/bootstrap/init.go @@ -39,19 +39,25 @@ func Init(path string, statics fs.FS) { { "both", func() { - cache.Init(conf.SystemConfig.Mode == "slave") + cache.Init() }, }, { - "master", + "slave", func() { - model.Init() + model.InitSlaveDefaults() }, }, { "slave", func() { - model.InitSlaveDefaults() + cache.InitSlaveOverwrites() + }, + }, + { + "master", + func() { + model.Init() }, }, { diff --git a/go.mod b/go.mod index 08478fe..b613e35 100644 --- a/go.mod +++ b/go.mod @@ -119,6 +119,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/samber/lo v1.38.1 // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/soheilhy/cmux v0.1.5 // indirect @@ -146,13 +147,14 @@ require ( go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.16.0 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect - golang.org/x/mod v0.4.2 // indirect + golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect + golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.0 // indirect + golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a // indirect diff --git a/go.sum b/go.sum index 4d35a94..1421f3d 100644 --- a/go.sum +++ b/go.sum @@ -776,6 +776,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= @@ -988,6 +990,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1018,6 +1022,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1254,6 +1259,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/models/defaults.go b/models/defaults.go index c40f346..e63e2e4 100644 --- a/models/defaults.go +++ b/models/defaults.go @@ -112,6 +112,8 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti {Name: "thumb_ffmpeg_enabled", Value: "0", Type: "thumb"}, {Name: "thumb_vips_path", Value: "vips", Type: "thumb"}, {Name: "thumb_ffmpeg_path", Value: "ffmpeg", Type: "thumb"}, + {Name: "thumb_proxy_enabled", Value: "0", Type: "thumb"}, + {Name: "thumb_proxy_policy", Value: "[]", Type: "thumb"}, {Name: "pwa_small_icon", Value: "/static/img/favicon.ico", Type: "pwa"}, {Name: "pwa_medium_icon", Value: "/static/img/logo192.png", Type: "pwa"}, {Name: "pwa_large_icon", Value: "/static/img/logo512.png", Type: "pwa"}, diff --git a/models/file.go b/models/file.go index 349504f..9000dbc 100644 --- a/models/file.go +++ b/models/file.go @@ -466,3 +466,8 @@ func (file *File) GetPosition() string { func (file *File) ShouldLoadThumb() bool { return file.MetadataSerialized[ThumbStatusMetadataKey] != ThumbStatusNotAvailable } + +// return sidecar thumb file name +func (file *File) ThumbFile() string { + return file.SourceName + GetSettingByNameWithDefault("thumb_file_suffix", "._thumb") +} diff --git a/models/policy.go b/models/policy.go index 20e4631..706ae35 100644 --- a/models/policy.go +++ b/models/policy.go @@ -4,6 +4,7 @@ import ( "encoding/gob" "encoding/json" "github.com/gofrs/uuid" + "github.com/samber/lo" "path" "path/filepath" "strconv" @@ -227,3 +228,14 @@ func (policy *Policy) UpdateAccessKeyAndClearCache(s string) error { func (policy *Policy) ClearCache() { cache.Deletes([]string{strconv.FormatUint(uint64(policy.ID), 10)}, "policy_") } + +// CouldProxyThumb return if proxy thumbs is allowed for this policy. +func (policy *Policy) CouldProxyThumb() bool { + if policy.Type == "local" || !IsTrueVal(GetSettingByName("thumb_proxy_enabled")) { + return false + } + + allowed := make([]uint, 0) + _ = json.Unmarshal([]byte(GetSettingByName("thumb_proxy_policy")), &allowed) + return lo.Contains[uint](allowed, policy.ID) +} diff --git a/pkg/cache/driver.go b/pkg/cache/driver.go index 5a919a7..046d271 100644 --- a/pkg/cache/driver.go +++ b/pkg/cache/driver.go @@ -10,7 +10,7 @@ import ( var Store Driver = NewMemoStore() // Init 初始化缓存 -func Init(isSlave bool) { +func Init() { if conf.RedisConfig.Server != "" && gin.Mode() != gin.TestMode { Store = NewRedisStore( 10, @@ -20,12 +20,12 @@ func Init(isSlave bool) { conf.RedisConfig.DB, ) } +} - if isSlave { - err := Store.Sets(conf.OptionOverwrite, "setting_") - if err != nil { - util.Log().Warning("Failed to overwrite database setting: %s", err) - } +func InitSlaveOverwrites() { + err := Store.Sets(conf.OptionOverwrite, "setting_") + if err != nil { + util.Log().Warning("Failed to overwrite database setting: %s", err) } } diff --git a/pkg/filesystem/driver/local/handler.go b/pkg/filesystem/driver/local/handler.go index bceff31..c3a980a 100644 --- a/pkg/filesystem/driver/local/handler.go +++ b/pkg/filesystem/driver/local/handler.go @@ -203,7 +203,7 @@ func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.Co return nil, driver.ErrorThumbNotExist } - thumbFile, err := handler.Get(ctx, file.SourceName+model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")) + thumbFile, err := handler.Get(ctx, file.ThumbFile()) if err != nil { if errors.Is(err, os.ErrNotExist) { err = fmt.Errorf("thumb not exist: %w (%w)", err, driver.ErrorThumbNotExist) diff --git a/pkg/filesystem/driver/remote/handler.go b/pkg/filesystem/driver/remote/handler.go index 663f137..b88be6e 100644 --- a/pkg/filesystem/driver/remote/handler.go +++ b/pkg/filesystem/driver/remote/handler.go @@ -208,7 +208,7 @@ func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, er // Thumb 获取文件缩略图 func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) { - // quick check by extensions + // quick check by extension name supported := []string{"png", "jpg", "jpeg", "gif"} if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 { supported = handler.Policy.OptionsSerialized.ThumbExts diff --git a/pkg/filesystem/image.go b/pkg/filesystem/image.go index 75eb7fa..1e81169 100644 --- a/pkg/filesystem/image.go +++ b/pkg/filesystem/image.go @@ -3,7 +3,7 @@ package filesystem import ( "context" "errors" - "io" + "os" "sync" "runtime" @@ -32,17 +32,42 @@ func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentR }, ErrObjectNotExist } + file := fs.FileTarget[0] w, h := fs.GenerateThumbnailSize(0, 0) ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, [2]uint{w, h}) - ctx = context.WithValue(ctx, fsctx.FileModelCtx, fs.FileTarget[0]) - res, err := fs.Handler.Thumb(ctx, &fs.FileTarget[0]) + ctx = context.WithValue(ctx, fsctx.FileModelCtx, file) + res, err := fs.Handler.Thumb(ctx, &file) if errors.Is(err, driver.ErrorThumbNotExist) { // Regenerate thumb if the thumb is not initialized yet - fs.GenerateThumbnail(ctx, &fs.FileTarget[0]) - res, err = fs.Handler.Thumb(ctx, &fs.FileTarget[0]) + fs.GenerateThumbnail(ctx, &file) + res, err = fs.Handler.Thumb(ctx, &file) } else if errors.Is(err, driver.ErrorThumbNotSupported) { - // Policy handler explicitly indicates thumb not available - _ = updateThumbStatus(&fs.FileTarget[0], model.ThumbStatusNotAvailable) + // Policy handler explicitly indicates thumb not available, check if proxy is enabled + if fs.Policy.CouldProxyThumb() { + // if thumb id marked as existed, redirect to "sidecar" thumb file. + if file.MetadataSerialized != nil && + file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusExist { + // redirect to sidecar file + res = &response.ContentResponse{ + Redirect: true, + } + res.URL, err = fs.Handler.Source( + ctx, + file.ThumbFile(), + *model.GetSiteURL(), + int64(model.GetIntSetting("preview_timeout", 60)), + false, + 0, + ) + } else { + // if not exist, generate and upload the sidecar thumb. + fs.GenerateThumbnail(ctx, &file) + res, err = fs.Handler.Thumb(ctx, &file) + } + } else { + // thumb not supported and proxy is disabled, mark as not available + _ = updateThumbStatus(&file, model.ThumbStatusNotAvailable) + } } if err == nil && conf.SystemConfig.Mode == "master" { @@ -100,20 +125,7 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) { getThumbWorker().addWorker() defer getThumbWorker().releaseWorker() - r, w := io.Pipe() - defer w.Close() - - errChan := make(chan error, 1) - go func(errChan chan error) { - errChan <- fs.Handler.Put(newCtx, &fsctx.FileStream{ - Mode: fsctx.Overwrite, - File: io.NopCloser(r), - Seeker: nil, - SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"), - }) - }(errChan) - - if err = thumb.Generators.Generate(source, w, file.Name, model.GetSettingByNames( + thumbPath, err := thumb.Generators.Generate(source, file.Name, model.GetSettingByNames( "thumb_width", "thumb_height", "thumb_builtin_enabled", @@ -121,16 +133,33 @@ func (fs *FileSystem) GenerateThumbnail(ctx context.Context, file *model.File) { "thumb_ffmpeg_enabled", "thumb_vips_path", "thumb_ffmpeg_path", - )); err != nil { + )) + if err != nil { util.Log().Warning("Failed to generate thumb for %s: %s", file.Name, err) _ = updateThumbStatus(file, model.ThumbStatusNotAvailable) - w.Close() - <-errChan return } - w.Close() - if err = <-errChan; err != nil { + thumbFile, err := os.Open(thumbPath) + if err != nil { + util.Log().Warning("Failed to open temp thumb %q: %s", thumbFile, err) + return + } + + defer thumbFile.Close() + fileInfo, err := thumbFile.Stat() + if err != nil { + util.Log().Warning("Failed to stat temp thumb %q: %s", thumbFile, err) + return + } + + if err = fs.Handler.Put(newCtx, &fsctx.FileStream{ + Mode: fsctx.Overwrite, + File: thumbFile, + Seeker: thumbFile, + Size: uint64(fileInfo.Size()), + SavePath: file.SourceName + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb"), + }); err != nil { util.Log().Warning("Failed to save thumb for %s: %s", file.Name, err) return } diff --git a/pkg/thumb/builtin.go b/pkg/thumb/builtin.go index f7734e0..16c0618 100644 --- a/pkg/thumb/builtin.go +++ b/pkg/thumb/builtin.go @@ -12,7 +12,7 @@ import ( model "github.com/cloudreve/Cloudreve/v3/models" "github.com/cloudreve/Cloudreve/v3/pkg/util" - + "github.com/gofrs/uuid" //"github.com/nfnt/resize" "golang.org/x/image/draw" ) @@ -156,14 +156,30 @@ func (image *Thumb) CreateAvatar(uid uint) error { type Builtin struct{} -func (b Builtin) Generate(file io.Reader, w io.Writer, name string, options map[string]string) error { +func (b Builtin) Generate(file io.Reader, name string, options map[string]string) (string, error) { img, err := NewThumbFromFile(file, name) if err != nil { - return err + return "", err } img.GetThumb(thumbSize(options)) - return img.Save(w) + tempPath := filepath.Join( + util.RelativePath(model.GetSettingByName("temp_path")), + "thumb", + fmt.Sprintf("thumb_%s", uuid.Must(uuid.NewV4()).String()), + ) + + thumbFile, err := util.CreatNestedFile(tempPath) + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + + defer thumbFile.Close() + if err := img.Save(thumbFile); err != nil { + return "", err + } + + return tempPath, nil } func (b Builtin) Priority() int { diff --git a/pkg/thumb/pipeline.go b/pkg/thumb/pipeline.go index e38b466..c4a4eb7 100644 --- a/pkg/thumb/pipeline.go +++ b/pkg/thumb/pipeline.go @@ -12,7 +12,7 @@ import ( // Generator generates a thumbnail for a given reader. type Generator interface { - Generate(file io.Reader, w io.Writer, name string, options map[string]string) error + Generate(file io.Reader, name string, options map[string]string) (string, error) // Priority of execution order, smaller value means higher priority. Priority() int @@ -51,19 +51,19 @@ func RegisterGenerator(generator Generator) { sort.Sort(Generators) } -func (p GeneratorList) Generate(file io.Reader, w io.Writer, name string, options map[string]string) error { +func (p GeneratorList) Generate(file io.Reader, name string, options map[string]string) (string, error) { for _, generator := range p { if model.IsTrueVal(options[generator.EnableFlag()]) { - err := generator.Generate(file, w, name, options) + res, err := generator.Generate(file, name, options) if errors.Is(err, ErrPassThrough) { util.Log().Debug("Failed to generate thumbnail for %s: %s, passing through to next generator.", name, err) continue } - return err + return res, err } } - return ErrNotAvailable + return "", ErrNotAvailable } func (p GeneratorList) Priority() int {