diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6d4f47..ddc7f09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,10 +7,10 @@ jobs: name: Build runs-on: ubuntu-18.04 steps: - - name: Set up Go 1.17 + - name: Set up Go 1.18 uses: actions/setup-go@v2 with: - go-version: "1.17" + go-version: "1.18" id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10337c6..7217dd9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,10 +12,10 @@ jobs: name: Test runs-on: ubuntu-18.04 steps: - - name: Set up Go 1.17 + - name: Set up Go 1.18 uses: actions/setup-go@v2 with: - go-version: "1.17" + go-version: "1.18" id: go - name: Check out code into the Go module directory diff --git a/.travis.yml b/.travis.yml index 546dc3e..6f3891e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.17.x + - 1.18.x node_js: "12.16.3" git: depth: 1 diff --git a/Dockerfile b/Dockerfile index c657018..91f6910 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17-alpine as cloudreve_builder +FROM golang:1.18-alpine as cloudreve_builder # install dependencies and build tools diff --git a/README.md b/README.md index b951750..d23c9ec 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ chmod +x ./cloudreve ## :gear: 构建 -自行构建前需要拥有 `Go >= 1.17`、`node.js`、`yarn`、`zip` 等必要依赖。 +自行构建前需要拥有 `Go >= 1.18`、`node.js`、`yarn`、`zip` 等必要依赖。 #### 克隆代码 diff --git a/bootstrap/embed.go b/bootstrap/embed.go new file mode 100644 index 0000000..71f7567 --- /dev/null +++ b/bootstrap/embed.go @@ -0,0 +1,432 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package embed provides access to files embedded in the running Go program. +// +// Go source files that import "embed" can use the //go:embed directive +// to initialize a variable of type string, []byte, or FS with the contents of +// files read from the package directory or subdirectories at compile time. +// +// For example, here are three ways to embed a file named hello.txt +// and then print its contents at run time. +// +// Embedding one file into a string: +// +// import _ "embed" +// +// //go:embed hello.txt +// var s string +// print(s) +// +// Embedding one file into a slice of bytes: +// +// import _ "embed" +// +// //go:embed hello.txt +// var b []byte +// print(string(b)) +// +// Embedded one or more files into a file system: +// +// import "embed" +// +// //go:embed hello.txt +// var f embed.FS +// data, _ := f.ReadFile("hello.txt") +// print(string(data)) +// +// # Directives +// +// A //go:embed directive above a variable declaration specifies which files to embed, +// using one or more path.Match patterns. +// +// The directive must immediately precede a line containing the declaration of a single variable. +// Only blank lines and ‘//’ line comments are permitted between the directive and the declaration. +// +// The type of the variable must be a string type, or a slice of a byte type, +// or FS (or an alias of FS). +// +// For example: +// +// package server +// +// import "embed" +// +// // content holds our static web server content. +// //go:embed image/* template/* +// //go:embed html/index.html +// var content embed.FS +// +// The Go build system will recognize the directives and arrange for the declared variable +// (in the example above, content) to be populated with the matching files from the file system. +// +// The //go:embed directive accepts multiple space-separated patterns for +// brevity, but it can also be repeated, to avoid very long lines when there are +// many patterns. The patterns are interpreted relative to the package directory +// containing the source file. The path separator is a forward slash, even on +// Windows systems. Patterns may not contain ‘.’ or ‘..’ or empty path elements, +// nor may they begin or end with a slash. To match everything in the current +// directory, use ‘*’ instead of ‘.’. To allow for naming files with spaces in +// their names, patterns can be written as Go double-quoted or back-quoted +// string literals. +// +// If a pattern names a directory, all files in the subtree rooted at that directory are +// embedded (recursively), except that files with names beginning with ‘.’ or ‘_’ +// are excluded. So the variable in the above example is almost equivalent to: +// +// // content is our static web server content. +// //go:embed image template html/index.html +// var content embed.FS +// +// The difference is that ‘image/*’ embeds ‘image/.tempfile’ while ‘image’ does not. +// Neither embeds ‘image/dir/.tempfile’. +// +// If a pattern begins with the prefix ‘all:’, then the rule for walking directories is changed +// to include those files beginning with ‘.’ or ‘_’. For example, ‘all:image’ embeds +// both ‘image/.tempfile’ and ‘image/dir/.tempfile’. +// +// The //go:embed directive can be used with both exported and unexported variables, +// depending on whether the package wants to make the data available to other packages. +// It can only be used with variables at package scope, not with local variables. +// +// Patterns must not match files outside the package's module, such as ‘.git/*’ or symbolic links. +// Patterns must not match files whose names include the special punctuation characters " * < > ? ` ' | / \ and :. +// Matches for empty directories are ignored. After that, each pattern in a //go:embed line +// must match at least one file or non-empty directory. +// +// If any patterns are invalid or have invalid matches, the build will fail. +// +// # Strings and Bytes +// +// The //go:embed line for a variable of type string or []byte can have only a single pattern, +// and that pattern can match only a single file. The string or []byte is initialized with +// the contents of that file. +// +// The //go:embed directive requires importing "embed", even when using a string or []byte. +// In source files that don't refer to embed.FS, use a blank import (import _ "embed"). +// +// # File Systems +// +// For embedding a single file, a variable of type string or []byte is often best. +// The FS type enables embedding a tree of files, such as a directory of static +// web server content, as in the example above. +// +// FS implements the io/fs package's FS interface, so it can be used with any package that +// understands file systems, including net/http, text/template, and html/template. +// +// For example, given the content variable in the example above, we can write: +// +// http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(content)))) +// +// template.ParseFS(content, "*.tmpl") +// +// # Tools +// +// To support tools that analyze Go packages, the patterns found in //go:embed lines +// are available in “go list” output. See the EmbedPatterns, TestEmbedPatterns, +// and XTestEmbedPatterns fields in the “go help list” output. +package bootstrap + +import ( + "errors" + "io" + "io/fs" + "time" +) + +// An FS is a read-only collection of files, usually initialized with a //go:embed directive. +// When declared without a //go:embed directive, an FS is an empty file system. +// +// An FS is a read-only value, so it is safe to use from multiple goroutines +// simultaneously and also safe to assign values of type FS to each other. +// +// FS implements fs.FS, so it can be used with any package that understands +// file system interfaces, including net/http, text/template, and html/template. +// +// See the package documentation for more details about initializing an FS. +type FS struct { + // The compiler knows the layout of this struct. + // See cmd/compile/internal/staticdata's WriteEmbed. + // + // The files list is sorted by name but not by simple string comparison. + // Instead, each file's name takes the form "dir/elem" or "dir/elem/". + // The optional trailing slash indicates that the file is itself a directory. + // The files list is sorted first by dir (if dir is missing, it is taken to be ".") + // and then by base, so this list of files: + // + // p + // q/ + // q/r + // q/s/ + // q/s/t + // q/s/u + // q/v + // w + // + // is actually sorted as: + // + // p # dir=. elem=p + // q/ # dir=. elem=q + // w/ # dir=. elem=w + // q/r # dir=q elem=r + // q/s/ # dir=q elem=s + // q/v # dir=q elem=v + // q/s/t # dir=q/s elem=t + // q/s/u # dir=q/s elem=u + // + // This order brings directory contents together in contiguous sections + // of the list, allowing a directory read to use binary search to find + // the relevant sequence of entries. + files *[]file +} + +// split splits the name into dir and elem as described in the +// comment in the FS struct above. isDir reports whether the +// final trailing slash was present, indicating that name is a directory. +func split(name string) (dir, elem string, isDir bool) { + if name[len(name)-1] == '/' { + isDir = true + name = name[:len(name)-1] + } + i := len(name) - 1 + for i >= 0 && name[i] != '/' { + i-- + } + if i < 0 { + return ".", name, isDir + } + return name[:i], name[i+1:], isDir +} + +// trimSlash trims a trailing slash from name, if present, +// returning the possibly shortened name. +func trimSlash(name string) string { + if len(name) > 0 && name[len(name)-1] == '/' { + return name[:len(name)-1] + } + return name +} + +var ( + _ fs.ReadDirFS = FS{} + _ fs.ReadFileFS = FS{} +) + +// A file is a single file in the FS. +// It implements fs.FileInfo and fs.DirEntry. +type file struct { + // The compiler knows the layout of this struct. + // See cmd/compile/internal/staticdata's WriteEmbed. + name string + data string + hash [16]byte // truncated SHA256 hash +} + +var ( + _ fs.FileInfo = (*file)(nil) + _ fs.DirEntry = (*file)(nil) +) + +func (f *file) Name() string { _, elem, _ := split(f.name); return elem } +func (f *file) Size() int64 { return int64(len(f.data)) } +func (f *file) ModTime() time.Time { return time.Time{} } +func (f *file) IsDir() bool { _, _, isDir := split(f.name); return isDir } +func (f *file) Sys() any { return nil } +func (f *file) Type() fs.FileMode { return f.Mode().Type() } +func (f *file) Info() (fs.FileInfo, error) { return f, nil } + +func (f *file) Mode() fs.FileMode { + if f.IsDir() { + return fs.ModeDir | 0555 + } + return 0444 +} + +// dotFile is a file for the root directory, +// which is omitted from the files list in a FS. +var dotFile = &file{name: "./"} + +// lookup returns the named file, or nil if it is not present. +func (f FS) lookup(name string) *file { + if !fs.ValidPath(name) { + // The compiler should never emit a file with an invalid name, + // so this check is not strictly necessary (if name is invalid, + // we shouldn't find a match below), but it's a good backstop anyway. + return nil + } + if name == "." { + return dotFile + } + if f.files == nil { + return nil + } + + // Binary search to find where name would be in the list, + // and then check if name is at that position. + dir, elem, _ := split(name) + files := *f.files + i := sortSearch(len(files), func(i int) bool { + idir, ielem, _ := split(files[i].name) + return idir > dir || idir == dir && ielem >= elem + }) + if i < len(files) && trimSlash(files[i].name) == name { + return &files[i] + } + return nil +} + +// readDir returns the list of files corresponding to the directory dir. +func (f FS) readDir(dir string) []file { + if f.files == nil { + return nil + } + // Binary search to find where dir starts and ends in the list + // and then return that slice of the list. + files := *f.files + i := sortSearch(len(files), func(i int) bool { + idir, _, _ := split(files[i].name) + return idir >= dir + }) + j := sortSearch(len(files), func(j int) bool { + jdir, _, _ := split(files[j].name) + return jdir > dir + }) + return files[i:j] +} + +// Open opens the named file for reading and returns it as an fs.File. +// +// The returned file implements io.Seeker when the file is not a directory. +func (f FS) Open(name string) (fs.File, error) { + file := f.lookup(name) + if file == nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + if file.IsDir() { + return &openDir{file, f.readDir(name), 0}, nil + } + return &openFile{file, 0}, nil +} + +// ReadDir reads and returns the entire named directory. +func (f FS) ReadDir(name string) ([]fs.DirEntry, error) { + file, err := f.Open(name) + if err != nil { + return nil, err + } + dir, ok := file.(*openDir) + if !ok { + return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("not a directory")} + } + list := make([]fs.DirEntry, len(dir.files)) + for i := range list { + list[i] = &dir.files[i] + } + return list, nil +} + +// ReadFile reads and returns the content of the named file. +func (f FS) ReadFile(name string) ([]byte, error) { + file, err := f.Open(name) + if err != nil { + return nil, err + } + ofile, ok := file.(*openFile) + if !ok { + return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("is a directory")} + } + return []byte(ofile.f.data), nil +} + +// An openFile is a regular file open for reading. +type openFile struct { + f *file // the file itself + offset int64 // current read offset +} + +var ( + _ io.Seeker = (*openFile)(nil) +) + +func (f *openFile) Close() error { return nil } +func (f *openFile) Stat() (fs.FileInfo, error) { return f.f, nil } + +func (f *openFile) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.f.data)) { + return 0, io.EOF + } + if f.offset < 0 { + return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid} + } + n := copy(b, f.f.data[f.offset:]) + f.offset += int64(n) + return n, nil +} + +func (f *openFile) Seek(offset int64, whence int) (int64, error) { + switch whence { + case 0: + // offset += 0 + case 1: + offset += f.offset + case 2: + offset += int64(len(f.f.data)) + } + if offset < 0 || offset > int64(len(f.f.data)) { + return 0, &fs.PathError{Op: "seek", Path: f.f.name, Err: fs.ErrInvalid} + } + f.offset = offset + return offset, nil +} + +// An openDir is a directory open for reading. +type openDir struct { + f *file // the directory file itself + files []file // the directory contents + offset int // the read offset, an index into the files slice +} + +func (d *openDir) Close() error { return nil } +func (d *openDir) Stat() (fs.FileInfo, error) { return d.f, nil } + +func (d *openDir) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.f.name, Err: errors.New("is a directory")} +} + +func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) { + n := len(d.files) - d.offset + if n == 0 { + if count <= 0 { + return nil, nil + } + return nil, io.EOF + } + if count > 0 && n > count { + n = count + } + list := make([]fs.DirEntry, n) + for i := range list { + list[i] = &d.files[d.offset+i] + } + d.offset += n + return list, nil +} + +// sortSearch is like sort.Search, avoiding an import. +func sortSearch(n int, f func(int) bool) int { + // Define f(-1) == false and f(n) == true. + // Invariant: f(i-1) == false, f(j) == true. + i, j := 0, n + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + if !f(h) { + i = h + 1 // preserves f(i-1) == false + } else { + j = h // preserves f(j) == true + } + } + // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i. + return i +} diff --git a/bootstrap/fs.go b/bootstrap/fs.go new file mode 100644 index 0000000..d2f7c1e --- /dev/null +++ b/bootstrap/fs.go @@ -0,0 +1,75 @@ +package bootstrap + +import ( + "archive/zip" + "crypto/sha256" + "github.com/cloudreve/Cloudreve/v3/pkg/util" + "github.com/pkg/errors" + "io" + "io/fs" + "sort" + "strings" +) + +func NewFS(zipContent string) fs.FS { + zipReader, err := zip.NewReader(strings.NewReader(zipContent), int64(len(zipContent))) + if err != nil { + util.Log().Panic("静态资源不是合法的zip文件: %s", err) + } + + var files []file + err = fs.WalkDir(zipReader, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", path, err) + } + + if path == "." { + return nil + } + + var f file + if d.IsDir() { + f.name = path + "/" + } else { + f.name = path + + rc, err := zipReader.Open(path) + if err != nil { + return errors.Errorf("无法打开文件[%s], %s, 跳过...", path, err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return errors.Errorf("无法读取文件[%s], %s, 跳过...", path, err) + } + + f.data = string(data) + + hash := sha256.Sum256(data) + for i := range f.hash { + f.hash[i] = ^hash[i] + } + } + files = append(files, f) + return nil + }) + if err != nil { + util.Log().Panic("初始化静态资源失败: %s", err) + } + + sort.Slice(files, func(i, j int) bool { + fi, fj := files[i], files[j] + di, ei, _ := split(fi.name) + dj, ej, _ := split(fj.name) + + if di != dj { + return di < dj + } + return ei < ej + }) + + var embedFS FS + embedFS.files = &files + return embedFS +} diff --git a/go.mod b/go.mod index 988657e..7c87063 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cloudreve/Cloudreve/v3 -go 1.17 +go 1.18 require ( github.com/DATA-DOG/go-sqlmock v1.3.3 diff --git a/main.go b/main.go index b813c9e..beb875c 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,11 @@ import ( "context" _ "embed" "flag" - "io" "io/fs" "net" "net/http" "os" "os/signal" - "strings" "syscall" "time" @@ -19,8 +17,6 @@ import ( "github.com/cloudreve/Cloudreve/v3/pkg/conf" "github.com/cloudreve/Cloudreve/v3/pkg/util" "github.com/cloudreve/Cloudreve/v3/routers" - - "github.com/mholt/archiver/v4" ) var ( @@ -40,10 +36,7 @@ func init() { flag.StringVar(&scriptName, "database-script", "", "Name of database util script.") flag.Parse() - staticFS = archiver.ArchiveFS{ - Stream: io.NewSectionReader(strings.NewReader(staticZip), 0, int64(len(staticZip))), - Format: archiver.Zip{}, - } + staticFS = bootstrap.NewFS(staticZip) bootstrap.Init(confPath, staticFS) } diff --git a/service/callback/upload.go b/service/callback/upload.go index 7c8aeef..0dd7924 100644 --- a/service/callback/upload.go +++ b/service/callback/upload.go @@ -175,7 +175,7 @@ func (service *OneDriveCallback) PreProcess(c *gin.Context) serializer.Response // SharePoint 会对 Office 文档增加 meta data 导致文件大小不一致，这里增加 1 MB 宽容 // See: https://github.com/OneDrive/onedrive-api-docs/issues/935 - if strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") && isSizeCheckFailed && (info.Size > uploadSession.Size) && (info.Size-uploadSession.Size <= 1048576) { + if (strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.com") || strings.Contains(fs.Policy.OptionsSerialized.OdDriver, "sharepoint.cn")) && isSizeCheckFailed && (info.Size > uploadSession.Size) && (info.Size-uploadSession.Size <= 1048576) { isSizeCheckFailed = false }