use go:embed for static resources (#1107)

* feat: use go:embed to embed static files

* ci: fix broken test

* docs: update readme.md

* chore: remove statik

* feat: simplify code

Co-authored-by: AaronLiu <abslant@126.com>

.github/workflows/build.yml

@@ -9,53 +9,49 @@ jobs:
     name: Build
     runs-on: ubuntu-18.04
     steps:
-
-    - name: Set up Golang
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.17
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-      with:
-        clean: false
-        submodules: 'recursive'
-    - run: |
-        git fetch --prune --unshallow --tags
-
-    - name: Get dependencies and build
-      run: |
-        go install github.com/rakyll/statik
-        export PATH=$PATH:~/go/bin/
-        statik -src=models -f
-        sudo apt-get update
-        sudo apt-get -y install gcc-mingw-w64-x86-64
-        sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
-        sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-        chmod +x ./build.sh
-        ./build.sh -r b
-
-    - name: Upload binary files (windows_amd64)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_windows_amd64
-        path: release/cloudreve*windows_amd64.*
-
-    - name: Upload binary files (linux_amd64)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_linux_amd64
-        path: release/cloudreve*linux_amd64.*
-
-    - name: Upload binary files (linux_arm)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_linux_arm
-        path: release/cloudreve*linux_arm.*
-
-    - name: Upload binary files (linux_arm64)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_linux_arm64
-        path: release/cloudreve*linux_arm64.*
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.17"
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          clean: false
+          submodules: "recursive"
+      - run: |
+          git fetch --prune --unshallow --tags
+
+      - name: Get dependencies and build
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install gcc-mingw-w64-x86-64
+          sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
+          sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+          chmod +x ./build.sh
+          ./build.sh -r b
+
+      - name: Upload binary files (windows_amd64)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_windows_amd64
+          path: release/cloudreve*windows_amd64.*
+
+      - name: Upload binary files (linux_amd64)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_linux_amd64
+          path: release/cloudreve*linux_amd64.*
+
+      - name: Upload binary files (linux_arm)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_linux_arm
+          path: release/cloudreve*linux_arm.*
+
+      - name: Upload binary files (linux_arm64)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_linux_arm64
+          path: release/cloudreve*linux_arm64.*

.github/workflows/test.yml

@@ -2,46 +2,43 @@ name: Test
 
 on:
   pull_request:
-      branches:
+    branches:
       - master
   push:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
-
   test:
     name: Test
     runs-on: ubuntu-18.04
     steps:
-
-    - name: Set up Golang
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.17
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-      with:
-        submodules: 'recursive'
-
-    - name: Get dependencies
-      run: |
-        go get github.com/rakyll/statik
-        export PATH=$PATH:~/go/bin/
-        statik -src=models -f
-
-    - name: Test
-      run: go test -coverprofile=coverage.txt -covermode=atomic ./...
-
-    - name: Upload binary files (linux_arm)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_linux_arm
-        path: release/cloudreve*linux_arm.*
-
-    - name: Upload binary files (linux_arm64)
-      uses: actions/upload-artifact@v2
-      with:
-        name: cloudreve_linux_arm64
-        path: release/cloudreve*linux_arm64.*
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.17"
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+
+      - name: Build static files
+        run: |
+          mkdir assets/build
+          touch assets/build/test.html
+
+      - name: Test
+        run: go test -coverprofile=coverage.txt -covermode=atomic ./...
+
+      - name: Upload binary files (linux_arm)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_linux_arm
+          path: release/cloudreve*linux_arm.*
+
+      - name: Upload binary files (linux_arm64)
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloudreve_linux_arm64
+          path: release/cloudreve*linux_arm64.*

.travis.yml

@@ -4,10 +4,9 @@ go:
 node_js: "12.16.3"
 git:
   depth: 1
-install:
-  - go get github.com/rakyll/statik
 before_script:
-  - statik -src=models -f
+  - mkdir assets/build
+  - touch assets/build/test.html
 script:
   - go test -coverprofile=coverage.txt -covermode=atomic ./...
 after_success:
@@ -27,4 +26,4 @@ deploy:
   draft: true
   skip_cleanup: true
   on:
-    tags: true
+    tags: true

Dockerfile

@@ -28,8 +28,6 @@ RUN set -ex \
     && apk add gcc libc-dev git \
     && export COMMIT_SHA=$(git rev-parse --short HEAD) \
     && export VERSION=$(git describe --tags) \
-    && (cd && go get github.com/rakyll/statik) \
-    && statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f \
     && go install -ldflags "-X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=${VERSION}' \
                             -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=${COMMIT_SHA}'\
                             -w -s"

README.md

@@ -68,7 +68,7 @@ chmod +x ./cloudreve
 
 ## :gear: 构建
 
-自行构建前需要拥有 `Go >= 1.13`、`yarn`等必要依赖。
+自行构建前需要拥有 `Go >= 1.17`、`yarn`等必要依赖。
 
 #### 克隆代码
 
@@ -85,19 +85,7 @@ cd assets
 yarn install
 # 开始构建
 yarn run build
-```
-
-#### 嵌入静态资源
-
-```shell
-# 回到项目主目录
-cd ../
-
-# 安装 statik, 用于嵌入静态资源
-go get github.com/rakyll/statik
 
-# 开始嵌入
-statik -src=assets/build/  -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f
 ```
 
 #### 编译项目
@@ -108,7 +96,7 @@ export COMMIT_SHA=$(git rev-parse --short HEAD)
 export VERSION=$(git describe --tags)
 
 # 开始编译
-go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
+go build -a -o cloudreve -ldflags "-s -w -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
 ```
 
 你也可以使用项目根目录下的`build.sh`快速开始构建：

bootstrap/static.go

@@ -1,21 +1,26 @@
 package bootstrap
 
 import (
+	"bufio"
+	"embed"
 	"encoding/json"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"net/http"
-	"path"
+	"path/filepath"
+
+	"github.com/pkg/errors"
 
 	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
-	_ "github.com/cloudreve/Cloudreve/v3/statik"
+
 	"github.com/gin-contrib/static"
-	"github.com/rakyll/statik/fs"
 )
 
 const StaticFolder = "statics"
 
+var StaticEmbed embed.FS
+
 type GinFS struct {
 	FS http.FileSystem
 }
@@ -35,124 +40,100 @@ func (b *GinFS) Open(name string) (http.File, error) {
 
 // Exists 文件是否存在
 func (b *GinFS) Exists(prefix string, filepath string) bool {
-
 	if _, err := b.FS.Open(filepath); err != nil {
 		return false
 	}
 	return true
-
 }
 
 // InitStatic 初始化静态资源文件
 func InitStatic() {
-	var err error
-
 	if util.Exists(util.RelativePath(StaticFolder)) {
 		util.Log().Info("检测到 statics 目录存在，将使用此目录下的静态资源文件")
 		StaticFS = static.LocalFile(util.RelativePath("statics"), false)
-
-		// 检查静态资源的版本
-		f, err := StaticFS.Open("version.json")
-		if err != nil {
-			util.Log().Warning("静态资源版本标识文件不存在，请重新构建或删除 statics 目录")
-			return
-		}
-
-		b, err := ioutil.ReadAll(f)
+	} else {
+		// 初始化静态资源
+		embedFS, err := fs.Sub(StaticEmbed, "assets/build")
 		if err != nil {
-			util.Log().Warning("无法读取静态资源文件版本，请重新构建或删除 statics 目录")
-			return
+			util.Log().Panic("无法初始化静态资源, %s", err)
 		}
 
-		var v staticVersion
-		if err := json.Unmarshal(b, &v); err != nil {
-			util.Log().Warning("无法解析静态资源文件版本, %s", err)
-			return
+		StaticFS = &GinFS{
+			FS: http.FS(embedFS),
 		}
+	}
+	// 检查静态资源的版本
+	f, err := StaticFS.Open("version.json")
+	if err != nil {
+		util.Log().Warning("静态资源版本标识文件不存在，请重新构建或删除 statics 目录")
+		return
+	}
 
-		staticName := "cloudreve-frontend"
-		if conf.IsPro == "true" {
-			staticName += "-pro"
-		}
+	b, err := io.ReadAll(f)
+	if err != nil {
+		util.Log().Warning("无法读取静态资源文件版本，请重新构建或删除 statics 目录")
+		return
+	}
 
-		if v.Name != staticName {
-			util.Log().Warning("静态资源版本不匹配，请重新构建或删除 statics 目录")
-			return
-		}
+	var v staticVersion
+	if err := json.Unmarshal(b, &v); err != nil {
+		util.Log().Warning("无法解析静态资源文件版本, %s", err)
+		return
+	}
 
-		if v.Version != conf.RequiredStaticVersion {
-			util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s]，请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
-			return
-		}
+	staticName := "cloudreve-frontend"
+	if conf.IsPro == "true" {
+		staticName += "-pro"
+	}
 
-	} else {
-		StaticFS = &GinFS{}
-		StaticFS.(*GinFS).FS, err = fs.New()
-		if err != nil {
-			util.Log().Panic("无法初始化静态资源, %s", err)
-		}
+	if v.Name != staticName {
+		util.Log().Warning("静态资源版本不匹配，请重新构建或删除 statics 目录")
+		return
 	}
 
+	if v.Version != conf.RequiredStaticVersion {
+		util.Log().Warning("静态资源版本不匹配 [当前 %s, 需要: %s]，请重新构建或删除 statics 目录", v.Version, conf.RequiredStaticVersion)
+		return
+	}
 }
 
 // Eject 抽离内置静态资源
 func Eject() {
-	staticFS, err := fs.New()
+	// 初始化静态资源
+	embedFS, err := fs.Sub(StaticEmbed, "assets/build")
 	if err != nil {
 		util.Log().Panic("无法初始化静态资源, %s", err)
 	}
 
-	root, err := staticFS.Open("/")
-	if err != nil {
-		util.Log().Panic("根目录不存在, %s", err)
-	}
-
-	var walk func(relPath string, object http.File)
-	walk = func(relPath string, object http.File) {
-		stat, err := object.Stat()
+	var walk func(relPath string, d fs.DirEntry, err error) error
+	walk = func(relPath string, d fs.DirEntry, err error) error {
 		if err != nil {
-			util.Log().Error("无法获取[%s]的信息, %s, 跳过...", relPath, err)
-			return
+			return errors.Errorf("无法获取[%s]的信息, %s, 跳过...", relPath, err)
 		}
 
-		if !stat.IsDir() {
+		if !d.IsDir() {
 			// 写入文件
-			out, err := util.CreatNestedFile(util.RelativePath(StaticFolder + relPath))
+			out, err := util.CreatNestedFile(filepath.Join(util.RelativePath(""), StaticFolder, relPath))
 			defer out.Close()
 
 			if err != nil {
-				util.Log().Error("无法创建文件[%s], %s, 跳过...", relPath, err)
-				return
+				return errors.Errorf("无法创建文件[%s], %s, 跳过...", relPath, err)
 			}
 
 			util.Log().Info("导出 [%s]...", relPath)
-			if _, err := io.Copy(out, object); err != nil {
-				util.Log().Error("无法写入文件[%s], %s, 跳过...", relPath, err)
-				return
+			obj, _ := embedFS.Open(relPath)
+			if _, err := io.Copy(out, bufio.NewReader(obj)); err != nil {
+				return errors.Errorf("无法写入文件[%s], %s, 跳过...", relPath, err)
 			}
-		} else {
-			// 列出目录
-			objects, err := object.Readdir(0)
-			if err != nil {
-				util.Log().Error("无法步入子目录[%s], %s, 跳过...", relPath, err)
-				return
-			}
-
-			// 递归遍历子目录
-			for _, newObject := range objects {
-				newPath := path.Join(relPath, newObject.Name())
-				newRoot, err := staticFS.Open(newPath)
-				if err != nil {
-					util.Log().Error("无法打开对象[%s], %s, 跳过...", newPath, err)
-					continue
-				}
-				walk(newPath, newRoot)
-			}
-
 		}
+		return nil
 	}
 
-	util.Log().Info("开始导出内置静态资源...")
-	walk("/", root)
+	// util.Log().Info("开始导出内置静态资源...")
+	err = fs.WalkDir(embedFS, ".", walk)
+	if err != nil {
+		util.Log().Error("导出内置静态资源遇到错误：%s", err)
+		return
+	}
 	util.Log().Info("内置静态资源导出完成")
 }

build.sh

@@ -1,13 +1,16 @@
 #!/bin/bash
 
-REPO=$(cd $(dirname $0); pwd)
+REPO=$(
+  cd $(dirname $0)
+  pwd
+)
 COMMIT_SHA=$(git rev-parse --short HEAD)
 VERSION=$(git describe --tags)
 ASSETS="false"
 BINARY="false"
 RELEASE="false"
 
-debugInfo () {
+debugInfo() {
   echo "Repo:           $REPO"
   echo "Build assets:   $ASSETS"
   echo "Build binary:   $BINARY"
@@ -16,10 +19,9 @@ debugInfo () {
   echo "Commit:        $COMMIT_SHA"
 }
 
-buildAssets () {
+buildAssets() {
   cd $REPO
   rm -rf assets/build
-  rm -f statik/statik.go
 
   export CI=false
 
@@ -27,94 +29,88 @@ buildAssets () {
 
   yarn install
   yarn run build
-
-  if ! [ -x "$(command -v statik)" ]; then
-    export CGO_ENABLED=0
-    go get github.com/rakyll/statik
-  fi
-
-  cd $REPO
-  statik -src=assets/build/  -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico,*.ttf -f
+  cd build
+  rm -rf *.map
 }
 
-buildBinary () {
+buildBinary() {
   cd $REPO
   go build -a -o cloudreve -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
 }
 
 _build() {
-    local osarch=$1
-    IFS=/ read -r -a arr <<<"$osarch"
-    os="${arr[0]}"
-    arch="${arr[1]}"
-    gcc="${arr[2]}"
-
-    # Go build to build the binary.
-    export GOOS=$os
-    export GOARCH=$arch
-    export CC=$gcc
-    export CGO_ENABLED=1
-
-    if [ -n "$VERSION" ]; then
-        out="release/cloudreve_${VERSION}_${os}_${arch}"
-    else
-        out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
-    fi
-
-    go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
-
-    if [ "$os" = "windows" ]; then
-      mv $out release/cloudreve.exe
-      zip -j -q "${out}.zip" release/cloudreve.exe
-      rm -f "release/cloudreve.exe"
-    else
-      mv $out release/cloudreve
-      tar -zcvf "${out}.tar.gz" -C release cloudreve
-      rm -f "release/cloudreve"
-    fi
+  local osarch=$1
+  IFS=/ read -r -a arr <<<"$osarch"
+  os="${arr[0]}"
+  arch="${arr[1]}"
+  gcc="${arr[2]}"
+
+  # Go build to build the binary.
+  export GOOS=$os
+  export GOARCH=$arch
+  export CC=$gcc
+  export CGO_ENABLED=1
+
+  if [ -n "$VERSION" ]; then
+    out="release/cloudreve_${VERSION}_${os}_${arch}"
+  else
+    out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
+  fi
+
+  go build -a -o "${out}" -ldflags " -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.BackendVersion=$VERSION' -X 'github.com/cloudreve/Cloudreve/v3/pkg/conf.LastCommit=$COMMIT_SHA'"
+
+  if [ "$os" = "windows" ]; then
+    mv $out release/cloudreve.exe
+    zip -j -q "${out}.zip" release/cloudreve.exe
+    rm -f "release/cloudreve.exe"
+  else
+    mv $out release/cloudreve
+    tar -zcvf "${out}.tar.gz" -C release cloudreve
+    rm -f "release/cloudreve"
+  fi
 }
 
-release(){
+release() {
   cd $REPO
   ## List of architectures and OS to test coss compilation.
   SUPPORTED_OSARCH="linux/amd64/gcc linux/arm/arm-linux-gnueabihf-gcc windows/amd64/x86_64-w64-mingw32-gcc linux/arm64/aarch64-linux-gnu-gcc"
 
   echo "Release builds for OS/Arch/CC: ${SUPPORTED_OSARCH}"
   for each_osarch in ${SUPPORTED_OSARCH}; do
-      _build "${each_osarch}"
+    _build "${each_osarch}"
   done
 }
 
 usage() {
-  echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2;
-  exit 1;
+  echo "Usage: $0 [-a] [-c] [-b] [-r]" 1>&2
+  exit 1
 }
 
 while getopts "bacr:d" o; do
   case "${o}" in
-    b)
-      ASSETS="true"
-      BINARY="true"
-      ;;
-    a)
-      ASSETS="true"
-      ;;
-    c)
-      BINARY="true"
-      ;;
-    r)
-      ASSETS="true"
-      RELEASE="true"
-      ;;
-    d)
-      DEBUG="true"
-      ;;
-    *)
-      usage
-      ;;
+  b)
+    ASSETS="true"
+    BINARY="true"
+    ;;
+  a)
+    ASSETS="true"
+    ;;
+  c)
+    BINARY="true"
+    ;;
+  r)
+    ASSETS="true"
+    RELEASE="true"
+    ;;
+  d)
+    DEBUG="true"
+    ;;
+  *)
+    usage
+    ;;
   esac
 done
-shift $((OPTIND-1))
+shift $((OPTIND - 1))
 
 if [ "$DEBUG" = "true" ]; then
   debugInfo

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudreve/Cloudreve/v3
 
-go 1.13
+go 1.17
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.3.3
@@ -43,4 +43,4 @@ require (
 	gopkg.in/go-playground/validator.v9 v9.29.1
 	gopkg.in/ini.v1 v1.51.0 // indirect
 	gopkg.in/mail.v2 v2.3.1 // indirect
-)
+)

go.sum

@@ -362,4 +362,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"flag"
 
 	"github.com/cloudreve/Cloudreve/v3/bootstrap"
@@ -15,6 +16,9 @@ var (
 	scriptName string
 )
 
+//go:embed assets/build
+var StaticEmbed embed.FS
+
 func init() {
 	flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
 	flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")

pkg/util/path.go

@@ -56,3 +56,4 @@ func RelativePath(name string) string {
 	e, _ := os.Executable()
 	return filepath.Join(filepath.Dir(e), name)
 }
+