msb-public
/
cloudreve
mirror of https://github.com/cloudreve/Cloudreve
1
0
Code Issues Projects Releases Wiki Activity

Merge pull request #1232 from cloudreve/archiver-decompress

Browse Source 
Archiver decompress
pull/1239/head
AaronLiu 2 years ago committed by GitHub
parent d10639fd19 86876a1c11
commit dc69a63217
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 180 additions and 102 deletions
Show Stats Download Patch File Download Diff File

3
Dockerfile
Unescape View File

@ -2,7 +2,7 @@ FROM golang:1.17-alpine as cloudreve_builder
# install dependencies and build tools
RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc
RUN apk update && apk add --no-cache wget curl git yarn build-base gcc abuild binutils binutils-doc gcc-doc zip
WORKDIR /cloudreve_builder
RUN git clone --recurse-submodules https://github.com/cloudreve/Cloudreve.git
@ -14,6 +14,7 @@ RUN yarn run build && rm -rf build/*.map
# build backend
WORKDIR /cloudreve_builder/Cloudreve
RUN zip -r - assets >assets.zip
RUN tag_name=$(git describe --tags) \
&& export COMMIT_SHA=$(git rev-parse --short HEAD) \
&& go build -a -o cloudreve -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$tag_name' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'"

0
assets.zip
Unescape View File

4
bootstrap/init.go
Unescape View File

@ -1,7 +1,6 @@
package bootstrap
import (
"embed"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/models/scripts"
"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
@ -14,10 +13,11 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/task"
"github.com/gin-gonic/gin"
"io/fs"
)
// Init 初始化启动
func Init(path string, statics embed.FS) {
func Init(path string, statics fs.FS) {
InitApplication()
conf.Init(path)
// Debug 关闭时，切换为生产模式

5
bootstrap/static.go
Unescape View File

@ -2,7 +2,6 @@ package bootstrap
import (
"bufio"
"embed"
"encoding/json"
"io"
"io/fs"
@ -45,7 +44,7 @@ func (b *GinFS) Exists(prefix string, filepath string) bool {
}
// InitStatic 初始化静态资源文件
func InitStatic(statics embed.FS) {
func InitStatic(statics fs.FS) {
if util.Exists(util.RelativePath(StaticFolder)) {
util.Log().Info("检测到 statics 目录存在，将使用此目录下的静态资源文件")
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
@ -96,7 +95,7 @@ func InitStatic(statics embed.FS) {
}
// Eject 抽离内置静态资源
func Eject(statics embed.FS) {
func Eject(statics fs.FS) {
// 初始化静态资源
embedFS, err := fs.Sub(statics, "assets/build")
if err != nil {

2
build.sh
Unescape View File

@ -31,6 +31,8 @@ buildAssets() {
yarn run build
cd build
rm -rf *.map
cd $REPO
zip -r - assets >assets.zip
}
buildBinary() {

10
go.mod
Unescape View File

@ -22,6 +22,7 @@ require (
github.com/hashicorp/go-version v1.3.0
github.com/jinzhu/gorm v1.9.11
github.com/juju/ratelimit v1.0.1
github.com/mholt/archiver/v4 v4.0.0-alpha.6
github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.2.0
@ -41,6 +42,7 @@ require (
require (
cloud.google.com/go v0.37.4 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@ -48,6 +50,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
@ -55,6 +58,7 @@ require (
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/protobuf v1.3.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
@ -63,6 +67,8 @@ require (
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/json-iterator/go v1.1.9 // indirect
github.com/katzenpost/core v0.0.7 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
@ -72,12 +78,16 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
golang.org/x/sys v0.0.0-20211020174200-9d6173849985 // indirect

25
go.sum
Unescape View File

@ -14,6 +14,8 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7I
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.31.5 h1:DFA7BzTydO4etqsTja+x7UfkOKQUv1xzEluLvNk81L0=
github.com/aws/aws-sdk-go v1.31.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
@ -38,6 +40,9 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 h1:tkum0XDgf
github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4 h1:rXVUYM3uQcdXgSvQ5Bo+JZFMnLi0H3jib+2mz7B6M4U=
github.com/duo-labs/webauthn v0.0.0-20191119193225-4bf9a0f776d4/go.mod h1:KR2KScxcZAWdZGOUnsPGjD3ow0cvNfv3WHXC/Xz+d9g=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@ -103,6 +108,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -156,6 +163,12 @@ github.com/katzenpost/core v0.0.7 h1:ZI4oYACe/3n2iS2XsCccGbTFdhhRJunjFW05Utwna+g
github.com/katzenpost/core v0.0.7/go.mod h1:UXMLmMXlBHrhMXhWTy4DvCXqwTRLOh4DP/mR1Cm1sR8=
github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nVr8KZVXJSS97Jo8pJ0jgq29P6H7dG0oplUA86MQw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@ -184,6 +197,8 @@ github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc=
github.com/mholt/archiver/v4 v4.0.0-alpha.6 h1:3wvos9Kn1GpKNBz+MpozinGREPslLo1ds1W16vTkErQ=
github.com/mholt/archiver/v4 v4.0.0-alpha.6/go.mod h1:9PTygYq90FQBWPspdwAng6dNjYiBuTYKqmA6c15KuCo=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -197,11 +212,16 @@ github.com/mojocn/base64Captcha v0.0.0-20190801020520-752b1cd608b2/go.mod h1:wAQ
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=
github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -254,11 +274,16 @@ github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible h1:dqpmYaez7VB
github.com/tencentcloud/tencentcloud-sdk-go v3.0.125+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac h1:PSBhZblOjdwH7SIVgcue+7OlnLHkM45KuScLZ+PiVbQ=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0=
github.com/upyun/go-sdk v2.1.0+incompatible/go.mod h1:eu3F5Uz4b9ZE5bE5QsCL6mgSNWRwfj0zpJ9J626HEqs=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=

22
main.go
Unescape View File

@ -1,13 +1,18 @@
package main
import (
"embed"
_ "embed"
"flag"
"io"
"io/fs"
"strings"
"github.com/cloudreve/Cloudreve/v3/bootstrap"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/cloudreve/Cloudreve/v3/routers"
"github.com/mholt/archiver/v4"
)
var (
@ -16,21 +21,28 @@ var (
scriptName string
)
//go:embed assets/build
var staticEmbed embed.FS
//go:embed assets.zip
var staticZip string
var staticFS fs.FS
func init() {
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")
flag.StringVar(&scriptName, "database-script", "", "运行内置数据库助手脚本")
flag.Parse()
bootstrap.Init(confPath, staticEmbed)
staticFS = archiver.ArchiveFS{
Stream: io.NewSectionReader(strings.NewReader(staticZip), 0, int64(len(staticZip))),
Format: archiver.Zip{},
}
bootstrap.Init(confPath, staticFS)
}
func main() {
if isEject {
// 开始导出内置静态资源文件
bootstrap.Eject(staticEmbed)
bootstrap.Eject(staticFS)
return
}

123
pkg/filesystem/archive.go
Unescape View File

@ -2,11 +2,9 @@ package filesystem
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
@ -18,8 +16,7 @@ import (
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gin-gonic/gin"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
"github.com/mholt/archiver/v4"
)
/* ===============
@ -168,7 +165,7 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
}
// Decompress 解压缩给定压缩文件到dst目录
func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
func (fs *FileSystem) Decompress(ctx context.Context, src, dst, encoding string) error {
err := fs.ResetFileIfNotExist(ctx, src)
if err != nil {
return err
@ -206,21 +203,41 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
}
defer zipFile.Close()
_, err = io.Copy(zipFile, fileStream)
// 下载前先判断是否是可解压的格式
format, readStream, err := archiver.Identify(fs.FileTarget[0].SourceName, fileStream)
if err != nil {
util.Log().Warning("无法写入临时压缩文件 %s , %s", tempZipFilePath, err)
util.Log().Warning("无法识别文件格式 %s , %s", fs.FileTarget[0].SourceName, err)
return err
}
zipFile.Close()
fileStream.Close()
extractor, ok := format.(archiver.Extractor)
if !ok {
return fmt.Errorf("file not an extractor %s", fs.FileTarget[0].SourceName)
}
// 解压缩文件
r, err := zip.OpenReader(tempZipFilePath)
if err != nil {
return err
// 只有zip格式可以多个文件同时上传
var isZip bool
switch extractor.(type) {
case archiver.Zip:
extractor = archiver.Zip{TextEncoding: encoding}
isZip = true
}
// 除了zip必须下载到本地，其余的可以边下载边解压
reader := readStream
if isZip {
_, err = io.Copy(zipFile, readStream)
if err != nil {
util.Log().Warning("无法写入临时压缩文件 %s , %s", tempZipFilePath, err)
return err
}
fileStream.Close()
// 设置文件偏移量
zipFile.Seek(0, io.SeekStart)
reader = zipFile
}
defer r.Close()
// 重设存储策略
fs.Policy = &fs.User.Policy
@ -236,64 +253,64 @@ func (fs *FileSystem) Decompress(ctx context.Context, src, dst string) error {
worker <- i
}
for _, f := range r.File {
fileName := f.Name
// 处理非UTF-8编码
if f.NonUTF8 {
i := bytes.NewReader([]byte(fileName))
decoder := transform.NewReader(i, simplifiedchinese.GB18030.NewDecoder())
content, _ := ioutil.ReadAll(decoder)
fileName = string(content)
// 上传文件函数
uploadFunc := func(fileStream io.ReadCloser, size int64, savePath, rawPath string) {
defer func() {
if isZip {
worker <- 1
wg.Done()
}
if err := recover(); err != nil {
util.Log().Warning("上传压缩包内文件时出错")
fmt.Println(err)
}
}()
err := fs.UploadFromStream(ctx, &fsctx.FileStream{
File: fileStream,
Size: uint64(size),
Name: path.Base(savePath),
VirtualPath: path.Dir(savePath),
}, true)
fileStream.Close()
if err != nil {
util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err)
}
}
rawPath := util.FormSlash(fileName)
// 解压缩文件，回调函数如果出错会停止解压的下一步进行，全部return nil
err = extractor.Extract(ctx, reader, nil, func(ctx context.Context, f archiver.File) error {
rawPath := util.FormSlash(f.NameInArchive)
savePath := path.Join(dst, rawPath)
// 路径是否合法
if !strings.HasPrefix(savePath, util.FillSlash(path.Clean(dst))) {
return fmt.Errorf("%s: illegal file path", f.Name)
util.Log().Warning("%s: illegal file path", f.NameInArchive)
return nil
}
// 如果是目录
if f.FileInfo().IsDir() {
if f.FileInfo.IsDir() {
fs.CreateDirectory(ctx, savePath)
continue
return nil
}
// 上传文件
fileStream, err := f.Open()
if err != nil {
util.Log().Warning("无法打开压缩包内文件%s , %s , 跳过", rawPath, err)
continue
return nil
}
select {
case <-worker:
if !isZip {
uploadFunc(fileStream, f.FileInfo.Size(), savePath, rawPath)
} else {
<-worker
wg.Add(1)
go func(fileStream io.ReadCloser, size int64) {
defer func() {
worker <- 1
wg.Done()
if err := recover(); err != nil {
util.Log().Warning("上传压缩包内文件时出错")
fmt.Println(err)
}
}()
err = fs.UploadFromStream(ctx, &fsctx.FileStream{
File: fileStream,
Size: uint64(size),
Name: path.Base(savePath),
VirtualPath: path.Dir(savePath),
}, true)
fileStream.Close()
if err != nil {
util.Log().Debug("无法上传压缩包内的文件%s , %s , 跳过", rawPath, err)
}
}(fileStream, f.FileInfo().Size())
go uploadFunc(fileStream, f.FileInfo.Size(), savePath, rawPath)
}
}
return nil
})
wg.Wait()
return nil
return err
}

47
pkg/filesystem/archive_test.go
Unescape View File

@ -8,6 +8,8 @@ import (
testMock "github.com/stretchr/testify/mock"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
@ -147,12 +149,24 @@ func (m MockRSC) Close() error {
return nil
}
var basepath string
func init() {
_, currentFile, _, _ := runtime.Caller(0)
basepath = filepath.Dir(currentFile)
}
func Path(rel string) string {
return filepath.Join(basepath, rel)
}
func TestFileSystem_Decompress(t *testing.T) {
asserts := assert.New(t)
ctx := context.Background()
fs := FileSystem{
User: &model.User{Model: gorm.Model{ID: 1}},
}
os.RemoveAll(util.RelativePath("tests/decompress"))
// 压缩文件不存在
{
@ -162,7 +176,7 @@ func TestFileSystem_Decompress(t *testing.T) {
// 查找压缩文件，未找到
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
err := fs.Decompress(ctx, "/1.zip", "/")
err := fs.Decompress(ctx, "/1.zip", "/", "")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
@ -174,7 +188,7 @@ func TestFileSystem_Decompress(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, errors.New("error"))
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
err := fs.Decompress(ctx, "/1.zip", "/", "")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.EqualError(err, "error")
@ -188,7 +202,7 @@ func TestFileSystem_Decompress(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{}, nil)
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
err := fs.Decompress(ctx, "/1.zip", "/", "")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
}
@ -201,13 +215,13 @@ func TestFileSystem_Decompress(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockNopRSC("1"), nil)
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
err := fs.Decompress(ctx, "/1.zip", "/", "")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.EqualError(err, "read error")
asserts.Contains(err.Error(), "read error")
}
// 无效zip文件
// 无法重设上传策略
{
cache.Set("setting_temp_path", "tests", 0)
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
@ -215,22 +229,7 @@ func TestFileSystem_Decompress(t *testing.T) {
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(MockRSC{rs: strings.NewReader("read")}, nil)
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.EqualError(err, "zip: not a valid zip file")
}
// 无法重设上传策略
{
zipFile, _ := os.Open(util.RelativePath("filesystem/tests/test.zip"))
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
fs.FileTarget[0].Policy.ID = 1
testHandler := new(FileHeaderMock)
testHandler.On("Get", testMock.Anything, "1.zip").Return(zipFile, nil)
fs.Handler = testHandler
err := fs.Decompress(ctx, "/1.zip", "/")
zipFile.Close()
err := fs.Decompress(ctx, "/1.zip", "/", "")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.True(util.IsEmpty(util.RelativePath("tests/decompress")))
@ -239,7 +238,7 @@ func TestFileSystem_Decompress(t *testing.T) {
// 无法上传，容量不足
{
cache.Set("setting_max_parallel_transfer", "1", 0)
zipFile, _ := os.Open(util.RelativePath("filesystem/tests/test.zip"))
zipFile, _ := os.Open(Path("tests/test.zip"))
fs.FileTarget = []model.File{{SourceName: "1.zip", Policy: model.Policy{Type: "mock"}}}
fs.FileTarget[0].Policy.ID = 1
fs.User.Policy.Type = "mock"
@ -247,7 +246,7 @@ func TestFileSystem_Decompress(t *testing.T) {
testHandler.On("Get", testMock.Anything, "1.zip").Return(zipFile, nil)
fs.Handler = testHandler
fs.Decompress(ctx, "/1.zip", "/")
fs.Decompress(ctx, "/1.zip", "/", "")
zipFile.Close()

14
pkg/task/decompress.go
Unescape View File

@ -20,8 +20,9 @@ type DecompressTask struct {
// DecompressProps 压缩任务属性
type DecompressProps struct {
Src string `json:"src"`
Dst string `json:"dst"`
Src string `json:"src"`
Dst string `json:"dst"`
Encoding string `json:"encoding"`
}
// Props 获取任务属性
@ -82,7 +83,7 @@ func (job *DecompressTask) Do() {
job.TaskModel.SetProgress(DecompressingProgress)
err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst)
err = fs.Decompress(context.Background(), job.TaskProps.Src, job.TaskProps.Dst, job.TaskProps.Encoding)
if err != nil {
job.SetErrorMsg("解压缩失败", err)
return
@ -91,12 +92,13 @@ func (job *DecompressTask) Do() {
}
// NewDecompressTask 新建压缩任务
func NewDecompressTask(user *model.User, src, dst string) (Job, error) {
func NewDecompressTask(user *model.User, src, dst, encoding string) (Job, error) {
newTask := &DecompressTask{
User: user,
TaskProps: DecompressProps{
Src: src,
Dst: dst,
Src: src,
Dst: dst,
Encoding: encoding,
},
}

4
pkg/task/decompress_test.go
Unescape View File

@ -99,7 +99,7 @@ func TestNewDecompressTask(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
job, err := NewDecompressTask(&model.User{}, "/", "/")
job, err := NewDecompressTask(&model.User{}, "/", "/", "utf-8")
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(job)
asserts.NoError(err)
@ -110,7 +110,7 @@ func TestNewDecompressTask(t *testing.T) {
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
job, err := NewDecompressTask(&model.User{}, "/", "/")
job, err := NewDecompressTask(&model.User{}, "/", "/", "utf-8")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(job)
asserts.Error(err)

23
service/explorer/objects.go
Unescape View File

@ -55,8 +55,9 @@ type ItemCompressService struct {
// ItemDecompressService 文件解压缩任务服务
type ItemDecompressService struct {
Src string `json:"src"`
Dst string `json:"dst" binding:"required,min=1,max=65535"`
Src string `json:"src"`
Dst string `json:"dst" binding:"required,min=1,max=65535"`
Encoding string `json:"encoding"`
}
// ItemPropertyService 获取对象属性服务
@ -127,13 +128,23 @@ func (service *ItemDecompressService) CreateDecompressTask(c *gin.Context) seria
return serializer.Err(serializer.CodeParamErr, "文件太大", nil)
}
// 必须是zip压缩包
if !strings.HasSuffix(file.Name, ".zip") {
return serializer.Err(serializer.CodeParamErr, "只能解压 ZIP 格式的压缩文件", nil)
// 支持的压缩格式后缀
var (
suffixes = []string{".zip", ".gz", ".xz", ".tar", ".rar"}
matched bool
)
for _, suffix := range suffixes {
if strings.HasSuffix(file.Name, suffix) {
matched = true
break
}
}
if !matched {
return serializer.Err(serializer.CodeParamErr, "不支持该格式的压缩文件", nil)
}
// 创建任务
job, err := task.NewDecompressTask(fs.User, service.Src, service.Dst)
job, err := task.NewDecompressTask(fs.User, service.Src, service.Dst, service.Encoding)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "任务创建失败", err)
}

Write Preview
Loading…
Cancel
Save