Feat: creat uri aria2 download task

pull/247/head
HFO4 4 years ago
parent 7804b4efb1
commit fe8f1b1ef5

@ -30,6 +30,7 @@ require (
github.com/stretchr/testify v1.4.0
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20200120023323-87ff3bc489ac
github.com/upyun/go-sdk v2.1.0+incompatible
github.com/zyxar/argo v0.0.0-20190709183644-6096bc0e6414
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
gopkg.in/go-playground/validator.v8 v8.18.2
gopkg.in/ini.v1 v1.51.0 // indirect

@ -89,6 +89,8 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+
github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU=
github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/gorm v1.9.11 h1:gaHGvE+UnWGlbWG4Y3FUwY1EcZ5n6S9WtqBA/uySMLE=
@ -113,11 +115,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@ -135,6 +139,7 @@ github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISe
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@ -182,6 +187,8 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0=
github.com/upyun/go-sdk v2.1.0+incompatible/go.mod h1:eu3F5Uz4b9ZE5bE5QsCL6mgSNWRwfj0zpJ9J626HEqs=
github.com/zyxar/argo v0.0.0-20190709183644-6096bc0e6414 h1:Lik9S6KuMnplY7s8tSeCvYwIUOxHtnO0bPLEHOFBL5g=
github.com/zyxar/argo v0.0.0-20190709183644-6096bc0e6414/go.mod h1:UdVgwBBjhPE3QYySyknw2WlXX0CFrHKoehgZoFI92r8=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

@ -2,6 +2,7 @@ package main
import (
"github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/aria2"
"github.com/HFO4/cloudreve/pkg/auth"
"github.com/HFO4/cloudreve/pkg/authn"
"github.com/HFO4/cloudreve/pkg/cache"
@ -22,6 +23,7 @@ func init() {
model.Init()
authn.Init()
task.Init()
aria2.Init()
}
auth.Init()
}

@ -10,9 +10,10 @@ type Download struct {
gorm.Model
Status int // 任务状态
Type int // 任务类型
Source string // 文件下载地址
Name string // 任务文件名
Size uint64 // 文件大小
PID string // 任务ID
GID string // 任务ID
Path string `gorm:"type:text"` // 存储路径
Attrs string `gorm:"type:text"` // 任务状态属性
FolderID uint // 存储父目录ID

@ -25,13 +25,14 @@ type Group struct {
// GroupOption 用户组其他配置
type GroupOption struct {
ArchiveDownload bool `json:"archive_download,omitempty"`
ArchiveTask bool `json:"archive_task,omitempty"`
CompressSize uint64 `json:"compress_size,omitempty"`
ArchiveDownload bool `json:"archive_download,omitempty"` // 打包下载
ArchiveTask bool `json:"archive_task,omitempty"` // 在线压缩
CompressSize uint64 `json:"compress_size,omitempty"` // 可压缩大小
DecompressSize uint64 `json:"decompress_size,omitempty"`
OneTimeDownload bool `json:"one_time_download,omitempty"`
ShareDownload bool `json:"share_download,omitempty"`
ShareFree bool `json:"share_free,omitempty"`
Aria2 bool `json:"aria2,omitempty"` // 离线下载
}
// GetAria2Option 获取用户离线下载设备

@ -29,7 +29,8 @@ func migration() {
if conf.DatabaseConfig.Type == "mysql" {
DB = DB.Set("gorm:table_options", "ENGINE=InnoDB")
}
DB.AutoMigrate(&User{}, &Setting{}, &Group{}, &Policy{}, &Folder{}, &File{}, &StoragePack{}, &Share{}, &Task{})
DB.AutoMigrate(&User{}, &Setting{}, &Group{}, &Policy{}, &Folder{}, &File{}, &StoragePack{}, &Share{},
&Task{}, &Download{})
// 创建初始存储策略
addDefaultPolicy()
@ -110,6 +111,7 @@ solid #e9e9e9;"bgcolor="#fff"><tbody><tr style="font-family: 'Helvetica Neue',He
{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
{Name: "aria2_call_timeout", Value: `5`, Type: "timeout"},
{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
{Name: "allowdVisitorDownload", Value: `false`, Type: "share"},
{Name: "login_captcha", Value: `0`, Type: "login"},
@ -155,9 +157,9 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"light":"#7986cb","main":"#3f51b5","dark":"#303f9f","contrastText":"#fff"},"secondary":{"light":"#ff4081","main":"#f50057","dark":"#c51162","contrastText":"#fff"},"error":{"light":"#e57373","main":"#f44336","dark":"#d32f2f","contrastText":"#fff"},"explorer":{"filename":"#474849","icon":"#8f8f8f","bgSelected":"#D5DAF0","emptyIcon":"#e8e8e8"}}}}`, Type: "basic"},
{Name: "refererCheck", Value: `true`, Type: "share"},
{Name: "header", Value: `X-Sendfile`, Type: "download"},
{Name: "aria2_tmppath", Value: `/path/to/public/download`, Type: "aria2"},
{Name: "aria2_token", Value: `your token`, Type: "aria2"},
{Name: "aria2_rpcurl", Value: `http://127.0.0.1:6800/`, Type: "aria2"},
{Name: "aria2_token", Value: `your token`, Type: "aria2"},
{Name: "aria2_temp_path", Value: `F:\aria2-1.33.1-win-64bit-build1\temp`, Type: "aria2"},
{Name: "aria2_options", Value: `{"max-tries":5}`, Type: "aria2"},
{Name: "max_worker_num", Value: `10`, Type: "task"},
{Name: "max_parallel_transfer", Value: `4`, Type: "task"},

@ -39,7 +39,7 @@ func GetSettingByName(name string) string {
}
// GetSettingByNames 用多个 Name 获取设置值
func GetSettingByNames(names []string) map[string]string {
func GetSettingByNames(names ...string) map[string]string {
var queryRes []Setting
res, miss := cache.GetSettings(names, "setting_")

@ -67,7 +67,7 @@ func TestGetSettingByNames(t *testing.T) {
AddRow("siteName", "Cloudreve", "basic").
AddRow("siteDes", "Something wonderful", "basic")
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
settings := GetSettingByNames([]string{"siteName", "siteDes"})
settings := GetSettingByNames("siteName", "siteDes")
asserts.Equal(map[string]string{
"siteName": "Cloudreve",
"siteDes": "Something wonderful",
@ -78,7 +78,7 @@ func TestGetSettingByNames(t *testing.T) {
rows = sqlmock.NewRows([]string{"name", "value", "type"}).
AddRow("siteName2", "Cloudreve", "basic")
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
settings = GetSettingByNames([]string{"siteName2", "siteDes2333"})
settings = GetSettingByNames("siteName2", "siteDes2333")
asserts.Equal(map[string]string{
"siteName2": "Cloudreve",
}, settings)
@ -87,14 +87,14 @@ func TestGetSettingByNames(t *testing.T) {
//找不到设置时
rows = sqlmock.NewRows([]string{"name", "value", "type"})
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WillReturnRows(rows)
settings = GetSettingByNames([]string{"siteName2333", "siteDes2333"})
settings = GetSettingByNames("siteName2333", "siteDes2333")
asserts.Equal(map[string]string{}, settings)
asserts.NoError(mock.ExpectationsWereMet())
// 一个设置命中缓存
mock.ExpectQuery("^SELECT \\* FROM `(.+)` WHERE `(.+)`\\.`deleted_at` IS NULL AND(.+)$").WithArgs("siteDes2").WillReturnRows(sqlmock.NewRows([]string{"name", "value", "type"}).
AddRow("siteDes2", "Cloudreve2", "basic"))
settings = GetSettingByNames([]string{"siteName", "siteDes2"})
settings = GetSettingByNames("siteName", "siteDes2")
asserts.Equal(map[string]string{
"siteName": "Cloudreve",
"siteDes2": "Cloudreve2",

@ -0,0 +1,75 @@
package aria2
import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"net/url"
)
// Instance 默认使用的Aria2处理实例
var Instance Aria2 = &DummyAria2{}
// Aria2 离线下载处理接口
type Aria2 interface {
// CreateTask 创建新的任务
CreateTask(task *model.Download) error
}
const (
// URLTask 从URL添加的任务
URLTask = iota
// TorrentTask 种子任务
TorrentTask
)
const (
// Ready 准备就绪
Ready = iota
)
var (
// ErrNotEnabled 功能未开启错误
ErrNotEnabled = serializer.NewError(serializer.CodeNoPermissionErr, "离线下载功能未开启", nil)
)
// DummyAria2 未开启Aria2功能时使用的默认处理器
type DummyAria2 struct {
}
// CreateTask 创建新任务,此处直接返回未开启错误
func (instance *DummyAria2) CreateTask(task *model.Download) error {
return ErrNotEnabled
}
// Init 初始化
func Init() {
options := model.GetSettingByNames("aria2_rpcurl", "aria2_token", "aria2_options")
timeout := model.GetIntSetting("aria2_call_timeout", 5)
if options["aria2_rpcurl"] == "" {
// 未开启Aria2服务
return
}
util.Log().Info("初始化 aria2 RPC 服务[%s]", options["aria2_rpcurl"])
client := &RPCService{}
if previousClient, ok := Instance.(*RPCService); ok {
client = previousClient
}
// 解析RPC服务地址
server, err := url.Parse(options["aria2_rpcurl"])
if err != nil {
util.Log().Warning("无法解析 aria2 RPC 服务地址,%s", err)
return
}
server.Path = "/jsonrpc"
// todo 加载自定义下载配置
if err := client.Init(server.String(), options["aria2_token"], timeout, []interface{}{}); err != nil {
util.Log().Warning("初始化 aria2 RPC 服务失败,%s", err)
return
}
Instance = client
}

@ -0,0 +1,58 @@
package aria2
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/zyxar/argo/rpc"
"path/filepath"
"strconv"
"time"
)
// RPCService 通过RPC服务的Aria2任务管理器
type RPCService struct {
options *clientOptions
caller rpc.Client
}
type clientOptions struct {
Options []interface{} // 创建下载时额外添加的设置
}
// Init 初始化
func (client *RPCService) Init(server, secret string, timeout int, options []interface{}) error {
// 客户端已存在,则关闭先前连接
if client.caller != nil {
client.caller.Close()
}
client.options = &clientOptions{
Options: options,
}
caller, err := rpc.New(context.Background(), server, secret, time.Duration(timeout)*time.Second,
rpc.DummyNotifier{})
client.caller = caller
return err
}
// CreateTask 创建新任务
func (client *RPCService) CreateTask(task *model.Download) error {
// 生成存储路径
task.Path = filepath.Join(
model.GetSettingByName("aria2_temp_path"),
"aria2",
strconv.FormatInt(time.Now().UnixNano(), 10),
)
// 创建下载任务
gid, err := client.caller.AddURI(task.Source, map[string]string{"dir": task.Path})
if err != nil || gid == "" {
return err
}
// 保存到数据库
task.GID = gid
_, err = task.Create()
return err
}

@ -290,7 +290,8 @@ func (client *Client) BatchDelete(ctx context.Context, dst []string) ([]string,
return finalRes, err
}
// Delete 并行删除文件返回删除失败的文件及第一个遇到的错误最多删除20个
// Delete 并行删除文件,返回删除失败的文件,及第一个遇到的错误,
// 由于API限制最多删除20个
func (client *Client) Delete(ctx context.Context, dst []string) ([]string, error) {
body := client.makeBatchDeleteRequestsBody(dst)
res, err := client.requestWithStr(ctx, "POST", client.getRequestURL("$batch"), body, 200)

@ -0,0 +1,17 @@
package controllers
import (
"github.com/HFO4/cloudreve/service/aria2"
"github.com/gin-gonic/gin"
)
// AddAria2URL 添加离线下载URL
func AddAria2URL(c *gin.Context) {
var addService aria2.AddURLService
if err := c.ShouldBindJSON(&addService); err == nil {
res := addService.Add(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}

@ -16,6 +16,7 @@ func ParamErrorMsg(filed string, tag string) string {
"Password": "密码",
"Path": "路径",
"SourceID": "原始资源",
"URL": "链接",
}
// 未通过的规则与中文对应
tagMap := map[string]string{

@ -11,7 +11,7 @@ import (
// SiteConfig 获取站点全局配置
func SiteConfig(c *gin.Context) {
siteConfig := model.GetSettingByNames([]string{
siteConfig := model.GetSettingByNames(
"siteName",
"login_captcha",
"qq_login",
@ -25,7 +25,7 @@ func SiteConfig(c *gin.Context) {
"share_score_rate",
"home_view_method",
"share_view_method",
})
)
// 如果已登录,则同时返回用户信息
user, _ := c.Get("user")

@ -274,7 +274,10 @@ func InitMasterRouter() *gin.Engine {
}
// 离线下载任务
//aria2 := auth.Group("aria2")
aria2 := auth.Group("aria2")
{
aria2.POST("url", controllers.AddAria2URL)
}
// 目录
directory := auth.Group("directory")

@ -0,0 +1,53 @@
package aria2
import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/aria2"
"github.com/HFO4/cloudreve/pkg/filesystem"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/gin-gonic/gin"
)
// AddURLService 添加URL离线下载服务
type AddURLService struct {
URL string `json:"url" binding:"required"`
Dst string `json:"dst" binding:"required,min=1,max=65535"`
}
// Add 创建新的链接离线下载任务
func (service *AddURLService) Add(c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 检查用户组权限
if !fs.User.Group.OptionsSerialized.Aria2 {
return serializer.Err(serializer.CodeGroupNotAllowed, "当前用户组无法进行此操作", nil)
}
// 存放目录是否存在
var (
exist bool
parent *model.Folder
)
if exist, parent = fs.IsPathExist(service.Dst); !exist {
return serializer.Err(serializer.CodeNotFound, "存放路径不存在", nil)
}
// 创建任务
task := &model.Download{
Status: aria2.Ready,
Type: aria2.URLTask,
FolderID: parent.ID,
UserID: fs.User.ID,
Source: service.URL,
}
if err := aria2.Instance.CreateTask(task); err != nil {
return serializer.Err(serializer.CodeNotSet, "任务创建失败", err)
}
return serializer.Response{}
}
Loading…
Cancel
Save