You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
cloudreve/routers/controllers/slave.go

279 lines
7.1 KiB

package controllers
import (
"context"
"net/url"
"strconv"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver/local"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/service/admin"
Feat: aria2 download and transfer in slave node (#1040) * Feat: retrieve nodes from data table * Feat: master node ping slave node in REST API * Feat: master send scheduled ping request * Feat: inactive nodes recover loop * Modify: remove database operations from aria2 RPC caller implementation * Feat: init aria2 client in master node * Feat: Round Robin load balancer * Feat: create and monitor aria2 task in master node * Feat: salve receive and handle heartbeat * Fix: Node ID will be 0 in download record generated in older version * Feat: sign request headers with all `X-` prefix * Feat: API call to slave node will carry meta data in headers * Feat: call slave aria2 rpc method from master * Feat: get slave aria2 task status Feat: encode slave response data using gob * Feat: aria2 callback to master node / cancel or select task to slave node * Fix: use dummy aria2 client when caller initialize failed in master node * Feat: slave aria2 status event callback / salve RPC auth * Feat: prototype for slave driven filesystem * Feat: retry for init aria2 client in master node * Feat: init request client with global options * Feat: slave receive async task from master * Fix: competition write in request header * Refactor: dependency initialize order * Feat: generic message queue implementation * Feat: message queue implementation * Feat: master waiting slave transfer result * Feat: slave transfer file in stateless policy * Feat: slave transfer file in slave policy * Feat: slave transfer file in local policy * Feat: slave transfer file in OneDrive policy * Fix: failed to initialize update checker http client * Feat: list slave nodes for dashboard * Feat: test aria2 rpc connection in slave * Feat: add and save node * Feat: add and delete node in node pool * Fix: temp file cannot be removed when aria2 task fails * Fix: delete node in admin panel * Feat: edit node and get node info * Modify: delete unused settings
3 years ago
"github.com/cloudreve/Cloudreve/v3/service/aria2"
"github.com/cloudreve/Cloudreve/v3/service/explorer"
Feat: aria2 download and transfer in slave node (#1040) * Feat: retrieve nodes from data table * Feat: master node ping slave node in REST API * Feat: master send scheduled ping request * Feat: inactive nodes recover loop * Modify: remove database operations from aria2 RPC caller implementation * Feat: init aria2 client in master node * Feat: Round Robin load balancer * Feat: create and monitor aria2 task in master node * Feat: salve receive and handle heartbeat * Fix: Node ID will be 0 in download record generated in older version * Feat: sign request headers with all `X-` prefix * Feat: API call to slave node will carry meta data in headers * Feat: call slave aria2 rpc method from master * Feat: get slave aria2 task status Feat: encode slave response data using gob * Feat: aria2 callback to master node / cancel or select task to slave node * Fix: use dummy aria2 client when caller initialize failed in master node * Feat: slave aria2 status event callback / salve RPC auth * Feat: prototype for slave driven filesystem * Feat: retry for init aria2 client in master node * Feat: init request client with global options * Feat: slave receive async task from master * Fix: competition write in request header * Refactor: dependency initialize order * Feat: generic message queue implementation * Feat: message queue implementation * Feat: master waiting slave transfer result * Feat: slave transfer file in stateless policy * Feat: slave transfer file in slave policy * Feat: slave transfer file in local policy * Feat: slave transfer file in OneDrive policy * Fix: failed to initialize update checker http client * Feat: list slave nodes for dashboard * Feat: test aria2 rpc connection in slave * Feat: add and save node * Feat: add and delete node in node pool * Fix: temp file cannot be removed when aria2 task fails * Fix: delete node in admin panel * Feat: edit node and get node info * Modify: delete unused settings
3 years ago
"github.com/cloudreve/Cloudreve/v3/service/node"
"github.com/gin-gonic/gin"
)
// SlaveUpload 从机文件上传
func SlaveUpload(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
ctx = context.WithValue(ctx, fsctx.GinCtx, c)
defer cancel()
// 创建匿名文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
c.JSON(200, serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err))
return
}
5 years ago
fs.Handler = local.Driver{}
// 从请求中取得上传策略
uploadPolicyRaw := c.GetHeader("X-Policy")
if uploadPolicyRaw == "" {
c.JSON(200, serializer.ParamErr("未指定上传策略", nil))
return
}
// 解析上传策略
uploadPolicy, err := serializer.DecodeUploadPolicy(uploadPolicyRaw)
if err != nil {
c.JSON(200, serializer.ParamErr("上传策略格式有误", err))
return
}
ctx = context.WithValue(ctx, fsctx.UploadPolicyCtx, *uploadPolicy)
// 取得文件大小
fileSize, err := strconv.ParseUint(c.Request.Header.Get("Content-Length"), 10, 64)
if err != nil {
c.JSON(200, ErrorResponse(err))
return
}
// 解码文件名和路径
fileName, err := url.QueryUnescape(c.Request.Header.Get("X-FileName"))
if err != nil {
c.JSON(200, ErrorResponse(err))
return
}
fileData := local.FileStream{
MIMEType: c.Request.Header.Get("Content-Type"),
File: c.Request.Body,
Name: fileName,
Size: fileSize,
}
// 给文件系统分配钩子
fs.Use("BeforeUpload", filesystem.HookSlaveUploadValidate)
fs.Use("AfterUploadCanceled", filesystem.HookDeleteTempFile)
fs.Use("AfterUpload", filesystem.SlaveAfterUpload)
fs.Use("AfterValidateFailed", filesystem.HookDeleteTempFile)
// 是否允许覆盖
if c.Request.Header.Get("X-Overwrite") == "false" {
ctx = context.WithValue(ctx, fsctx.DisableOverwrite, true)
}
// 执行上传
err = fs.Upload(ctx, fileData)
if err != nil {
c.JSON(200, serializer.Err(serializer.CodeUploadFailed, err.Error(), err))
return
}
c.JSON(200, serializer.Response{
Code: 0,
})
}
// SlaveDownload 从机文件下载,此请求返回的HTTP状态码不全为200
func SlaveDownload(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.SlaveDownloadService
if err := c.ShouldBindUri(&service); err == nil {
res := service.ServeFile(ctx, c, true)
if res.Code != 0 {
c.JSON(400, res)
}
} else {
c.JSON(400, ErrorResponse(err))
}
}
// SlavePreview 从机文件预览
func SlavePreview(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.SlaveDownloadService
if err := c.ShouldBindUri(&service); err == nil {
res := service.ServeFile(ctx, c, false)
if res.Code != 0 {
c.JSON(200, res)
}
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveThumb 从机文件缩略图
func SlaveThumb(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.SlaveFileService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Thumb(ctx, c)
if res.Code != 0 {
c.JSON(200, res)
}
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveDelete 从机删除
func SlaveDelete(c *gin.Context) {
// 创建上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var service explorer.SlaveFilesService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Delete(ctx, c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlavePing 从机测试
func SlavePing(c *gin.Context) {
var service admin.SlavePingService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Test()
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveList 从机列出文件
func SlaveList(c *gin.Context) {
var service explorer.SlaveListService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.List(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
Feat: aria2 download and transfer in slave node (#1040) * Feat: retrieve nodes from data table * Feat: master node ping slave node in REST API * Feat: master send scheduled ping request * Feat: inactive nodes recover loop * Modify: remove database operations from aria2 RPC caller implementation * Feat: init aria2 client in master node * Feat: Round Robin load balancer * Feat: create and monitor aria2 task in master node * Feat: salve receive and handle heartbeat * Fix: Node ID will be 0 in download record generated in older version * Feat: sign request headers with all `X-` prefix * Feat: API call to slave node will carry meta data in headers * Feat: call slave aria2 rpc method from master * Feat: get slave aria2 task status Feat: encode slave response data using gob * Feat: aria2 callback to master node / cancel or select task to slave node * Fix: use dummy aria2 client when caller initialize failed in master node * Feat: slave aria2 status event callback / salve RPC auth * Feat: prototype for slave driven filesystem * Feat: retry for init aria2 client in master node * Feat: init request client with global options * Feat: slave receive async task from master * Fix: competition write in request header * Refactor: dependency initialize order * Feat: generic message queue implementation * Feat: message queue implementation * Feat: master waiting slave transfer result * Feat: slave transfer file in stateless policy * Feat: slave transfer file in slave policy * Feat: slave transfer file in local policy * Feat: slave transfer file in OneDrive policy * Fix: failed to initialize update checker http client * Feat: list slave nodes for dashboard * Feat: test aria2 rpc connection in slave * Feat: add and save node * Feat: add and delete node in node pool * Fix: temp file cannot be removed when aria2 task fails * Fix: delete node in admin panel * Feat: edit node and get node info * Modify: delete unused settings
3 years ago
// SlaveHeartbeat 接受主机心跳包
func SlaveHeartbeat(c *gin.Context) {
var service serializer.NodePingReq
if err := c.ShouldBindJSON(&service); err == nil {
res := node.HandleMasterHeartbeat(&service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveAria2Create 创建 Aria2 任务
func SlaveAria2Create(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.Add(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveAria2Status 查询从机 Aria2 任务状态
func SlaveAria2Status(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveStatus(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveCancelAria2Task 取消从机离线下载任务
func SlaveCancelAria2Task(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveCancel(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveSelectTask 从机选取离线下载文件
func SlaveSelectTask(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveSelect(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveCreateTransferTask 从机创建中转任务
func SlaveCreateTransferTask(c *gin.Context) {
var service serializer.SlaveTransferReq
if err := c.ShouldBindJSON(&service); err == nil {
res := explorer.CreateTransferTask(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveNotificationPush 处理从机发送的消息推送
func SlaveNotificationPush(c *gin.Context) {
var service node.SlaveNotificationService
if err := c.ShouldBindUri(&service); err == nil {
res := service.HandleSlaveNotificationPush(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveGetOneDriveCredential 从机获取主机的OneDrive存储策略凭证
func SlaveGetOneDriveCredential(c *gin.Context) {
var service node.OneDriveCredentialService
if err := c.ShouldBindUri(&service); err == nil {
res := service.Get(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveSelectTask 从机删除离线下载临时文件
func SlaveDeleteTempFile(c *gin.Context) {
var service serializer.SlaveAria2Call
if err := c.ShouldBindJSON(&service); err == nil {
res := aria2.SlaveDeleteTemp(c, &service)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}