msb-public-java
/
xxl-job
mirror of https://gitee.com/xuxueli0323/xxl-job.git
4
2
Code Issues Projects Releases Wiki Activity

脚本任务实现：Shell和Python

Browse Source
v1.7
xueli.xue 8 years ago
parent 7a129468a4
commit 935cc1276f
4 changed files with 128 additions and 113 deletions
Show Stats Download Patch File Download Diff File

16
xxl-job-admin/src/main/webapp/WEB-INF/template/jobinfo/jobinfo.index.ftl
Unescape View File

@ -193,17 +193,15 @@ public class DemoGlueJobHandler extends IJobHandler {
<textarea class="glueSource_shell" style="display:none;" >
#!/bin/bash
echo hello shell
echo "xxl-job: hello shell"
for x in 1 2 3 4
for item in 1 2 3
do
echo number=$x
echo "shell : $item"
sleep 1s
done
echo1 111
printf 666
echo2 222
echo "Good bye!"
</textarea>
<textarea class="glueSource_python" style="display:none;" >
#!/usr/bin/python
@ -214,11 +212,11 @@ import time
logging.basicConfig(level=logging.DEBUG)
logging.info('hello python')
logging.info('xxl-job: hello python')
for num in range(0, 3):
for num in range(1, 3):
time.sleep(1)
logging.info('当前序号 :' + str(num) )
logging.info('python :' + str(num) )
logging.info('Good bye!')
</textarea>

77
xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java
Unescape View File

@ -9,15 +9,12 @@ import com.xxl.job.core.glue.GlueFactory;
import com.xxl.job.core.glue.GlueTypeEnum;
import com.xxl.job.core.handler.IJobHandler;
import com.xxl.job.core.handler.impl.GlueJobHandler;
import com.xxl.job.core.handler.impl.ScriptJobHandler;
import com.xxl.job.core.log.XxlJobFileAppender;
import com.xxl.job.core.thread.JobThread;
import com.xxl.job.core.util.ScriptUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Date;
/**
@ -85,11 +82,11 @@ public class ExecutorBizImpl implements ExecutorBiz {
} else if (GlueTypeEnum.GLUE_GROOVY==GlueTypeEnum.match(triggerParam.getGlueType())) {
// valid exists job thread：change handler or glue timeout, need kill old thread
// valid exists job thread：change handler or gluesource updated, need kill old thread
if (jobThread != null &&
!(jobThread.getHandler() instanceof GlueJobHandler
&& ((GlueJobHandler) jobThread.getHandler()).getGlueUpdatetime()==triggerParam.getGlueUpdatetime() )) {
// change glue model or glue timeout, kill old job thread
// change glue model or gluesource updated, kill old job thread
jobThread.toStop("更换任务模式或JobHandler,终止旧任务线程");
jobThread.interrupt();
XxlJobExecutor.removeJobThread(triggerParam.getJobId());
@ -107,61 +104,25 @@ public class ExecutorBizImpl implements ExecutorBiz {
}
jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), new GlueJobHandler(jobHandler, triggerParam.getGlueUpdatetime()));
}
} else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType())) {
// make path
String scriptPath = XxlJobFileAppender.filePath + "gluesource/";
String scriptFileName = triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".sh";
// valid file
File scriptFile = new File(scriptPath, scriptFileName);
if (!scriptFile.exists()) {
// valid glue source
if (triggerParam.getGlueSource()==null) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "glueSource is null.");
}
// .../gluesource/
File scriptPathDir = new File(scriptPath);
if (!scriptPathDir.exists()) {
scriptPathDir.mkdirs();
}
// .../gluesource/666-156465656.sh
scriptFile = new File(scriptPath, scriptFileName);
FileOutputStream fos = null;
try {
scriptFile.createNewFile();
fos = new FileOutputStream(scriptFile, true);
fos.write(triggerParam.getGlueSource().getBytes("utf-8"));
fos.flush();
} catch (IOException e) {
logger.error(e.getMessage(), e);
return new ReturnT<String>(ReturnT.FAIL_CODE, e.getMessage());
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
} else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType())
|| GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType()) ) {
// valid exists job thread：change script or gluesource updated, need kill old thread
if (jobThread != null &&
!(jobThread.getHandler() instanceof ScriptJobHandler
&& ((ScriptJobHandler) jobThread.getHandler()).getGlueUpdatetime()==triggerParam.getGlueUpdatetime() )) {
// change glue model or gluesource updated, kill old job thread
jobThread.toStop("更换任务模式或JobHandler,终止旧任务线程");
jobThread.interrupt();
XxlJobExecutor.removeJobThread(triggerParam.getJobId());
jobThread = null;
}
// log File
String logFileName = XxlJobFileAppender.makeLogFileName(new Date(triggerParam.getLogDateTim()), triggerParam.getLogId());
// run script
ScriptUtil.execToFile("python", scriptFile.getName(), (XxlJobFileAppender.filePath + logFileName) );
return ReturnT.FAIL;
} else if (GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType())) {
String scriptFilePath = XxlJobFileAppender.filePath + "gluesource/" + triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".py";
// make thread: new or exists invalid
if (jobThread == null) {
ScriptJobHandler scriptJobHandler = new ScriptJobHandler(triggerParam.getJobId(), triggerParam.getGlueUpdatetime(), triggerParam.getGlueSource(), GlueTypeEnum.match(triggerParam.getGlueType()));
jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), scriptJobHandler);
}
} else {
return new ReturnT<String>(ReturnT.FAIL_CODE, "glueType[" + triggerParam.getGlueType() + "] is not valid.");
}

55
xxl-job-core/src/main/java/com/xxl/job/core/handler/impl/ScriptJobHandler.java
Unescape View File

@ -0,0 +1,55 @@
package com.xxl.job.core.handler.impl;
import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.glue.GlueTypeEnum;
import com.xxl.job.core.handler.IJobHandler;
import com.xxl.job.core.log.XxlJobFileAppender;
import com.xxl.job.core.util.ScriptUtil;
/**
* Created by xuxueli on 17/4/27.
*/
public class ScriptJobHandler extends IJobHandler {
private int jobId;
private long glueUpdatetime;
private String gluesource;
private GlueTypeEnum glueType;
public ScriptJobHandler(int jobId, long glueUpdatetime, String gluesource, GlueTypeEnum glueType){
this.jobId = jobId;
this.glueUpdatetime = glueUpdatetime;
this.gluesource = gluesource;
this.glueType = glueType;
}
public long getGlueUpdatetime() {
return glueUpdatetime;
}
@Override
public ReturnT<String> execute(String... params) throws Exception {
// cmd + script-file-name
String cmd = "bash";
String scriptFileName = null;
if (GlueTypeEnum.GLUE_SHELL == glueType) {
cmd = "bash";
scriptFileName = XxlJobFileAppender.filePath.concat("gluesource/").concat(String.valueOf(jobId)).concat("_").concat(String.valueOf(glueUpdatetime)).concat(".sh");
} else if (GlueTypeEnum.GLUE_PYTHON == glueType) {
cmd = "python";
scriptFileName = XxlJobFileAppender.filePath.concat("gluesource/").concat(String.valueOf(jobId)).concat("_").concat(String.valueOf(glueUpdatetime)).concat(".py");
}
// make script file
ScriptUtil.markScriptFile(scriptFileName, gluesource);
// log file
String logFileName = XxlJobFileAppender.filePath.concat(XxlJobFileAppender.contextHolder.get());
// invoke
ScriptUtil.execToFile(cmd, scriptFileName, logFileName);
return ReturnT.SUCCESS;
}
}

93
xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java
Unescape View File

@ -1,49 +1,57 @@
package com.xxl.job.core.util;
import com.xxl.job.core.log.XxlJobFileAppender;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
/**
* 1、内嵌编译器如"PythonInterpreter"无法引用扩展包，因此推荐使用java调用控制台进程方式"Runtime.getRuntime().exec()"来运行脚本(shell或python)；
* 2、因为通过java调用控制台进程方式实现，需要保证目标机器PATH路径正确配置对应编译器；
* 3、暂时脚本执行日志只能在脚本执行结束后一次性获取，无法保证实时性；因此为确保日志实时性，可改为将脚本打印的日志存储在指定的日志文件上；
*
* 知识点：
* 1、日志输出到日志文件：[>>logfile 2>&1]：将错误输出2以及标准输出1都一起以附加写方式导入logfile文件
* 2、python 异常输出优先级高于标准输出，体现在Log文件中，因此推荐通过logging方式打日志保持和异常信息一致；否则用prinf日志顺序会错乱
* 4、python 异常输出优先级高于标准输出，体现在Log文件中，因此推荐通过logging方式打日志保持和异常信息一致；否则用prinf日志顺序会错乱
*
* Created by xuxueli on 17/2/25.
*/
public class ScriptUtil {
private static String pyCmd = "python";
private static String shllCmd = "bash";
private static String pyFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/pytest.py";
private static String shellFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/shelltest.sh";
private static String pyLogFile = "/Users/xuxueli/Downloads/tmp/pylog.log";
private static String shLogFile = "/Users/xuxueli/Downloads/tmp/shlog.log";
public static void main(String[] args) {
String command = pyCmd;
String filename = pyFile;
String logFile = pyLogFile;
if (false) {
command = shllCmd;
filename = shellFile;
logFile = shLogFile;
/**
* make script file
*
* @param scriptFileName
* @param content
* @throws IOException
*/
public static void markScriptFile(String scriptFileName, String content) throws IOException {
// filePath/
File filePathDir = new File(XxlJobFileAppender.filePath);
if (!filePathDir.exists()) {
filePathDir.mkdirs();
}
execToFile(command, filename, logFile);
}
// filePath/gluesource/
File filePathSourceDir = new File(filePathDir, "gluesource");
if (!filePathSourceDir.exists()) {
filePathSourceDir.mkdirs();
}
public static File markScriptFile(){
return null;
// make file, filePath/gluesource/666-123456789.py
FileOutputStream fileOutputStream = null;
try {
fileOutputStream = new FileOutputStream(scriptFileName);
fileOutputStream.write(content.getBytes("UTF-8"));
fileOutputStream.close();
} catch (Exception e) {
throw e;
}finally{
if(fileOutputStream != null){
fileOutputStream.close();
}
}
}
/**
@ -58,29 +66,22 @@ public class ScriptUtil {
* @param scriptFile
* @param logFile
*/
public static void execToFile(String command, String scriptFile, String logFile){
try {
// 标准输出：print （null if watchdog timeout）
// 错误输出：logging + 异常 （still exists if watchdog timeout）
// 标准输出
FileOutputStream fileOutputStream = new FileOutputStream(logFile);
PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null);
public static void execToFile(String command, String scriptFile, String logFile) throws IOException {
// 标准输出：print （null if watchdog timeout）
// 错误输出：logging + 异常 （still exists if watchdog timeout）
// 标准输入
FileOutputStream fileOutputStream = new FileOutputStream(logFile, true);
PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null);
// command
CommandLine commandline = new CommandLine(command);
commandline.addArgument(scriptFile);
// command
CommandLine commandline = new CommandLine(command);
commandline.addArgument(scriptFile);
// exec
DefaultExecutor exec = new DefaultExecutor();
exec.setExitValues(null);
exec.setStreamHandler(streamHandler);
int exitValue = exec.execute(commandline);
} catch (Exception e) {
e.printStackTrace();
}
/*Process process = Runtime.getRuntime().exec(cmdarray);
IOUtils.copy(process.getInputStream(), out);
IOUtils.copy(process.getErrorStream(), out);*/
// exec
DefaultExecutor exec = new DefaultExecutor();
exec.setExitValues(null);
exec.setStreamHandler(streamHandler);
int exitValue = exec.execute(commandline);
}
}

Write Preview
Loading…
Cancel
Save