From d2eafe20cd0088559be1a7336c17745454a0065f Mon Sep 17 00:00:00 2001 From: "xueli.xue" Date: Wed, 26 Apr 2017 23:30:51 +0800 Subject: [PATCH] Coding --- xxl-job-core/pom.xml | 7 ++ .../job/core/biz/impl/ExecutorBizImpl.java | 60 +++++++++++++ .../xxl/job/core/executor/XxlJobExecutor.java | 2 + .../com/xxl/job/core/glue/GlueFactory.java | 27 ++---- .../xxl/job/core/glue/loader/GlueLoader.java | 32 +++---- .../core/glue/loader/impl/DbGlueLoader.java | 60 ++++++------- .../com/xxl/job/core/util/ScriptUtil.java | 86 +++++++++++++++++++ .../resources/applicationcontext-xxl-job.xml | 7 +- 8 files changed, 211 insertions(+), 70 deletions(-) create mode 100644 xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java diff --git a/xxl-job-core/pom.xml b/xxl-job-core/pom.xml index cb18fc10..d3788249 100644 --- a/xxl-job-core/pom.xml +++ b/xxl-job-core/pom.xml @@ -79,6 +79,13 @@ 2.4.5 + + + org.apache.commons + commons-exec + 1.3 + + \ No newline at end of file diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java b/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java index f110e4f3..d4b4c752 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/biz/impl/ExecutorBizImpl.java @@ -11,9 +11,13 @@ import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.impl.GlueJobHandler; import com.xxl.job.core.log.XxlJobFileAppender; import com.xxl.job.core.thread.JobThread; +import com.xxl.job.core.util.ScriptUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import java.util.Date; /** @@ -103,7 +107,63 @@ public class ExecutorBizImpl implements ExecutorBiz { } jobThread = XxlJobExecutor.registJobThread(triggerParam.getJobId(), new GlueJobHandler(jobHandler, triggerParam.getGlueUpdatetime())); } + } else if (GlueTypeEnum.GLUE_SHELL==GlueTypeEnum.match(triggerParam.getGlueType())) { + + // make path + String scriptPath = XxlJobFileAppender.filePath + "gluesource/"; + String scriptFileName = triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".sh"; + + // valid file + File scriptFile = new File(scriptPath, scriptFileName); + if (!scriptFile.exists()) { + // valid glue source + if (triggerParam.getGlueSource()==null) { + return new ReturnT(ReturnT.FAIL_CODE, "glueSource is null."); + } + + // .../gluesource/ + File scriptPathDir = new File(scriptPath); + if (!scriptPathDir.exists()) { + scriptPathDir.mkdirs(); + } + + // .../gluesource/666-156465656.sh + scriptFile = new File(scriptPath, scriptFileName); + FileOutputStream fos = null; + try { + scriptFile.createNewFile(); + + fos = new FileOutputStream(scriptFile, true); + fos.write(triggerParam.getGlueSource().getBytes("utf-8")); + fos.flush(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return new ReturnT(ReturnT.FAIL_CODE, e.getMessage()); + } finally { + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + } + + } + + // log File + String logFileName = XxlJobFileAppender.makeLogFileName(new Date(triggerParam.getLogDateTim()), triggerParam.getLogId()); + + // run script + ScriptUtil.execToFile("python", scriptFile.getName(), (XxlJobFileAppender.filePath + logFileName) ); + + return ReturnT.FAIL; + } else if (GlueTypeEnum.GLUE_PYTHON==GlueTypeEnum.match(triggerParam.getGlueType())) { + String scriptFilePath = XxlJobFileAppender.filePath + "gluesource/" + triggerParam.getJobId() + "_" + triggerParam.getGlueUpdatetime() + ".py"; + + } else { + return new ReturnT(ReturnT.FAIL_CODE, "glueType[" + triggerParam.getGlueType() + "] is not valid."); } // push data to queue diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/executor/XxlJobExecutor.java b/xxl-job-core/src/main/java/com/xxl/job/core/executor/XxlJobExecutor.java index d2339e2c..ad1655c9 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/executor/XxlJobExecutor.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/executor/XxlJobExecutor.java @@ -78,8 +78,10 @@ public class XxlJobExecutor implements ApplicationContextAware, ApplicationListe } // ---------------------------------- init job handler ------------------------------------ + public static ApplicationContext applicationContext; @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + XxlJobExecutor.applicationContext = applicationContext; // init job handler action Map serviceBeanMap = applicationContext.getBeansWithAnnotation(JobHander.class); diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/glue/GlueFactory.java b/xxl-job-core/src/main/java/com/xxl/job/core/glue/GlueFactory.java index 0ac26e98..e16ccf0a 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/glue/GlueFactory.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/glue/GlueFactory.java @@ -1,14 +1,12 @@ package com.xxl.job.core.glue; +import com.xxl.job.core.executor.XxlJobExecutor; import com.xxl.job.core.handler.IJobHandler; import groovy.lang.GroovyClassLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; import org.springframework.core.annotation.AnnotationUtils; import javax.annotation.Resource; @@ -19,7 +17,7 @@ import java.lang.reflect.Modifier; * glue factory, product class/object by name * @author xuxueli 2016-1-2 20:02:27 */ -public class GlueFactory implements ApplicationContextAware { +public class GlueFactory { private static Logger logger = LoggerFactory.getLogger(GlueFactory.class); /** @@ -28,18 +26,11 @@ public class GlueFactory implements ApplicationContextAware { private GroovyClassLoader groovyClassLoader = new GroovyClassLoader(); // ----------------------------- spring support ----------------------------- - private static ApplicationContext applicationContext; - private static GlueFactory glueFactory; + private static GlueFactory glueFactory = new GlueFactory(); public static GlueFactory getInstance(){ return glueFactory; } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - GlueFactory.applicationContext = applicationContext; - GlueFactory.glueFactory = (GlueFactory) applicationContext.getBean("glueFactory"); - } - + /** * inject action of spring * @param instance @@ -61,21 +52,21 @@ public class GlueFactory implements ApplicationContextAware { try { Resource resource = AnnotationUtils.getAnnotation(field, Resource.class); if (resource.name()!=null && resource.name().length()>0){ - fieldBean = applicationContext.getBean(resource.name()); + fieldBean = XxlJobExecutor.applicationContext.getBean(resource.name()); } else { - fieldBean = applicationContext.getBean(field.getName()); + fieldBean = XxlJobExecutor.applicationContext.getBean(field.getName()); } } catch (Exception e) { } if (fieldBean==null ) { - fieldBean = applicationContext.getBean(field.getType()); + fieldBean = XxlJobExecutor.applicationContext.getBean(field.getType()); } } else if (AnnotationUtils.getAnnotation(field, Autowired.class) != null) { Qualifier qualifier = AnnotationUtils.getAnnotation(field, Qualifier.class); if (qualifier!=null && qualifier.value()!=null && qualifier.value().length()>0) { - fieldBean = applicationContext.getBean(qualifier.value()); + fieldBean = XxlJobExecutor.applicationContext.getBean(qualifier.value()); } else { - fieldBean = applicationContext.getBean(field.getType()); + fieldBean = XxlJobExecutor.applicationContext.getBean(field.getType()); } } diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/GlueLoader.java b/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/GlueLoader.java index 04a4d7e7..244648ec 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/GlueLoader.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/GlueLoader.java @@ -1,16 +1,16 @@ -package com.xxl.job.core.glue.loader; - -/** - * code source loader - * @author xuxueli 2016-1-2 20:01:39 - */ -public interface GlueLoader { - - /** - * load code source by name, ensure every load is the latest. - * @param jobId - * @return code source - */ - public String load(int jobId); - -} +//package com.xxl.job.core.glue.loader; +// +///** +// * code source loader +// * @author xuxueli 2016-1-2 20:01:39 +// */ +//public interface GlueLoader { +// +// /** +// * load code source by name, ensure every load is the latest. +// * @param jobId +// * @return code source +// */ +// public String load(int jobId); +// +//} diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/impl/DbGlueLoader.java b/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/impl/DbGlueLoader.java index 2d23516b..8587d79c 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/impl/DbGlueLoader.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/glue/loader/impl/DbGlueLoader.java @@ -1,30 +1,30 @@ -package com.xxl.job.core.glue.loader.impl; - -import com.xxl.job.core.glue.loader.GlueLoader; -import com.xxl.job.core.util.DBUtil; - -import javax.sql.DataSource; -import java.util.List; -import java.util.Map; - -/** - * Created by xuxueli on 16/9/30. - */ -public class DbGlueLoader implements GlueLoader { - - private DataSource dataSource; - public void setDataSource(DataSource dataSource) { - this.dataSource = dataSource; - } - - @Override - public String load(int jobId) { - String sql = "SELECT glue_source FROM XXL_JOB_QRTZ_TRIGGER_INFO WHERE id = ?"; - List> result = DBUtil.query(dataSource, sql, new Object[]{jobId}); - if (result!=null && result.size()==1 && result.get(0)!=null && result.get(0).get("glue_source")!=null ) { - return (String) result.get(0).get("glue_source"); - } - return null; - } - -} +//package com.xxl.job.core.glue.loader.impl; +// +//import com.xxl.job.core.glue.loader.GlueLoader; +//import com.xxl.job.core.util.DBUtil; +// +//import javax.sql.DataSource; +//import java.util.List; +//import java.util.Map; +// +///** +// * Created by xuxueli on 16/9/30. +// */ +//public class DbGlueLoader implements GlueLoader { +// +// private DataSource dataSource; +// public void setDataSource(DataSource dataSource) { +// this.dataSource = dataSource; +// } +// +// @Override +// public String load(int jobId) { +// String sql = "SELECT glue_source FROM XXL_JOB_QRTZ_TRIGGER_INFO WHERE id = ?"; +// List> result = DBUtil.query(dataSource, sql, new Object[]{jobId}); +// if (result!=null && result.size()==1 && result.get(0)!=null && result.get(0).get("glue_source")!=null ) { +// return (String) result.get(0).get("glue_source"); +// } +// return null; +// } +// +//} diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java b/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java new file mode 100644 index 00000000..a5518646 --- /dev/null +++ b/xxl-job-core/src/main/java/com/xxl/job/core/util/ScriptUtil.java @@ -0,0 +1,86 @@ +package com.xxl.job.core.util; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.PumpStreamHandler; + +import java.io.File; +import java.io.FileOutputStream; + +/** + * 1、内嵌编译器如"PythonInterpreter"无法引用扩展包，因此推荐使用java调用控制台进程方式"Runtime.getRuntime().exec()"来运行脚本(shell或python)； + * 2、因为通过java调用控制台进程方式实现，需要保证目标机器PATH路径正确配置对应编译器； + * 3、暂时脚本执行日志只能在脚本执行结束后一次性获取，无法保证实时性；因此为确保日志实时性，可改为将脚本打印的日志存储在指定的日志文件上； + * + * 知识点： + * 1、日志输出到日志文件：[>>logfile 2>&1]：将错误输出2以及标准输出1都一起以附加写方式导入logfile文件 + * 2、python 异常输出优先级高于标准输出，体现在Log文件中，因此推荐通过logging方式打日志保持和异常信息一致；否则用prinf日志顺序会错乱 + * + * Created by xuxueli on 17/2/25. + */ +public class ScriptUtil { + + private static String pyCmd = "python"; + private static String shllCmd = "bash"; + private static String pyFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/pytest.py"; + private static String shellFile = "/Users/xuxueli/workspaces/idea-git-workspace/github/xxl-incubator/xxl-util/src/main/resources/script/shelltest.sh"; + private static String pyLogFile = "/Users/xuxueli/Downloads/tmp/pylog.log"; + private static String shLogFile = "/Users/xuxueli/Downloads/tmp/shlog.log"; + + public static void main(String[] args) { + + String command = pyCmd; + String filename = pyFile; + String logFile = pyLogFile; + if (false) { + command = shllCmd; + filename = shellFile; + logFile = shLogFile; + } + + execToFile(command, filename, logFile); + + } + + public static File markScriptFile(){ + return null; + } + + /** + * 日志文件输出方式 + * + * 优点：支持将目标数据实时输出到指定日志文件中去 + * 缺点： + * 标准输出和错误输出优先级固定，可能和脚本中顺序不一致 + * Java无法实时获取 + * + * @param command + * @param scriptFile + * @param logFile + */ + public static void execToFile(String command, String scriptFile, String logFile){ + try { + // 标准输出：print （null if watchdog timeout） + // 错误输出：logging + 异常 （still exists if watchdog timeout） + // 标准输出 + FileOutputStream fileOutputStream = new FileOutputStream(logFile); + PumpStreamHandler streamHandler = new PumpStreamHandler(fileOutputStream, fileOutputStream, null); + + // command + CommandLine commandline = new CommandLine(command); + commandline.addArgument(scriptFile); + + // exec + DefaultExecutor exec = new DefaultExecutor(); + exec.setExitValues(null); + exec.setStreamHandler(streamHandler); + int exitValue = exec.execute(commandline); + } catch (Exception e) { + e.printStackTrace(); + } + /*Process process = Runtime.getRuntime().exec(cmdarray); + IOUtils.copy(process.getInputStream(), out); + IOUtils.copy(process.getErrorStream(), out);*/ + } + +} diff --git a/xxl-job-executor-example/src/main/resources/applicationcontext-xxl-job.xml b/xxl-job-executor-example/src/main/resources/applicationcontext-xxl-job.xml index 00c69389..e0d07252 100644 --- a/xxl-job-executor-example/src/main/resources/applicationcontext-xxl-job.xml +++ b/xxl-job-executor-example/src/main/resources/applicationcontext-xxl-job.xml @@ -38,14 +38,9 @@ - - - - - - +