From 9c03285d94d125209cc64be9a16c7d83d1f12c88 Mon Sep 17 00:00:00 2001 From: "xueli.xue" Date: Fri, 10 Mar 2017 22:41:37 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E4=BB=BB=E5=8A=A1=E6=96=B0=E5=A2=9E"?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=99=A8=E8=B7=AF=E7=94=B1=E7=AD=96=E7=95=A5?= =?UTF-8?q?"=E5=B1=9E=E6=80=A7=EF=BC=9B=202=E3=80=81=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E7=AD=96=E7=95=A5=E9=80=BB=E8=BE=91=EF=BC=9A=E7=AC=AC=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E3=80=81=E6=9C=80=E5=90=8E=E4=B8=80=E4=B8=AA=E3=80=81?= =?UTF-8?q?=E8=BD=AE=E8=AE=AD=E3=80=81=E9=9A=8F=E6=9C=BA=E3=80=81=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E6=80=A7HASH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/tables_xxl_job.sql | 4 +- .../xxl/job/admin/core/model/XxlJobInfo.java | 15 +++- .../core/route/ExecutorRouteStrategyEnum.java | 41 ++++++++++ .../job/admin/core/route/ExecutorRouter.java | 51 ++++++++++++ .../strategy/ExecutorRouteConsistentHash.java | 80 +++++++++++++++++++ .../route/strategy/ExecutorRouteFirst.java | 17 ++++ .../route/strategy/ExecutorRouteLast.java | 17 ++++ .../route/strategy/ExecutorRouteRandom.java | 20 +++++ .../route/strategy/ExecutorRouteRound.java | 26 ++++++ .../mybatis-mapper/XxlJobInfoMapper.xml | 5 ++ 10 files changed, 271 insertions(+), 5 deletions(-) create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouteStrategyEnum.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouter.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteConsistentHash.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteFirst.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteLast.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRandom.java create mode 100644 xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRound.java diff --git a/db/tables_xxl_job.sql b/db/tables_xxl_job.sql index 7b34f92a..b4d23181 100644 --- a/db/tables_xxl_job.sql +++ b/db/tables_xxl_job.sql @@ -145,7 +145,7 @@ CREATE TABLE XXL_JOB_QRTZ_LOCKS -CREATE TABLE XXL_JOB_QRTZ_TRIGGER_INFO ( +CREATE TABLE `XXL_JOB_QRTZ_TRIGGER_INFO` ( `id` int(11) NOT NULL AUTO_INCREMENT, `job_group` int(11) NOT NULL COMMENT '任务组(执行器ID)', `job_name` varchar(255) NOT NULL COMMENT '任务名', @@ -155,6 +155,7 @@ CREATE TABLE XXL_JOB_QRTZ_TRIGGER_INFO ( `update_time` datetime DEFAULT NULL, `author` varchar(64) DEFAULT NULL COMMENT '作者', `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', + `executor_route_strategy` varchar(20) DEFAULT NULL COMMENT '执行器路由策略', `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', `executor_param` varchar(255) DEFAULT NULL COMMENT '执行器任务参数', `glue_switch` int(11) DEFAULT '0' COMMENT 'GLUE模式开关：0-否，1-是', @@ -164,7 +165,6 @@ CREATE TABLE XXL_JOB_QRTZ_TRIGGER_INFO ( PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - CREATE TABLE `XXL_JOB_QRTZ_TRIGGER_LOG` ( `id` int(11) NOT NULL AUTO_INCREMENT, `job_group` int(11) NOT NULL COMMENT '任务组', diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobInfo.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobInfo.java index ce8fc2a2..cca886cd 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobInfo.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/model/XxlJobInfo.java @@ -21,8 +21,9 @@ public class XxlJobInfo { private String author; // 负责人 private String alarmEmail; // 报警邮件 - private String executorHandler; // 执行器，任务Handler名称 - private String executorParam; // 执行器，任务参数 + private String executorRouteStrategy; // 执行器路由策略 + private String executorHandler; // 执行器，任务Handler名称 + private String executorParam; // 执行器，任务参数 private int glueSwitch; // GLUE模式开关：0-否，1-是 private String glueSource; // GLUE源代码 @@ -105,7 +106,15 @@ public class XxlJobInfo { this.alarmEmail = alarmEmail; } - public String getExecutorHandler() { + public String getExecutorRouteStrategy() { + return executorRouteStrategy; + } + + public void setExecutorRouteStrategy(String executorRouteStrategy) { + this.executorRouteStrategy = executorRouteStrategy; + } + + public String getExecutorHandler() { return executorHandler; } diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouteStrategyEnum.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouteStrategyEnum.java new file mode 100644 index 00000000..770d78d2 --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouteStrategyEnum.java @@ -0,0 +1,41 @@ +package com.xxl.job.admin.core.route; + +import com.xxl.job.admin.core.route.strategy.*; + +/** + * Created by xuxueli on 17/3/10. + */ +public enum ExecutorRouteStrategyEnum { + + FIRST("第一个", new ExecutorRouteFirst()), + LAST("最后一个", new ExecutorRouteLast()), + ROUND("轮询", new ExecutorRouteRound()), + RANDOM("随机", new ExecutorRouteRandom()), + CONSISTENT_HASH("一致性HASH", new ExecutorRouteConsistentHash()), + FAILOVER("故障转移", null); + + ExecutorRouteStrategyEnum(String title, ExecutorRouter router) { + this.title = title; + this.router = router; + } + + private String title; + private ExecutorRouter router; + + public String getTitle() { + return title; + } + public ExecutorRouter getRouter() { + return router; + } + + public static ExecutorRouteStrategyEnum match(String name, ExecutorRouteStrategyEnum defaultItem){ + for (ExecutorRouteStrategyEnum item: ExecutorRouteStrategyEnum.values()) { + if (item.name().equals(name)) { + return item; + } + } + return defaultItem; + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouter.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouter.java new file mode 100644 index 00000000..ea6cbf85 --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/ExecutorRouter.java @@ -0,0 +1,51 @@ +package com.xxl.job.admin.core.route; + +import org.apache.commons.collections.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Created by xuxueli on 17/3/10. + */ +public abstract class ExecutorRouter { + + public abstract String route(int jobId, ArrayList addressList); + + public static String route(int jobId, ArrayList addressList, String executorRouteStrategy){ + if (CollectionUtils.isEmpty(addressList)) { + return null; + } + ExecutorRouteStrategyEnum strategy = ExecutorRouteStrategyEnum.match(executorRouteStrategy, ExecutorRouteStrategyEnum.FIRST); + String routeAddress = strategy.getRouter().route(jobId, addressList); + return routeAddress; + } + + public static void main(String[] args) { + + int c1 = 0; + int c2 = 0; + int c3 = 0; + + long start = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String ret = ExecutorRouter.route(i, new ArrayList(Arrays.asList("127.0.0.1:0000", "127.0.0.1:2222", "127.0.0.1:3333")), ExecutorRouteStrategyEnum.CONSISTENT_HASH.name()); + if (ret.equals("127.0.0.1:0000")) { + c1++; + } else if (ret.equals("127.0.0.1:2222")) { + c2++; + } else if (ret.equals("127.0.0.1:3333")) { + c3++; + } + } + long end = System.currentTimeMillis(); + System.out.println(end - start); // 1000*100=740、1000*1=162、 + + + System.out.println(c1); + System.out.println(c2); + System.out.println(c3); + + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteConsistentHash.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteConsistentHash.java new file mode 100644 index 00000000..4a42d759 --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteConsistentHash.java @@ -0,0 +1,80 @@ +package com.xxl.job.admin.core.route.strategy; + +import com.xxl.job.admin.core.route.ExecutorRouter; + +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * 分组下JOB均匀散列在不同机器上；且每个JOB固定调度其中一台机器； + * a、virtual node：解决不均衡问题 + * b、hash method replace hashCode：String的hashCode可能重复，需要进一步扩大hashCode的取值范围 + * Created by xuxueli on 17/3/10. + */ +public class ExecutorRouteConsistentHash extends ExecutorRouter { + + private static int VIRTUAL_NODE_NUM = 5; + + @Override + public String route(int jobId, ArrayList addressList) { + + // ------A1------A2-------A3------ + // -----------J1------------------ + TreeMap addressRing = new TreeMap(); + for (String address: addressList) { + for (int i = 0; i < VIRTUAL_NODE_NUM; i++) { + long addressHash = hash("SHARD-" + address + "-NODE-" + i); + addressRing.put(addressHash, address); + } + } + + long jobHash = hash(String.valueOf(jobId)); + SortedMap lastRing = addressRing.tailMap(jobHash); + if (!lastRing.isEmpty()) { + return lastRing.get(lastRing.firstKey()); + } + return addressRing.firstEntry().getValue(); + } + + + + /** + * get hash code on 2^32 ring (md5散列的方式计算hash值) + * @param key + * @return + */ + private static long hash(String key) { + + // md5 byte + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5 not supported", e); + } + md5.reset(); + byte[] keyBytes = null; + try { + keyBytes = key.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unknown string :" + key, e); + } + + md5.update(keyBytes); + byte[] digest = md5.digest(); + + // hash code, Truncate to 32-bits + long hashCode = ((long) (digest[3] & 0xFF) << 24) + | ((long) (digest[2] & 0xFF) << 16) + | ((long) (digest[1] & 0xFF) << 8) + | (digest[0] & 0xFF); + + long truncateHashCode = hashCode & 0xffffffffL; + return truncateHashCode; + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteFirst.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteFirst.java new file mode 100644 index 00000000..bbcd5dad --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteFirst.java @@ -0,0 +1,17 @@ +package com.xxl.job.admin.core.route.strategy; + +import com.xxl.job.admin.core.route.ExecutorRouter; + +import java.util.ArrayList; + +/** + * Created by xuxueli on 17/3/10. + */ +public class ExecutorRouteFirst extends ExecutorRouter { + + @Override + public String route(int jobId, ArrayList addressList) { + return addressList.get(0); + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteLast.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteLast.java new file mode 100644 index 00000000..1600e61a --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteLast.java @@ -0,0 +1,17 @@ +package com.xxl.job.admin.core.route.strategy; + +import com.xxl.job.admin.core.route.ExecutorRouter; + +import java.util.ArrayList; + +/** + * Created by xuxueli on 17/3/10. + */ +public class ExecutorRouteLast extends ExecutorRouter { + + @Override + public String route(int jobId, ArrayList addressList) { + return addressList.get(addressList.size()-1); + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRandom.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRandom.java new file mode 100644 index 00000000..70e0e28a --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRandom.java @@ -0,0 +1,20 @@ +package com.xxl.job.admin.core.route.strategy; + +import com.xxl.job.admin.core.route.ExecutorRouter; + +import java.util.ArrayList; +import java.util.Random; + +/** + * Created by xuxueli on 17/3/10. + */ +public class ExecutorRouteRandom extends ExecutorRouter { + + private static Random localRandom = new Random(); + + @Override + public String route(int jobId, ArrayList addressList) { + return addressList.get(localRandom.nextInt(addressList.size())); + } + +} diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRound.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRound.java new file mode 100644 index 00000000..4997c778 --- /dev/null +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/core/route/strategy/ExecutorRouteRound.java @@ -0,0 +1,26 @@ +package com.xxl.job.admin.core.route.strategy; + +import com.xxl.job.admin.core.route.ExecutorRouter; + +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by xuxueli on 17/3/10. + */ +public class ExecutorRouteRound extends ExecutorRouter { + + private static ConcurrentHashMap routeCountEachJob = new ConcurrentHashMap(); + private static int count(int jobId) { + Integer count = routeCountEachJob.get(jobId); + count = (count==null)?0:++count; + routeCountEachJob.put(jobId, count); + return count; + } + + @Override + public String route(int jobId, ArrayList addressList) { + return addressList.get(count(jobId)%addressList.size()); + } + +} diff --git a/xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobInfoMapper.xml b/xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobInfoMapper.xml index ce65b40b..e8533338 100644 --- a/xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobInfoMapper.xml +++ b/xxl-job-admin/src/main/resources/mybatis-mapper/XxlJobInfoMapper.xml @@ -17,6 +17,7 @@ + @@ -37,6 +38,7 @@ t.update_time, t.author, t.alarm_email, + t.executor_route_strategy, t.executor_handler, t.executor_param, t.glue_switch, @@ -83,6 +85,7 @@ update_time, author, alarm_email, + executor_route_strategy, executor_handler, executor_param, glue_switch, @@ -98,6 +101,7 @@ NOW(), #{author}, #{alarmEmail}, + #{executorRouteStrategy}, #{executorHandler}, #{executorParam}, #{glueSwitch}, @@ -125,6 +129,7 @@ update_time = NOW(), author = #{author}, alarm_email = #{alarmEmail}, + executor_route_strategy = #{executorRouteStrategy}, executor_handler = #{executorHandler}, executor_param = #{executorParam}, glue_switch = #{glueSwitch},