diff --git a/doc/XXL-JOB官方文档.md b/doc/XXL-JOB官方文档.md index 99ca1050..38ff511e 100644 --- a/doc/XXL-JOB官方文档.md +++ b/doc/XXL-JOB官方文档.md @@ -929,6 +929,9 @@ xxl.job.i18n=zh_CN xxl.job.triggerpool.fast.max=300 xxl.job.triggerpool.slow.max=200 +### 调度触发后,批量更新任务批次数量【必填】 +xxl.job.schedule.batchsize=100 + ### 调度中心日志表数据保存天数 [必填]:过期日志自动清理;限制大于等于7时生效,否则, 如-1,关闭自动清理功能; xxl.job.logretentiondays=30 ``` @@ -1273,7 +1276,7 @@ public void demoJobHandler() throws Exception { { "input": "{输入信息,必填信息}", "prompt": "{模型prompt,可选信息}", - "model": "{模型实现,如qwen3:0.6b,可选信息}" + "model": "{模型实现,如qwen3.5:2b,可选信息}" } ``` - b、difyWorkflowJobHandler:DifyWorkflow 任务,支持自定义inputs、user、baseUrl、apiKey 等输入信息,示例参数如下; @@ -2785,10 +2788,15 @@ public void execute() { - 10、【优化】统一项目依赖管理结构,依赖版本统一到父级pom提升可维护性; ### 7.44 版本 v3.4.0 Release Notes[ING] -- 1、【优化】父POM依赖配置优化,移除容易配置;合并PR-3926; -- 2、【调整】Docker基础镜像调整为eclipse-temurin; -- 3、【TODO】调度触发性能优化:调度触发后任务分批批量更新,提升调度性能; -- 4、【TODO】调度中心OpenAPI完善,提供任务管理能力;封装Agent Skill并推送ClawHub; +- 1、【新增】调度性能提升:任务触发后分批批量更新,高频调度场景可百倍降低SQL操作合并执行,提升调度性能; +(任务触发后批量更新配置“xxl.job.schedule.batchsize”) +- 2、【调整】固定频率调度策略调整,修复小概率下触发时间偏差问题; +- 3、【调整】Docker基础镜像调整为eclipse-temurin; +- 4、【优化】父POM依赖配置优化,移除容易配置;合并PR-3926; +- 5、【升级】升级多项maven依赖至较新版本; +- 6、【优化】调度日志优化:支持执行器维度查看调度日志;新增调度日志索引,提升查询效率; +(数据库新增索引脚本:``` create index I_jobgroup on xxl_job_log (job_group); ``` ) +- 7、【TODO】调度中心OpenAPI完善,提供任务管理能力;封装Agent Skill并推送ClawHub; ### TODO LIST @@ -2827,11 +2835,10 @@ public void execute() { - 调度日志:全局配置:废弃; 新增“调度日志策略”:任务维度自定义,保留3天、7天、1个月、3个月、一年、永久; - 执行日志:新增“执行RollingLog开关”:任务维度自定义,支持:RollingLog、普通日志(slf4j输出)、关闭(不输出); - 21、AccessToken:废弃全局配置;支持在线管理,动态生成、动态启停; -- 22、任务执行后分批批量更新,提升调度性能; -- 23、任务管理OpenAPI; -- 24、调度中心启动参数线上配置:告警发送邮箱、Token,支持线上配置生效,修改不需重启机器; -- 25、执行器内嵌server切换tomcat,精简依赖; -- 26、日志策略新增: +- 22、任务管理OpenAPI; +- 23、调度中心启动参数线上配置:告警发送邮箱、Token,支持线上配置生效,修改不需重启机器; +- 24、执行器内嵌server切换tomcat,精简依赖; +- 25、日志策略新增: - 调度日志策略:任务级设置,最少保留1天。 - 执行日志策略:可选 RollingLog、slf4jLog; - 清理逻辑,性能重构。 diff --git a/doc/db/tables_xxl_job.sql b/doc/db/tables_xxl_job.sql index 36e1ceb9..ceb5f958 100644 --- a/doc/db/tables_xxl_job.sql +++ b/doc/db/tables_xxl_job.sql @@ -100,8 +100,8 @@ CREATE TABLE `xxl_job_log` PRIMARY KEY (`id`), KEY `I_trigger_time` (`trigger_time`), KEY `I_handle_code` (`handle_code`), - KEY `I_jobid_jobgroup` (`job_id`,`job_group`), - KEY `I_job_id` (`job_id`) + KEY `I_jobgroup` (`job_group`), + KEY `I_jobid` (`job_id`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; @@ -161,7 +161,7 @@ VALUES (1, 1, '示例任务01', now(), now(), 'XXL', '', 'CRON', '0 0 0 * * ? *' 'DO_NOTHING', 'FIRST', 'ollamaJobHandler', '{ "input": "慢SQL问题分析思路", "prompt": "你是一个研发工程师,擅长解决技术类问题。", - "model": "qwen3:0.6b" + "model": "qwen3.5:2b" }', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', now(), ''), (3, 2, 'Dify示例任务', now(), now(), 'XXL', '', 'NONE', '', diff --git a/pom.xml b/pom.xml index d3c2c3fc..06f0c48b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,29 +25,29 @@ 17 true - 3.14.1 + 3.15.0 3.4.0 3.12.0 3.2.8 - 0.9.0 + 0.10.0 2.0.17 - 6.0.1 + 6.0.3 3.0.0 - 4.0.1 - 7.0.2 + 4.0.5 + 7.0.6 4.0.1 - 9.5.0 + 9.6.0 - 4.2.9.Final + 4.2.12.Final - 5.0.3 + 5.0.4 2.3.2 @@ -57,9 +57,9 @@ 2.13.2 - 2.0.0-M1 + 2.0.0-M4 - 1.2.3 + 1.2.5 diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/controller/biz/JobLogController.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/controller/biz/JobLogController.java index 11b31029..19988321 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/controller/biz/JobLogController.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/controller/biz/JobLogController.java @@ -62,6 +62,7 @@ public class JobLogController { @RequestParam(value = "jobGroup", required = false, defaultValue = "0") Integer jobGroup, @RequestParam(value = "jobId", required = false, defaultValue = "0") Integer jobId) { + // 1、init JobGroupList // find all jobGroup List jobGroupListTotal = xxlJobGroupMapper.findAll(); @@ -70,44 +71,31 @@ public class JobLogController { if (CollectionTool.isEmpty(jobGroupList)) { throw new XxlJobException(I18nUtil.getString("jobgroup_empty")); } + List jobGroupIds = jobGroupList.stream().map(XxlJobGroup::getId).toList(); - // parse jobGroup + // 2、check jobId if (jobId > 0) { - // assign jobId (+ jobGroup) + // valid jobId XxlJobInfo jobInfo = xxlJobInfoMapper.loadById(jobId); if (jobInfo == null) { - // jobId not exist, inteceptor throw new RuntimeException(I18nUtil.getString("jobinfo_field_id") + I18nUtil.getString("system_unvalid")); } + // valid jobGroup jobGroup = jobInfo.getJobGroup(); - } else if (jobGroup > 0) { - // assign jobGroup - Integer finalJobGroup = jobGroup; - if (CollectionTool.isEmpty(jobGroupListTotal.stream().filter(item -> item.getId() == finalJobGroup).toList())) { - // jobGroup not exist, use first - jobGroup = jobGroupList.get(0).getId(); - } - jobId = 0; - } else { - // default first valid jobGroup - jobGroup = jobGroupList.get(0).getId(); - jobId = 0; } - /*// valid permission - JobGroupPermissionUtil.validJobGroupPermission(request, jobGroup);*/ + // 3、init jobGroup, default first 1 + if (!jobGroupIds.contains(jobGroup)) { + jobGroup = jobGroupList.get(0).getId(); + } - // find jobList + // 4、init jobInfoList List jobInfoList = xxlJobInfoMapper.getJobsByGroup(jobGroup); + List jobIds = jobInfoList.stream().map(XxlJobInfo::getId).toList(); - // parse jobId - if (CollectionTool.isEmpty(jobInfoList)) { + // 5、init JobId, default 0 + if (!jobIds.contains(jobId)) { jobId = 0; - } else { - if (!jobInfoList.stream().map(XxlJobInfo::getId).toList().contains(jobId)) { - // jobId not exist, use first - jobId = jobInfoList.get(0).getId(); - } } // write @@ -133,9 +121,9 @@ public class JobLogController { JobGroupPermissionUtil.validJobGroupPermission(request, jobGroup); // valid jobId - if (jobId < 1) { + /*if (jobId < 1) { return Response.ofFail(I18nUtil.getString("system_please_choose") + I18nUtil.getString("jobinfo_job")); - } + }*/ // parse param Date triggerTimeStart = null; diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/mapper/XxlJobInfoMapper.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/mapper/XxlJobInfoMapper.java index 64932cf0..b2919fca 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/mapper/XxlJobInfoMapper.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/mapper/XxlJobInfoMapper.java @@ -61,5 +61,12 @@ public interface XxlJobInfoMapper { */ public int scheduleUpdate(XxlJobInfo xxlJobInfo); + /** + * batch update job info + * + * @param jobInfoList + * @return + */ + public int scheduleBatchUpdate(@Param("list") List jobInfoList); } diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/config/XxlJobAdminBootstrap.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/config/XxlJobAdminBootstrap.java index 52475262..05d13a12 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/config/XxlJobAdminBootstrap.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/config/XxlJobAdminBootstrap.java @@ -181,6 +181,9 @@ public class XxlJobAdminBootstrap implements InitializingBean, DisposableBean { @Value("${xxl.job.triggerpool.slow.max}") private int triggerPoolSlowMax; + @Value("${xxl.job.schedule.batchsize}") + private int scheduleBatchSize; + @Value("${xxl.job.logretentiondays}") private int logretentiondays; @@ -244,6 +247,13 @@ public class XxlJobAdminBootstrap implements InitializingBean, DisposableBean { return triggerPoolSlowMax; } + public int getScheduleBatchSize() { + if (!(scheduleBatchSize >=50 && scheduleBatchSize <= 500)) { + return 100; + } + return scheduleBatchSize; + } + public int getLogretentiondays() { if (logretentiondays < 3) { return -1; // Limit greater than or equal to 3, otherwise close diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/thread/JobScheduleHelper.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/thread/JobScheduleHelper.java index 9d66d54e..e8beadd2 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/thread/JobScheduleHelper.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/thread/JobScheduleHelper.java @@ -59,7 +59,7 @@ public class JobScheduleHelper { } logger.info(">>>>>>>>> init xxl-job admin scheduler success."); - // pre-read count: treadpool-size * trigger-qps (each trigger cost 100ms, qps = 1000/100 = 100) + // pre-read count: treadpool-size * 10 (trigger-qps: 1000ms / 100ms each trigger cost) int preReadCount = (XxlJobAdminBootstrap.getInstance().getTriggerPoolFastMax() + XxlJobAdminBootstrap.getInstance().getTriggerPoolSlowMax()) * 10; // do schedule @@ -138,8 +138,14 @@ public class JobScheduleHelper { } // 3、update trigger info - for (XxlJobInfo jobInfo: scheduleList) { + /*for (XxlJobInfo jobInfo: scheduleList) { XxlJobAdminBootstrap.getInstance().getXxlJobInfoMapper().scheduleUpdate(jobInfo); + }*/ + int batchSize = XxlJobAdminBootstrap.getInstance().getScheduleBatchSize(); + List> scheduleListBatches = CollectionTool.split(scheduleList, batchSize); + for (List scheduleListBatch : scheduleListBatches) { + int totalAffected = XxlJobAdminBootstrap.getInstance().getXxlJobInfoMapper().scheduleBatchUpdate(scheduleListBatch); + logger.debug(">>>>>>>>>>> xxl-job, JobScheduleHelper scheduleBatchUpdate records:" + totalAffected); } } else { diff --git a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/type/strategy/FixRateScheduleType.java b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/type/strategy/FixRateScheduleType.java index 290a4200..298ab7d1 100644 --- a/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/type/strategy/FixRateScheduleType.java +++ b/xxl-job-admin/src/main/java/com/xxl/job/admin/scheduler/type/strategy/FixRateScheduleType.java @@ -2,6 +2,7 @@ package com.xxl.job.admin.scheduler.type.strategy; import com.xxl.job.admin.model.XxlJobInfo; import com.xxl.job.admin.scheduler.type.ScheduleType; +import com.xxl.tool.core.DateTool; import java.util.Date; @@ -9,8 +10,17 @@ public class FixRateScheduleType extends ScheduleType { @Override public Date generateNextTriggerTime(XxlJobInfo jobInfo, Date fromTime) throws Exception { + // generate next trigger time, fix rate delay - return new Date(fromTime.getTime() + Long.parseLong(jobInfo.getScheduleConf()) * 1000L); + Date nextTriggerTime = new Date(fromTime.getTime() + Long.parseLong(jobInfo.getScheduleConf()) * 1000L); + + // assign second: + if (nextTriggerTime.getTime() % 1000 != 0) { + nextTriggerTime = DateTool.addSeconds(DateTool.setMilliseconds(nextTriggerTime, 0), 1); + } + + return nextTriggerTime; + } } diff --git a/xxl-job-admin/src/main/resources/application.properties b/xxl-job-admin/src/main/resources/application.properties index abac8fca..4ea78699 100644 --- a/xxl-job-admin/src/main/resources/application.properties +++ b/xxl-job-admin/src/main/resources/application.properties @@ -64,6 +64,9 @@ xxl.job.i18n=zh_CN xxl.job.triggerpool.fast.max=300 xxl.job.triggerpool.slow.max=200 +## xxl-job, schedule batch size +xxl.job.schedule.batchsize=100 + ### xxl-job, log retention days xxl.job.logretentiondays=30 diff --git a/xxl-job-admin/src/main/resources/mapper/XxlJobInfoMapper.xml b/xxl-job-admin/src/main/resources/mapper/XxlJobInfoMapper.xml index 7a4be97f..51b10505 100644 --- a/xxl-job-admin/src/main/resources/mapper/XxlJobInfoMapper.xml +++ b/xxl-job-admin/src/main/resources/mapper/XxlJobInfoMapper.xml @@ -240,34 +240,39 @@ AND trigger_status = 1 - + + diff --git a/xxl-job-admin/src/main/resources/mapper/XxlJobLogMapper.xml b/xxl-job-admin/src/main/resources/mapper/XxlJobLogMapper.xml index 58dda467..5948eb53 100644 --- a/xxl-job-admin/src/main/resources/mapper/XxlJobLogMapper.xml +++ b/xxl-job-admin/src/main/resources/mapper/XxlJobLogMapper.xml @@ -48,7 +48,7 @@ SELECT FROM xxl_job_log AS t - + AND t.job_group = #{jobGroup} @@ -82,7 +82,7 @@ SELECT count(1) FROM xxl_job_log AS t - + AND t.job_group = #{jobGroup} diff --git a/xxl-job-admin/src/main/resources/templates/biz/log.list.ftl b/xxl-job-admin/src/main/resources/templates/biz/log.list.ftl index 43458f77..9a6ec166 100644 --- a/xxl-job-admin/src/main/resources/templates/biz/log.list.ftl +++ b/xxl-job-admin/src/main/resources/templates/biz/log.list.ftl @@ -37,12 +37,11 @@
${I18n.jobinfo_job}
diff --git a/xxl-job-admin/src/test/java/com/xxl/job/admin/mapper/XxlJobInfoMapperTest.java b/xxl-job-admin/src/test/java/com/xxl/job/admin/mapper/XxlJobInfoMapperTest.java index 617ff204..28b5bebb 100644 --- a/xxl-job-admin/src/test/java/com/xxl/job/admin/mapper/XxlJobInfoMapperTest.java +++ b/xxl-job-admin/src/test/java/com/xxl/job/admin/mapper/XxlJobInfoMapperTest.java @@ -1,8 +1,12 @@ package com.xxl.job.admin.mapper; +import com.xxl.job.admin.constant.TriggerStatus; import com.xxl.job.admin.model.XxlJobInfo; +import com.xxl.job.admin.scheduler.config.XxlJobAdminBootstrap; import com.xxl.job.admin.scheduler.misfire.MisfireStrategyEnum; import com.xxl.job.admin.scheduler.type.ScheduleTypeEnum; +import com.xxl.tool.core.CollectionTool; +import com.xxl.tool.core.DateTool; import jakarta.annotation.Resource; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -83,4 +87,28 @@ public class XxlJobInfoMapperTest { } + @Test + public void scheduleBatchUpdateTest(){ + + List list1 = xxlJobInfoMapper.pageList(0, 20, 0, -1, null, null, null); + int batchSize = 5; + + // update + List list2 = list1.stream().filter(item -> (item.getId()>=4 && item.getId()<=14)).toList(); + list2.forEach(item -> { + item.setTriggerLastTime(DateTool.addHours(new Date(), -1).getTime()); + item.setTriggerNextTime(DateTool.addHours(new Date(), 1).getTime()); + if (item.getId() == 5) { + item.setTriggerStatus(TriggerStatus.STOPPED.getValue()); + } + }); + + // batch update + List> scheduleListBatches = CollectionTool.split(list2, batchSize); + for (List scheduleListBatch : scheduleListBatches) { + int totalAffected = XxlJobAdminBootstrap.getInstance().getXxlJobInfoMapper().scheduleBatchUpdate(scheduleListBatch); + logger.info("scheduleBatchUpdate records:" + totalAffected); + } + } + } diff --git a/xxl-job-core/src/main/java/com/xxl/job/core/openapi/AdminBiz.java b/xxl-job-core/src/main/java/com/xxl/job/core/openapi/AdminBiz.java index 11cd42e5..d18774f4 100644 --- a/xxl-job-core/src/main/java/com/xxl/job/core/openapi/AdminBiz.java +++ b/xxl-job-core/src/main/java/com/xxl/job/core/openapi/AdminBiz.java @@ -42,7 +42,20 @@ public interface AdminBiz { public Response registryRemove(RegistryRequest registryRequest); - // ---------------------- biz (custome) ---------------------- - // group、job ... manage + // ---------------------- job operate ---------------------- + + // jobAdd + + // jobUpdate + + // jobDelete + + // jobQuery + + // jobStart + + // jobStop + + // jobTrigger } diff --git a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/controller/IndexController.java b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/controller/IndexController.java index 903c3efc..0b5f2358 100644 --- a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/controller/IndexController.java +++ b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/controller/IndexController.java @@ -50,7 +50,7 @@ public class IndexController { @Resource private OllamaChatModel ollamaChatModel; private String prompt = "你好,你是一个研发工程师,擅长解决技术类问题。"; - private String modle = "qwen3:0.6b"; + private String modle = "qwen3.5:2b"; /** * ChatClient 简单调用 diff --git a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/jobhandler/AIXxlJob.java b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/jobhandler/AIXxlJob.java index d4a67841..1485b30b 100644 --- a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/jobhandler/AIXxlJob.java +++ b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/main/java/com/xxl/job/executor/jobhandler/AIXxlJob.java @@ -70,7 +70,7 @@ public class AIXxlJob { return; } if (ollamaParam.getModel()==null || ollamaParam.getModel().isBlank()) { - ollamaParam.setModel("qwen3:0.6b"); + ollamaParam.setModel("qwen3.5:2b"); } } catch (Exception e) { XxlJobHelper.log(new RuntimeException("OllamaParam parse error", e)); diff --git a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/test/java/com/xxl/job/executor/test/ollama/OllamaTest.java b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/test/java/com/xxl/job/executor/test/ollama/OllamaTest.java index 140c5567..3caf5fde 100644 --- a/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/test/java/com/xxl/job/executor/test/ollama/OllamaTest.java +++ b/xxl-job-executor-samples/xxl-job-executor-sample-springboot-ai/src/test/java/com/xxl/job/executor/test/ollama/OllamaTest.java @@ -32,7 +32,7 @@ public class OllamaTest { @Test public void chatTest() { - String model = "qwen3:0.6b"; + String model = "qwen3.5:2b"; String prompt = "背景说明:你是一个研发工程师,擅长解决技术类问题。"; String input = "请写一个java程序,实现一个方法,输入一个字符串,返回字符串的长度。"; @@ -59,7 +59,7 @@ public class OllamaTest { @Test public void chatStreamTest() throws InterruptedException { - String model = "qwen3:0.6b"; + String model = "qwen3.5:2b"; String prompt = "背景说明:你是一个研发工程师,擅长解决技术类问题。"; String input = "请写一个java程序,实现一个方法,输入一个字符串,返回字符串的长度。";