refactor:调度组件守护线程代码重构，提升稳定性以及可维护性；

doc/XXL-JOB官方文档.md

@@ -2846,8 +2846,9 @@ alter table xxl_job_log
 - 3、【优化】任务参数长度调整，最长支持2048字符；
 - 4、【升级】调度中心UI交互优化，任务及日志管理支持下拉框模糊搜索，提升交互体验；
 - 5、【修复】XxlJobFileAppender自定义地址callbackLogPath设置无效问题修复；合并ISSUS-3963；
-- 6、【TODO】调度中心OpenAPI完善，提供任务管理能力；封装Agent Skill并推送ClawHub；
+- 6、【优化】调度组件守护线程代码重构，提升稳定性以及可维护性；
-- 7、【TODO】AccessToken升级：执行器维度隔离，支持线上化配置；升级双端OpenApi，适配AccessToken升级；
+- 7、【TODO】调度中心OpenAPI完善，提供任务管理能力；封装Agent Skill并推送ClawHub；
+- 8、【TODO】AccessToken升级：执行器维度隔离，支持线上化配置；升级双端OpenApi，适配AccessToken升级；
 
 
 ### TODO LIST

xxl-job-admin/src/main/java/com/xxl/job/admin/business/scheduler/thread/JobCompleteHelper.java

@@ -3,8 +3,9 @@ package com.xxl.job.admin.business.scheduler.thread;
 import com.xxl.job.admin.business.model.XxlJobLog;
 import com.xxl.job.admin.business.scheduler.config.XxlJobAdminBootstrap;
 import com.xxl.job.admin.framework.util.I18nUtil;
-import com.xxl.job.core.openapi.model.CallbackRequest;
 import com.xxl.job.core.context.XxlJobContext;
+import com.xxl.job.core.openapi.model.CallbackRequest;
+import com.xxl.tool.concurrent.CyclicThread;
 import com.xxl.tool.core.DateTool;
 import com.xxl.tool.response.Response;
 import org.slf4j.Logger;
@@ -25,15 +26,14 @@ public class JobCompleteHelper {
 	// ---------------------- monitor ----------------------
 
 	private ThreadPoolExecutor callbackThreadPool = null;
-	private Thread monitorThread;
+	private CyclicThread jobMonitorThread;
-	private volatile boolean toStop = false;
 
 	/**
 	 * start
 	 */
 	public void start(){
 
-		// for callback
+		// 1、callbackThreadPool
 		callbackThreadPool = new ThreadPoolExecutor(
 				2,
 				20,
@@ -55,24 +55,10 @@ public class JobCompleteHelper {
 				});
 
 
-		// for monitor
+		// 2、jobMonitorThread
-		monitorThread = new Thread(new Runnable() {
+		jobMonitorThread = new CyclicThread("JobCompleteHelper#jobMonitorThread", true, new Runnable() {
-
 			@Override
 			public void run() {
-
-				// wait for JobTriggerPoolHelper-init
-				try {
-					TimeUnit.MILLISECONDS.sleep(50);
-				} catch (Throwable e) {
-					if (!toStop) {
-						logger.error(e.getMessage(), e);
-					}
-				}
-
-				// monitor
-				while (!toStop) {
-					try {
 				// 任务结果丢失处理：调度记录停留在 "运行中" 状态超过10min，且对应执行器心跳注册失败不在线，则将本地调度主动标记失败；
 				Date losedTime = DateTool.addMinutes(new Date(), -10);
 				List<Long> losedJobIds  = XxlJobAdminBootstrap.getInstance().getXxlJobLogMapper().findLostJobIds(losedTime);
@@ -91,52 +77,33 @@ public class JobCompleteHelper {
 					}
 
 				}
-					} catch (Throwable e) {
-						if (!toStop) {
-							logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e);
-						}
-					}
-
-                    try {
-                        TimeUnit.SECONDS.sleep(60);
-                    } catch (Throwable e) {
-                        if (!toStop) {
-                            logger.error(e.getMessage(), e);
-                        }
-                    }
-
 			}
+		}, 60 * 1000L, true);
+		jobMonitorThread.start();
 
-				logger.info(">>>>>>>>>>> xxl-job, JobLosedMonitorHelper stop");
-
-			}
-		});
-		monitorThread.setDaemon(true);
-		monitorThread.setName("xxl-job, admin JobLosedMonitorHelper");
-		monitorThread.start();
 	}
 
 	/**
 	 * stop
 	 */
 	public void stop(){
-		toStop = true;
 
-		// stop registryOrRemoveThreadPool
+		// 1、callbackThreadPool
 		callbackThreadPool.shutdownNow();
 
-		// stop monitorThread (interrupt and wait)
+		// 2、jobMonitorThread
-		monitorThread.interrupt();
+		jobMonitorThread.stop();
-		try {
-			monitorThread.join();
-		} catch (Throwable e) {
-			logger.error(e.getMessage(), e);
-		}
 	}
 
 
 	// ---------------------- helper ----------------------
 
+	/**
+	 * callback
+	 *
+	 * @param callbackParamList callback param
+	 * @return callback result
+	 */
 	public Response<String> callback(List<CallbackRequest> callbackParamList) {
 
 		callbackThreadPool.execute(new Runnable() {

xxl-job-admin/src/main/java/com/xxl/job/admin/business/scheduler/thread/JobFailAlarmMonitorHelper.java

@@ -5,11 +5,11 @@ import com.xxl.job.admin.business.model.XxlJobLog;
 import com.xxl.job.admin.business.scheduler.config.XxlJobAdminBootstrap;
 import com.xxl.job.admin.business.scheduler.trigger.TriggerTypeEnum;
 import com.xxl.job.admin.framework.util.I18nUtil;
+import com.xxl.tool.concurrent.CyclicThread;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 /**
  * job fail-monitor helper
@@ -22,22 +22,19 @@ public class JobFailAlarmMonitorHelper {
 
 	// ---------------------- monitor ----------------------
 
-	private Thread monitorThread;
+	/**
-	private volatile boolean toStop = false;
+	 * monitor thread
+	 */
+	private CyclicThread monitorThread;
 
 	/**
 	 * start
 	 */
 	public void start(){
-		monitorThread = new Thread(new Runnable() {
 
+		monitorThread = new CyclicThread("JobFailAlarmMonitorHelper#monitorThread", true, new Runnable() {
 			@Override
 			public void run() {
-
-				// monitor
-				while (!toStop) {
-					try {
-
 				List<Long> failLogIds = XxlJobAdminBootstrap.getInstance().getXxlJobLogMapper().findFailJobLogIds(1000);
 				if (failLogIds!=null && !failLogIds.isEmpty()) {
 					for (long failLogId: failLogIds) {
@@ -70,29 +67,8 @@ public class JobFailAlarmMonitorHelper {
 						XxlJobAdminBootstrap.getInstance().getXxlJobLogMapper().updateAlarmStatus(failLogId, -1, newAlarmStatus);
 					}
 				}
-
-					} catch (Throwable e) {
-						if (!toStop) {
-							logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e.getMessage(), e);
-						}
-					}
-
-                    try {
-                        TimeUnit.SECONDS.sleep(10);
-                    } catch (Throwable e) {
-                        if (!toStop) {
-                            logger.error(e.getMessage(), e);
 			}
-                    }
+		}, 10 * 1000L, true);
-
-                }
-
-				logger.info(">>>>>>>>>>> xxl-job, job fail monitor thread stop");
-
-			}
-		});
-		monitorThread.setDaemon(true);
-		monitorThread.setName("xxl-job, admin JobFailMonitorHelper");
 		monitorThread.start();
 	}
 
@@ -100,14 +76,7 @@ public class JobFailAlarmMonitorHelper {
 	 * stop
 	 */
 	public void stop(){
-		toStop = true;
+		monitorThread.stop();
-		// interrupt and wait
-		monitorThread.interrupt();
-		try {
-			monitorThread.join();
-		} catch (Throwable e) {
-			logger.error(e.getMessage(), e);
-		}
 	}
 
 }

xxl-job-admin/src/main/java/com/xxl/job/admin/business/scheduler/thread/JobLogReportHelper.java

@@ -1,7 +1,8 @@
 package com.xxl.job.admin.business.scheduler.thread;
 
-import com.xxl.job.admin.business.scheduler.config.XxlJobAdminBootstrap;
 import com.xxl.job.admin.business.model.XxlJobLogReport;
+import com.xxl.job.admin.business.scheduler.config.XxlJobAdminBootstrap;
+import com.xxl.tool.concurrent.CyclicThread;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -9,7 +10,7 @@ import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * job log report helper
@@ -19,28 +20,25 @@ import java.util.concurrent.TimeUnit;
 public class JobLogReportHelper {
     private static final Logger logger = LoggerFactory.getLogger(JobLogReportHelper.class);
 
-
+    private CyclicThread logReportThread;
-    private Thread logReportThread;
+    private AtomicLong lastCleanLogTime;
-    private volatile boolean toStop = false;
 
     /**
      * start
      */
     public void start(){
-        logReportThread = new Thread(new Runnable() {
 
+        /**
+         * last clean log time  （ Thread-safe concurrent reading and writing ）
+          */
+        lastCleanLogTime = new AtomicLong(0);
+
+        // log report thread
+        logReportThread = new CyclicThread("JobLogReportHelper#logReportThread", true, new Runnable() {
             @Override
             public void run() {
 
-                // last clean log time
-                long lastCleanLogTime = 0;
-
-
-                while (!toStop) {
-
                 // 1、log-report refresh: refresh log report in 3 days
-                    try {
-
                 for (int i = 0; i < 3; i++) {
 
                     // today
@@ -88,16 +86,9 @@ public class JobLogReportHelper {
                         }*/
                 }
 
-                    } catch (Throwable e) {
-                        if (!toStop) {
-                            logger.error(">>>>>>>>>>> xxl-job, JobLogReportHelper(log-report refresh) error:{}", e.getMessage(), e);
-                        }
-                    }
-
                 // 2、log-clean: switch open & once each day
-                    try {
                 if (XxlJobAdminBootstrap.getInstance().getLogretentiondays()>0
-                                && System.currentTimeMillis() - lastCleanLogTime > 24*60*60*1000) {
+                        && System.currentTimeMillis() - lastCleanLogTime.longValue() > 24*60*60*1000) {
 
                     // expire-time
                     Calendar expiredDay = Calendar.getInstance();
@@ -118,45 +109,20 @@ public class JobLogReportHelper {
                     } while (logIds!=null && !logIds.isEmpty());
 
                     // update clean time
-                            lastCleanLogTime = System.currentTimeMillis();
+                    lastCleanLogTime.set(System.currentTimeMillis());
-                        }
-                    } catch (Throwable e) {
-                        if (!toStop) {
-                            logger.error(">>>>>>>>>>> xxl-job, JobLogReportHelper(log-clean) error:{}", e.getMessage(), e);
-                        }
-                    }
-
-                    try {
-                        TimeUnit.MINUTES.sleep(1);
-                    } catch (Throwable e) {
-                        if (!toStop) {
-                            logger.error(e.getMessage(), e);
-                        }
                 }
 
             }
-
+        }, 60 * 1000L, true);
-                logger.info(">>>>>>>>>>> xxl-job, job log report thread stop");
-
-            }
-        });
-        logReportThread.setDaemon(true);
-        logReportThread.setName("xxl-job, admin JobLogReportHelper");
         logReportThread.start();
+
     }
 
     /**
      * stop
      */
     public void stop(){
-        toStop = true;
+        logReportThread.stop();
-        // interrupt and wait
-        logReportThread.interrupt();
-        try {
-            logReportThread.join();
-        } catch (Throwable e) {
-            logger.error(e.getMessage(), e);
-        }
     }
 
 }

xxl-job-admin/src/main/java/com/xxl/job/admin/business/scheduler/thread/JobRegistryHelper.java

@@ -3,9 +3,10 @@ package com.xxl.job.admin.business.scheduler.thread;
 import com.xxl.job.admin.business.model.XxlJobGroup;
 import com.xxl.job.admin.business.model.XxlJobRegistry;
 import com.xxl.job.admin.business.scheduler.config.XxlJobAdminBootstrap;
+import com.xxl.job.core.constant.Const;
 import com.xxl.job.core.constant.RegistType;
 import com.xxl.job.core.openapi.model.RegistryRequest;
-import com.xxl.job.core.constant.Const;
+import com.xxl.tool.concurrent.CyclicThread;
 import com.xxl.tool.core.StringTool;
 import com.xxl.tool.response.Response;
 import org.slf4j.Logger;
@@ -20,20 +21,25 @@ import java.util.concurrent.*;
  * @author xuxueli 2016-10-02 19:10:24
  */
 public class JobRegistryHelper {
-	private static Logger logger = LoggerFactory.getLogger(JobRegistryHelper.class);
+	private static final Logger logger = LoggerFactory.getLogger(JobRegistryHelper.class);
 
 
+	/**
+	 * registry or remove thread pool
+	 */
 	private ThreadPoolExecutor registryOrRemoveThreadPool = null;
-	private Thread registryMonitorThread;
-	private volatile boolean toStop = false;
 
+	/**
+	 * registry monitor thread
+	 */
+	private CyclicThread registryMonitorThread;
 
 	/**
 	 * start
 	 */
 	public void start(){
 
-		// for registry or remove
+		// 1、for registry or remove
 		registryOrRemoveThreadPool = new ThreadPoolExecutor(
 				2,
 				10,
@@ -54,12 +60,10 @@ public class JobRegistryHelper {
 					}
 				});
 
-		// for monitor
+		// 2、for registry monitor
-		registryMonitorThread = new Thread(new Runnable() {
+		registryMonitorThread = new CyclicThread("JobRegistryHelper#registryMonitorThread", true, new Runnable() {
 			@Override
 			public void run() {
-				while (!toStop) {
-					try {
 				// auto registry group
 				List<XxlJobGroup> groupList = XxlJobAdminBootstrap.getInstance().getXxlJobGroupMapper().findByAddressType(0);
 				if (groupList!=null && !groupList.isEmpty()) {
@@ -109,24 +113,8 @@ public class JobRegistryHelper {
 						XxlJobAdminBootstrap.getInstance().getXxlJobGroupMapper().update(group);
 					}
 				}
-					} catch (Throwable e) {
-						if (!toStop) {
-							logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e);
-						}
 			}
-					try {
+		}, Const.BEAT_TIMEOUT * 1000L, true);
-						TimeUnit.SECONDS.sleep(Const.BEAT_TIMEOUT);
-					} catch (Throwable e) {
-						if (!toStop) {
-							logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e);
-						}
-					}
-				}
-				logger.info(">>>>>>>>>>> xxl-job, job registry monitor thread stop");
-			}
-		});
-		registryMonitorThread.setDaemon(true);
-		registryMonitorThread.setName("xxl-job, admin JobRegistryMonitorHelper-registryMonitorThread");
 		registryMonitorThread.start();
 	}
 
@@ -135,18 +123,12 @@ public class JobRegistryHelper {
 	 * stop
 	 */
 	public void stop(){
-		toStop = true;
 
-		// stop registryOrRemoveThreadPool
+		// 1、registryOrRemoveThreadPool
 		registryOrRemoveThreadPool.shutdownNow();
 
-		// stop monitor (interrupt and wait)
+		// 2、registryMonitorThread
-		registryMonitorThread.interrupt();
+		registryMonitorThread.stop();
-		try {
-			registryMonitorThread.join();
-		} catch (Throwable e) {
-			logger.error(e.getMessage(), e);
-		}
 	}
 
 

xxl-job-admin/src/main/java/com/xxl/job/admin/business/scheduler/thread/JobScheduleHelper.java

@@ -44,7 +44,11 @@ public class JobScheduleHelper {
      */
     public void start(){
 
-        // schedule thread
+        // init thread flag
+        scheduleThreadToStop = false;
+        ringThreadToStop = false;
+
+        // 1、schedule thread
         scheduleThread = new Thread(new Runnable() {
             @Override
             public void run() {
@@ -191,8 +195,7 @@ public class JobScheduleHelper {
         scheduleThread.setName("xxl-job, admin JobScheduleHelper#scheduleThread");
         scheduleThread.start();
 
-
+        // 2、ring thread
-        // ring thread
         ringThread = new Thread(new Runnable() {
             @Override
             public void run() {
@@ -348,7 +351,7 @@ public class JobScheduleHelper {
             }
         }
 
-        // stop ring (wait job-in-memory stop)
+        // 2、stop ring (wait job-in-memory stop)
         ringThreadToStop = true;
         try {
             TimeUnit.SECONDS.sleep(1);