1.延迟队列实现 优化

2.完成定时任务 实现逻辑
pull/4/head
3y 3 years ago
parent d3e6265270
commit e27e420160

@ -54,7 +54,7 @@ austin项目**核心流程**`austin-api`接收到发送消息请求,直接
**1**、austin使用的MySQL版本**5.7x**。如果目前使用的MySQL版本8.0,注意改变`pom.xml`所依赖的版本 **1**、austin使用的MySQL版本**5.7x**。如果目前使用的MySQL版本8.0,注意改变`pom.xml`所依赖的版本
**2**、适配`application.properties`的配置信息(`srping.datasource`) **2**、适配`application.properties`的配置信息(`spring.datasource`)
**3**、执行`sql`文件夹下的`austin.sql`创建对应的表 **3**、执行`sql`文件夹下的`austin.sql`创建对应的表
@ -97,9 +97,9 @@ curl -XPOST "127.0.0.1:8080/send" -H 'Content-Type: application/json' -d '{"co
- [ ] 04、持续提高消息推送系统的影响力让更多的业务方了解其功能进而挖掘更多拉新和唤醒用户的玩法提高站内的次留率和转化率 - [ ] 04、持续提高消息推送系统的影响力让更多的业务方了解其功能进而挖掘更多拉新和唤醒用户的玩法提高站内的次留率和转化率
**近期更新时间**2022年1月25 **近期更新时间**2022年2月14
**近期更新功能**austin前端管理系统 **近期更新功能**接入xxl-job分布式定时任务框架并完成定时任务逻辑
## 项目交流 ## 项目交流

@ -21,6 +21,16 @@
<artifactId>austin-support</artifactId> <artifactId>austin-support</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
</dependency> </dependency>
<dependency>
<groupId>com.java3y.austin</groupId>
<artifactId>austin-service-api</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.java3y.austin</groupId>
<artifactId>austin-service-api-impl</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency> <dependency>
<groupId>com.xuxueli</groupId> <groupId>com.xuxueli</groupId>
<artifactId>xxl-job-core</artifactId> <artifactId>xxl-job-core</artifactId>

@ -1,9 +1,12 @@
package com.java3y.austin.cron.constants; package com.java3y.austin.cron.constants;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
/** /**
* @author 3y * @author 3y
* @date 2022/2/13 * @date 2022/2/13
* pending * pending
*/ */
public class PendingConstant { public class PendingConstant {
@ -23,9 +26,10 @@ public class PendingConstant {
public static final Long TIME_THRESHOLD = 1000L; public static final Long TIME_THRESHOLD = 1000L;
/** /**
* 线 * 线
*/ */
public static final Integer THREAD_NUM = 2; public static final Integer CORE_POOL_SIZE = 2;
public static final Integer MAX_POOL_SIZE = 2;
public static final BlockingQueue BLOCKING_QUEUE = new LinkedBlockingQueue<>(5);
} }

@ -27,7 +27,6 @@ public class CronTaskHandler {
log.info("CronTaskHandler#execute messageTemplateId:{} cron exec!", XxlJobHelper.getJobParam()); log.info("CronTaskHandler#execute messageTemplateId:{} cron exec!", XxlJobHelper.getJobParam());
Long messageTemplateId = Long.valueOf(XxlJobHelper.getJobParam()); Long messageTemplateId = Long.valueOf(XxlJobHelper.getJobParam());
taskHandler.handle(messageTemplateId); taskHandler.handle(messageTemplateId);
} }
} }

@ -1,39 +1,88 @@
package com.java3y.austin.cron.pending; package com.java3y.austin.cron.pending;
import com.java3y.austin.cron.domain.CrowdInfoVo; import cn.hutool.core.collection.CollUtil;
import com.java3y.austin.support.pending.BatchPendingThread; import cn.hutool.core.map.MapUtil;
import com.java3y.austin.support.pending.Pending; import cn.hutool.core.thread.ExecutorBuilder;
import cn.hutool.core.util.StrUtil;
import com.google.common.collect.Lists;
import com.java3y.austin.cron.constants.PendingConstant;
import com.java3y.austin.cron.vo.CrowdInfoVo;
import com.java3y.austin.service.api.domain.BatchSendRequest;
import com.java3y.austin.service.api.domain.MessageParam;
import com.java3y.austin.service.api.enums.BusinessCode;
import com.java3y.austin.service.api.service.SendService;
import com.java3y.austin.support.pending.AbstractLazyPending;
import com.java3y.austin.support.pending.PendingParam; import com.java3y.austin.support.pending.PendingParam;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
/** /**
* *
* batch
* *
* @author 3y * @author 3y
*/ */
@Component
@Slf4j @Slf4j
public class CrowdBatchTaskPending extends Pending<CrowdInfoVo> { @Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class CrowdBatchTaskPending extends AbstractLazyPending<CrowdInfoVo> {
@Autowired
private SendService sendService;
public CrowdBatchTaskPending() {
PendingParam<CrowdInfoVo> pendingParam = new PendingParam<>();
pendingParam.setNumThreshold(PendingConstant.NUM_THRESHOLD)
.setQueue(new LinkedBlockingQueue(PendingConstant.QUEUE_SIZE))
.setTimeThreshold(PendingConstant.TIME_THRESHOLD)
.setExecutorService(ExecutorBuilder.create()
.setCorePoolSize(PendingConstant.CORE_POOL_SIZE)
.setMaxPoolSize(PendingConstant.MAX_POOL_SIZE)
.setWorkQueue(PendingConstant.BLOCKING_QUEUE)
.setHandler(new ThreadPoolExecutor.CallerRunsPolicy())
.build());
this.pendingParam = pendingParam;
}
@Override @Override
public void initAndStart(PendingParam pendingParam) { public void doHandle(List<CrowdInfoVo> crowdInfoVos) {
threadNum = pendingParam.getThreadNum() == null ? threadNum : pendingParam.getThreadNum();
queue = pendingParam.getQueue(); // 1. 如果参数相同组装成同一个MessageParam发送
Map<Map<String, String>, String> paramMap = MapUtil.newHashMap();
for (int i = 0; i < threadNum; ++i) { for (CrowdInfoVo crowdInfoVo : crowdInfoVos) {
BatchPendingThread<CrowdInfoVo> batchPendingThread = new BatchPendingThread(); String receiver = crowdInfoVo.getReceiver();
batchPendingThread.setPendingParam(pendingParam); Map<String, String> vars = crowdInfoVo.getParams();
batchPendingThread.setName("batchPendingThread-" + i); if (paramMap.get(vars) == null) {
batchPendingThread.start(); paramMap.put(vars, receiver);
} else {
String newReceiver = StringUtils.join(new String[]{
paramMap.get(vars), receiver}, StrUtil.COMMA);
paramMap.put(vars, newReceiver);
} }
} }
@Override // 2. 组装参数
public void doHandle(List<CrowdInfoVo> list) { List<MessageParam> messageParams = Lists.newArrayList();
log.info("theadName:{},doHandle:{}", Thread.currentThread().getName(), list.size()); for (Map.Entry<Map<String, String>, String> entry : paramMap.entrySet()) {
MessageParam messageParam = MessageParam.builder().receiver(entry.getValue())
.variables(entry.getKey()).build();
messageParams.add(messageParam);
}
// 3. 调用批量发送接口发送消息
BatchSendRequest batchSendRequest = BatchSendRequest.builder().code(BusinessCode.COMMON_SEND.getCode())
.messageParamList(messageParams)
.messageTemplateId(CollUtil.getFirst(crowdInfoVos.iterator()).getMessageTemplateId())
.build();
sendService.batchSend(batchSendRequest);
} }
} }

@ -2,21 +2,19 @@ package com.java3y.austin.cron.service.impl;
import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import com.java3y.austin.cron.constants.PendingConstant;
import com.java3y.austin.cron.domain.CrowdInfoVo;
import com.java3y.austin.cron.pending.CrowdBatchTaskPending; import com.java3y.austin.cron.pending.CrowdBatchTaskPending;
import com.java3y.austin.cron.service.TaskHandler; import com.java3y.austin.cron.service.TaskHandler;
import com.java3y.austin.cron.utils.ReadFileUtils; import com.java3y.austin.cron.utils.ReadFileUtils;
import com.java3y.austin.cron.vo.CrowdInfoVo;
import com.java3y.austin.support.dao.MessageTemplateDao; import com.java3y.austin.support.dao.MessageTemplateDao;
import com.java3y.austin.support.domain.MessageTemplate; import com.java3y.austin.support.domain.MessageTemplate;
import com.java3y.austin.support.pending.PendingParam;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.HashMap; import java.util.HashMap;
import java.util.concurrent.LinkedBlockingQueue;
/** /**
* @author 3y * @author 3y
@ -27,14 +25,14 @@ import java.util.concurrent.LinkedBlockingQueue;
public class TaskHandlerImpl implements TaskHandler { public class TaskHandlerImpl implements TaskHandler {
@Autowired @Autowired
private MessageTemplateDao messageTemplateDao; private MessageTemplateDao messageTemplateDao;
@Autowired
private CrowdBatchTaskPending crowdBatchTaskPending;
@Autowired
private ApplicationContext context;
@Override @Override
@Async @Async
public void handle(Long messageTemplateId) { public void handle(Long messageTemplateId) {
log.info("start:{}", Thread.currentThread().getName()); log.info("TaskHandler handle:{}", Thread.currentThread().getName());
MessageTemplate messageTemplate = messageTemplateDao.findById(messageTemplateId).get(); MessageTemplate messageTemplate = messageTemplateDao.findById(messageTemplateId).get();
if (messageTemplate == null || StrUtil.isBlank(messageTemplate.getCronCrowdPath())) { if (messageTemplate == null || StrUtil.isBlank(messageTemplate.getCronCrowdPath())) {
@ -42,16 +40,8 @@ public class TaskHandlerImpl implements TaskHandler {
return; return;
} }
// 初始化pending的信息 CrowdBatchTaskPending crowdBatchTaskPending = context.getBean(CrowdBatchTaskPending.class);
PendingParam<CrowdInfoVo> pendingParam = new PendingParam<>(); // 读取文件得到每一行记录给到队列做lazy batch处理
pendingParam.setNumThreshold(PendingConstant.NUM_THRESHOLD)
.setQueue(new LinkedBlockingQueue(PendingConstant.QUEUE_SIZE))
.setTimeThreshold(PendingConstant.TIME_THRESHOLD)
.setThreadNum(PendingConstant.THREAD_NUM)
.setPending(crowdBatchTaskPending);
crowdBatchTaskPending.initAndStart(pendingParam);
// 读取文件得到每一行记录给到队列做batch处理
ReadFileUtils.getCsvRow(messageTemplate.getCronCrowdPath(), row -> { ReadFileUtils.getCsvRow(messageTemplate.getCronCrowdPath(), row -> {
if (CollUtil.isEmpty(row.getFieldMap()) if (CollUtil.isEmpty(row.getFieldMap())
|| StrUtil.isBlank(row.getFieldMap().get(ReadFileUtils.RECEIVER_KEY))) { || StrUtil.isBlank(row.getFieldMap().get(ReadFileUtils.RECEIVER_KEY))) {
@ -59,7 +49,7 @@ public class TaskHandlerImpl implements TaskHandler {
} }
HashMap<String, String> params = ReadFileUtils.getParamFromLine(row.getFieldMap()); HashMap<String, String> params = ReadFileUtils.getParamFromLine(row.getFieldMap());
CrowdInfoVo crowdInfoVo = CrowdInfoVo.builder().receiver(row.getFieldMap().get(ReadFileUtils.RECEIVER_KEY)) CrowdInfoVo crowdInfoVo = CrowdInfoVo.builder().receiver(row.getFieldMap().get(ReadFileUtils.RECEIVER_KEY))
.params(params).build(); .params(params).messageTemplateId(messageTemplateId).build();
crowdBatchTaskPending.pending(crowdInfoVo); crowdBatchTaskPending.pending(crowdInfoVo);
}); });

@ -4,7 +4,7 @@ import cn.hutool.core.io.FileUtil;
import cn.hutool.core.map.MapUtil; import cn.hutool.core.map.MapUtil;
import cn.hutool.core.text.csv.*; import cn.hutool.core.text.csv.*;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
import com.java3y.austin.cron.domain.CrowdInfoVo; import com.java3y.austin.cron.vo.CrowdInfoVo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import java.io.FileReader; import java.io.FileReader;

@ -1,4 +1,4 @@
package com.java3y.austin.cron.domain; package com.java3y.austin.cron.vo;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@ -21,6 +21,11 @@ import java.util.Map;
@Builder @Builder
public class CrowdInfoVo implements Serializable { public class CrowdInfoVo implements Serializable {
/**
* Id
*/
private Long messageTemplateId;
/** /**
* id * id
*/ */

@ -1,36 +0,0 @@
package com.java3y.austin.handler.config;
import com.java3y.austin.handler.pending.Task;
import com.java3y.austin.handler.receiver.Receiver;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
/**
* Handler
*
* @author 3y
*/
@Configuration
public class PrototypeBeanConfig {
/**
* Receiver
*/
@Bean
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public Receiver receiver() {
return new Receiver();
}
/**
* Task
*/
@Bean
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public Task task() {
return new Task();
}
}

@ -10,6 +10,9 @@ import lombok.Data;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
/** /**
* Task * Task
@ -22,6 +25,8 @@ import org.springframework.beans.factory.annotation.Autowired;
@Data @Data
@Accessors(chain = true) @Accessors(chain = true)
@Slf4j @Slf4j
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class Task implements Runnable { public class Task implements Runnable {
@Autowired @Autowired

@ -12,10 +12,13 @@ import com.java3y.austin.support.utils.LogUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Scope;
import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.KafkaHeaders; import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.handler.annotation.Header; import org.springframework.messaging.handler.annotation.Header;
import org.springframework.stereotype.Component;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@ -25,6 +28,8 @@ import java.util.Optional;
* MQ * MQ
*/ */
@Slf4j @Slf4j
@Component
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public class Receiver { public class Receiver {
private static final String LOG_BIZ_TYPE = "Receiver#consumer"; private static final String LOG_BIZ_TYPE = "Receiver#consumer";
@Autowired @Autowired

@ -51,8 +51,8 @@ public class AfterParamCheckAction implements BusinessProcess {
* @param taskInfo * @param taskInfo
*/ */
private void filterIllegalPhoneNum(List<TaskInfo> taskInfo) { private void filterIllegalPhoneNum(List<TaskInfo> taskInfo) {
Integer idType = taskInfo.get(0).getIdType(); Integer idType = CollUtil.getFirst(taskInfo.iterator()).getIdType();
Integer sendChannel = taskInfo.get(0).getSendChannel(); Integer sendChannel = CollUtil.getFirst(taskInfo.iterator()).getSendChannel();
if (IdType.PHONE.getCode().equals(idType) && ChannelType.SMS.getCode().equals(sendChannel)) { if (IdType.PHONE.getCode().equals(idType) && ChannelType.SMS.getCode().equals(sendChannel)) {
Iterator<TaskInfo> iterator = taskInfo.iterator(); Iterator<TaskInfo> iterator = taskInfo.iterator();
@ -66,7 +66,7 @@ public class AfterParamCheckAction implements BusinessProcess {
if (CollUtil.isNotEmpty(illegalPhone)) { if (CollUtil.isNotEmpty(illegalPhone)) {
task.getReceiver().removeAll(illegalPhone); task.getReceiver().removeAll(illegalPhone);
log.error("{} find illegal phone!{}", task.getMessageTemplateId(), JSON.toJSONString(illegalPhone)); log.error("messageTemplateId:{} find illegal phone!{}", task.getMessageTemplateId(), JSON.toJSONString(illegalPhone));
} }
if (CollUtil.isEmpty(task.getReceiver())) { if (CollUtil.isEmpty(task.getReceiver())) {
iterator.remove(); iterator.remove();

@ -1,5 +1,6 @@
package com.java3y.austin.service.api.impl.action; package com.java3y.austin.service.api.impl.action;
import cn.hutool.core.collection.CollUtil;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
@ -35,7 +36,8 @@ public class SendMqAction implements BusinessProcess {
} catch (Exception e) { } catch (Exception e) {
context.setNeedBreak(true).setResponse(BasicResultVO.fail(RespStatusEnum.SERVICE_ERROR)); context.setNeedBreak(true).setResponse(BasicResultVO.fail(RespStatusEnum.SERVICE_ERROR));
log.error("send kafka fail! e:{},params:{}", Throwables.getStackTraceAsString(e) log.error("send kafka fail! e:{},params:{}", Throwables.getStackTraceAsString(e)
, JSON.toJSONString(sendTaskModel.getTaskInfo().get(0))); , JSON.toJSONString(CollUtil.getFirst(sendTaskModel.getTaskInfo().listIterator())));
} }
} }
} }

@ -1,6 +1,9 @@
package com.java3y.austin.service.api.domain; package com.java3y.austin.service.api.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import java.util.List; import java.util.List;
@ -13,6 +16,9 @@ import java.util.List;
*/ */
@Data @Data
@Accessors(chain = true) @Accessors(chain = true)
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class BatchSendRequest { public class BatchSendRequest {

@ -1,7 +1,9 @@
package com.java3y.austin.service.api.domain; package com.java3y.austin.service.api.domain;
import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import java.util.Map; import java.util.Map;
@ -13,8 +15,9 @@ import java.util.Map;
*/ */
@Data @Data
@Accessors(chain = true) @Accessors(chain = true)
@AllArgsConstructor
@NoArgsConstructor
@Builder @Builder
public class MessageParam { public class MessageParam {
/** /**

@ -0,0 +1,113 @@
package com.java3y.austin.support.pending;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.thread.ThreadUtil;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* -
*
* @author 3y
*/
@Slf4j
@Data
public abstract class AbstractLazyPending<T> {
/**
*
*/
protected PendingParam<T> pendingParam;
/**
*
*/
private List<T> tasks = new ArrayList<>();
/**
*
*/
private Long lastHandleTime = System.currentTimeMillis();
/**
* 线
*/
@PostConstruct
public void initConsumePending() {
ThreadUtil.newSingleExecutor().execute(() -> {
while (true) {
try {
T obj = pendingParam.getQueue().poll(pendingParam.getTimeThreshold(), TimeUnit.MILLISECONDS);
if (null != obj) {
tasks.add(obj);
}
// 处理条件1. 数量超限 2. 时间超限
if (CollUtil.isNotEmpty(tasks) && dataReady()) {
List<T> taskRef = tasks;
tasks = Lists.newArrayList();
lastHandleTime = System.currentTimeMillis();
// 具体执行逻辑
pendingParam.getExecutorService().execute(() -> this.handle(taskRef));
}
} catch (Exception e) {
log.error("Pending#initConsumePending failed:{}", Throwables.getStackTraceAsString(e));
}
}
});
}
/**
* 1.
* 2.
* @return
*/
private boolean dataReady() {
return tasks.size() >= pendingParam.getNumThreshold() ||
(System.currentTimeMillis() - lastHandleTime >= pendingParam.getTimeThreshold());
}
/**
*
*
* @param t
*/
public void pending(T t) {
try {
pendingParam.getQueue().put(t);
} catch (InterruptedException e) {
log.error("Pending#pending error:{}", Throwables.getStackTraceAsString(e));
}
}
/**
*
*
* @param t
*/
public void handle(List<T> t) {
if (t.isEmpty()) {
return;
}
try {
doHandle(t);
} catch (Exception e) {
log.error("Pending#handle failed:{}", Throwables.getStackTraceAsString(e));
}
}
/**
*
*
* @param list
*/
public abstract void doHandle(List<T> list);
}

@ -1,64 +0,0 @@
package com.java3y.austin.support.pending;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* 线
*
*
* @author 3y
*/
@Data
@Accessors(chain = true)
@Slf4j
public class BatchPendingThread<T> extends Thread {
private PendingParam<T> pendingParam;
/**
*
*/
private List<T> tasks = new ArrayList<>();
/**
*
*/
private Integer total = 0;
/**
*
*/
private Long lastHandleTime = System.currentTimeMillis();
@Override
public void run() {
while (true) {
try {
T obj = pendingParam.getQueue().poll(pendingParam.getTimeThreshold(), TimeUnit.MILLISECONDS);
if (null != obj) {
tasks.add(obj);
}
// 处理条件1. 数量超限 2. 时间超限
if ((tasks.size() >= pendingParam.getNumThreshold())
|| (System.currentTimeMillis() - lastHandleTime >= pendingParam.getTimeThreshold())) {
List<T> taskRef = tasks;
tasks = Lists.newArrayList();
lastHandleTime = System.currentTimeMillis();
pendingParam.getPending().handle(taskRef);
}
} catch (Exception e) {
log.error("BatchPendingThread#run failed:{}", Throwables.getStackTraceAsString(e));
}
}
}
}

@ -1,79 +0,0 @@
package com.java3y.austin.support.pending;
import com.google.common.base.Throwables;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.concurrent.BlockingQueue;
/**
* -
*
* @author 3y
*/
@Slf4j
@Data
public abstract class Pending<T> {
/**
* 使线
*/
public static final int DEFAULT_THREAD_NUM = Runtime.getRuntime().availableProcessors();
/**
*
*/
protected BlockingQueue<T> queue;
/**
* 线 = 使线
*/
protected Integer threadNum = DEFAULT_THREAD_NUM;
/**
*
*
* @param t
*/
public void pending(T t) {
try {
queue.put(t);
} catch (InterruptedException e) {
log.error("Pending#pending error:{}", Throwables.getStackTraceAsString(e));
}
}
/**
*
*
* @param t
*/
public void handle(List<T> t) {
if (t.isEmpty()) {
return;
}
try {
doHandle(t);
} catch (Exception e) {
log.error("Pending#handle failed:{}", Throwables.getStackTraceAsString(e));
}
}
/**
*
*
* @param pendingParam
*/
public abstract void initAndStart(PendingParam pendingParam);
/**
*
*
* @param list
*/
public abstract void doHandle(List<T> list);
}

@ -8,6 +8,7 @@ import lombok.NoArgsConstructor;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
/** /**
@ -37,12 +38,8 @@ public class PendingParam<T> {
private Long timeThreshold; private Long timeThreshold;
/** /**
* pending * 线
*/ */
private Pending pending; protected ExecutorService executorService;
/**
* 线
*/
protected Integer threadNum;
} }

@ -11,7 +11,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
public class AustinApplication { public class AustinApplication {
public static void main(String[] args) { public static void main(String[] args) {
// TODO apollo的地址. // TODO apollo的地址.
//System.setProperty("apollo.config-service", "http://ip:7000"); System.setProperty("apollo.config-service", "http://119.91.205.248:7000");
SpringApplication.run(AustinApplication.class, args); SpringApplication.run(AustinApplication.class, args);
} }
} }

@ -5,14 +5,14 @@
##################### database properties ##################### ##################### database properties #####################
# notice:mysql version 5.7x !!! todo ip port username password # notice:mysql version 5.7x !!! todo ip port username password
spring.datasource.url=jdbc:mysql://${ip}:${port}/austin?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&zeroDateTimeBehavior=convertToNull spring.datasource.url=jdbc:mysql://localhost:3306/austin?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&zeroDateTimeBehavior=convertToNull
spring.datasource.username=${username} spring.datasource.username=root
spring.datasource.password=${password} spring.datasource.password=root123_A
spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.driver-class-name=com.mysql.jdbc.Driver
##################### kafka properties ##################### ##################### kafka properties #####################
# todo ip port # todo ip port
spring.kafka.bootstrap-servers=${ip}:${port} spring.kafka.bootstrap-servers=120.48.13.113:9092
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
@ -23,28 +23,28 @@ spring.kafka.consumer.enable-auto-commit=true
##################### redis properties ##################### ##################### redis properties #####################
# todo ip port password # todo ip port password
spring.redis.host=${ip} spring.redis.host=119.91.205.248
spring.redis.port=${port} spring.redis.port=6379
spring.redis.password=${password} spring.redis.password=austin
##################### business properties ##################### ##################### business properties #####################
austin.business.topic.name=austinTopicV2 austin.business.topic.name=austinTopicV2
# TODO if need graylog ,replace ip ! # TODO if need graylog ,replace ip !
austin.business.graylog.ip=${ip} austin.business.graylog.ip=120.48.13.113
# TODO if windows os ,replace path ! # TODO if windows os ,replace path !
austin.business.upload.crowd.path=/Users/3y/temp austin.business.upload.crowd.path=/Users/3y/temp
##################### xxl properties ##################### ##################### xxl properties #####################
# todo ip port # todo ip port
xxl.job.admin.addresses=http://${ip}:${port}/xxl-job-admin xxl.job.admin.addresses=http://localhost:6767/xxl-job-admin
xxl.job.admin.username=admin xxl.job.admin.username=admin
xxl.job.admin.password=123456 xxl.job.admin.password=123456
xxl.job.executor.appname=austin xxl.job.executor.appname=austin
xxl.job.executor.jobHandlerName=austinJob xxl.job.executor.jobHandlerName=austinJob
xxl.job.executor.ip= xxl.job.executor.ip=
xxl.job.executor.port=6666 xxl.job.executor.port=6666
xxl.job.executor.logpath= xxl.job.executor.logpath=logs/xxl
xxl.job.executor.logretentiondays=30 xxl.job.executor.logretentiondays=30
xxl.job.accessToken= xxl.job.accessToken=

Loading…
Cancel
Save