diff --git a/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java b/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java new file mode 100644 index 0000000..3cf4b1c --- /dev/null +++ b/austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java @@ -0,0 +1,37 @@ +package com.java3y.austin.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.List; + +/** + * 去重类型枚举 + * @author 3y + */ +@Getter +@ToString +@AllArgsConstructor +public enum DeduplicationType { + + CONTENT(10, "N分钟相同内容去重"), + FREQUENCY(20, "一天内N次相同渠道去重"), + ; + private Integer code; + private String description; + + + /** + * 获取去重渠道的列表 + * @return + */ + public static List getDeduplicationList() { + ArrayList result = new ArrayList<>(); + for (DeduplicationType value : DeduplicationType.values()) { + result.add(value.getCode()); + } + return result; + } +} diff --git a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java index 1c65366..8f73ea5 100644 --- a/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java +++ b/austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java @@ -2,8 +2,10 @@ package com.java3y.austin.domain; import com.alibaba.fastjson.annotation.JSONField; import com.java3y.austin.enums.AnchorState; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; /** * @author 3y @@ -12,6 +14,8 @@ import lombok.Data; */ @Builder @Data +@AllArgsConstructor +@NoArgsConstructor public class DeduplicationParam { /** * TaskIno信息 @@ -32,7 +36,7 @@ public class DeduplicationParam { private Integer countNum; /** - * 标识属于哪种去重 + * 标识属于哪种去重(数据埋点) */ private AnchorState anchorState; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java deleted file mode 100644 index b2cef57..0000000 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.java3y.austin.service.deduplication; - -/** - * @author huskey - * @date 2022/1/18 - */ -public class DeduplicationConstants { - - /** - * 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - */ - public static final String DEDUPLICATION_RULE_KEY = "deduplication"; - public static final String CONTENT_DEDUPLICATION = "contentDeduplication"; - public static final String FREQUENCY_DEDUPLICATION = "frequencyDeduplication"; -} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java new file mode 100644 index 0000000..342c5a8 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java @@ -0,0 +1,36 @@ +package com.java3y.austin.service.deduplication; + +import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.service.deduplication.service.DeduplicationService; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + + +/** + * @author huskey + * @date 2022/1/18 + */ +@Service +public class DeduplicationHolder { + + private Map builderHolder = new HashMap<>(4); + private Map serviceHolder = new HashMap<>(4); + + public Builder selectBuilder(Integer key) { + return builderHolder.get(key); + } + + public DeduplicationService selectService(Integer key) { + return serviceHolder.get(key); + } + + public void putBuilder(Integer key, Builder builder) { + builderHolder.put(key, builder); + } + + public void putService(Integer key, DeduplicationService service) { + serviceHolder.put(key, service); + } +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java index 2db2503..abfc5d0 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java @@ -2,13 +2,11 @@ package com.java3y.austin.service.deduplication; import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.spring.annotation.ApolloConfig; -import com.google.common.collect.Lists; import com.java3y.austin.constant.AustinConstant; import com.java3y.austin.domain.DeduplicationParam; import com.java3y.austin.domain.TaskInfo; -import com.java3y.austin.service.deduplication.build.BuilderFactory; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; import java.util.List; @@ -21,39 +19,27 @@ import java.util.List; @Service public class DeduplicationRuleService { + public static final String DEDUPLICATION_RULE_KEY = "deduplication"; - private static final String SERVICE = "Service"; @ApolloConfig("boss.austin") private Config config; @Autowired - private ApplicationContext applicationContext; - @Autowired - private BuilderFactory builderFactory; - - //需要去重的服务 - private static final List DEDUPLICATION_LIST = Lists.newArrayList(DeduplicationConstants.CONTENT_DEDUPLICATION, DeduplicationConstants.FREQUENCY_DEDUPLICATION); + private DeduplicationHolder deduplicationHolder; public void duplication(TaskInfo taskInfo) { - // 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}} - String deduplication = config.getProperty(DeduplicationConstants.DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT); - //去重 - DEDUPLICATION_LIST.forEach( - key -> { - DeduplicationParam deduplicationParam = builderFactory.select(key).build(deduplication, key); - if (deduplicationParam != null) { - deduplicationParam.setTaskInfo(taskInfo); - DeduplicationService deduplicationService = findService(key + SERVICE); - deduplicationService.deduplication(deduplicationParam); - } - } - ); - + // 配置样例：{"deduplication_10":{"num":1,"time":300},"deduplication_20":{"num":5}} + String deduplicationConfig = config.getProperty(DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT); + + // 去重 + List deduplicationList = DeduplicationType.getDeduplicationList(); + for (Integer deduplicationType : deduplicationList) { + DeduplicationParam deduplicationParam = deduplicationHolder.selectBuilder(deduplicationType).build(deduplicationConfig, taskInfo); + if (deduplicationParam != null) { + deduplicationHolder.selectService(deduplicationType).deduplication(deduplicationParam); + } + } } - private DeduplicationService findService(String beanName) { - return applicationContext.getBean(beanName, DeduplicationService.class); - - } } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java new file mode 100644 index 0000000..62ade65 --- /dev/null +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java @@ -0,0 +1,40 @@ +package com.java3y.austin.service.deduplication.build; + +import com.alibaba.fastjson.JSONObject; +import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.service.deduplication.DeduplicationHolder; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.PostConstruct; + +/** + * @author 3y + * @date 2022/1/19 + */ +public abstract class AbstractDeduplicationBuilder implements Builder { + + protected Integer deduplicationType; + + @Autowired + private DeduplicationHolder deduplicationHolder; + + @PostConstruct + public void init() { + deduplicationHolder.putBuilder(deduplicationType, this); + } + + public DeduplicationParam getParamsFromConfig(Integer key, String duplicationConfig, TaskInfo taskInfo) { + JSONObject object = JSONObject.parseObject(duplicationConfig); + if (object == null) { + return null; + } + DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(CONFIG_PRE + key), DeduplicationParam.class); + if (deduplicationParam == null) { + return null; + } + deduplicationParam.setTaskInfo(taskInfo); + return deduplicationParam; + } + +} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java index cd78e38..b2fe075 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java @@ -1,6 +1,7 @@ package com.java3y.austin.service.deduplication.build; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; /** * @author luohaojie @@ -8,5 +9,14 @@ import com.java3y.austin.domain.DeduplicationParam; */ public interface Builder { - DeduplicationParam build(String deduplication, String key); + String CONFIG_PRE = "deduplication_"; + + /** + * 根据配置构建去重参数 + * + * @param deduplication + * @param taskInfo + * @return + */ + DeduplicationParam build(String deduplication, TaskInfo taskInfo); } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java deleted file mode 100644 index 795dafa..0000000 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.java3y.austin.service.deduplication.build; - -import com.java3y.austin.service.deduplication.DeduplicationConstants; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; - - -/** - * @author huskey - * @date 2022/1/18 - */ - -@Service -public class BuilderFactory { - - - private Map builderFactory = new HashMap<>(4); - - @Autowired - private Builder contentDeduplicationBuilder; - - @Autowired - private Builder frequencyDeduplicationBuilder; - - @PostConstruct - public void init() { - builderFactory.put(DeduplicationConstants.CONTENT_DEDUPLICATION, contentDeduplicationBuilder); - builderFactory.put(DeduplicationConstants.FREQUENCY_DEDUPLICATION, frequencyDeduplicationBuilder); - } - - - public Builder select(String key) { - return builderFactory.get(key); - } - -} diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java index a542712..238819e 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java @@ -1,9 +1,9 @@ package com.java3y.austin.service.deduplication.build; -import com.alibaba.fastjson.JSONObject; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; import com.java3y.austin.enums.AnchorState; -import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; @@ -11,20 +11,20 @@ import org.springframework.stereotype.Service; * @author huskey * @date 2022/1/18 */ - @Service -public class ContentDeduplicationBuilder implements Builder { +public class ContentDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder { - public DeduplicationParam build(String deduplication, String key) { - JSONObject object = JSONObject.parseObject(deduplication); - if (object == null) { - return null; - } - DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); - if (deduplicationParam == null) { - return null; + public ContentDeduplicationBuilder() { + deduplicationType = DeduplicationType.CONTENT.getCode(); + } + + @Override + public DeduplicationParam build(String deduplication, TaskInfo taskInfo) { + DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo); + if (deduplication == null) { + return null; } - deduplicationParam.setAnchorState(AnchorState.RULE_DEDUPLICATION); + deduplicationParam.setAnchorState(AnchorState.CONTENT_DEDUPLICATION); return deduplicationParam; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java index 972ed87..b1d0ab4 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java @@ -1,10 +1,10 @@ package com.java3y.austin.service.deduplication.build; import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.JSONObject; import com.java3y.austin.domain.DeduplicationParam; +import com.java3y.austin.domain.TaskInfo; import com.java3y.austin.enums.AnchorState; -import com.java3y.austin.service.deduplication.build.Builder; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; import java.util.Date; @@ -15,15 +15,14 @@ import java.util.Date; */ @Service -public class FrequencyDeduplicationBuilder implements Builder { +public class FrequencyDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder { + public FrequencyDeduplicationBuilder() { + deduplicationType = DeduplicationType.FREQUENCY.getCode(); + } @Override - public DeduplicationParam build(String deduplication, String key) { - JSONObject object = JSONObject.parseObject(deduplication); - if (object == null) { - return null; - } - DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class); + public DeduplicationParam build(String deduplication, TaskInfo taskInfo) { + DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo); if (deduplicationParam == null) { return null; } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java similarity index 89% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java index d1fb8c6..b51b9db 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/AbstractDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java @@ -1,16 +1,17 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.core.collection.CollUtil; import com.java3y.austin.constant.AustinConstant; import com.java3y.austin.domain.AnchorInfo; import com.java3y.austin.domain.DeduplicationParam; import com.java3y.austin.domain.TaskInfo; -import com.java3y.austin.enums.AnchorState; +import com.java3y.austin.service.deduplication.DeduplicationHolder; import com.java3y.austin.utils.LogUtils; import com.java3y.austin.utils.RedisUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import javax.annotation.PostConstruct; import java.util.*; /** @@ -20,10 +21,20 @@ import java.util.*; */ @Slf4j public abstract class AbstractDeduplicationService implements DeduplicationService { + protected Integer deduplicationType; + + @Autowired + private DeduplicationHolder deduplicationHolder; + + @PostConstruct + private void init() { + deduplicationHolder.putService(deduplicationType, this); + } @Autowired private RedisUtils redisUtils; + @Override public void deduplication(DeduplicationParam param) { TaskInfo taskInfo = param.getTaskInfo(); Set filterReceiver = new HashSet<>(taskInfo.getReceiver().size()); diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java similarity index 79% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java index 1b41e44..a305359 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/ContentDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java @@ -1,8 +1,9 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.crypto.digest.DigestUtil; import com.alibaba.fastjson.JSON; import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; /** @@ -13,6 +14,10 @@ import org.springframework.stereotype.Service; @Service public class ContentDeduplicationService extends AbstractDeduplicationService { + public ContentDeduplicationService() { + deduplicationType = DeduplicationType.CONTENT.getCode(); + } + /** * 内容去重 构建key *

diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java similarity index 64% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java index 35c4b80..e0abf27 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java @@ -1,4 +1,4 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import com.java3y.austin.domain.DeduplicationParam; @@ -8,5 +8,9 @@ import com.java3y.austin.domain.DeduplicationParam; */ public interface DeduplicationService { + /** + * 去重 + * @param param + */ void deduplication(DeduplicationParam param); } diff --git a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java similarity index 80% rename from austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java rename to austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java index c135f46..bc23513 100644 --- a/austin-handler/src/main/java/com/java3y/austin/service/deduplication/FrequencyDeduplicationService.java +++ b/austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java @@ -1,7 +1,8 @@ -package com.java3y.austin.service.deduplication; +package com.java3y.austin.service.deduplication.service; import cn.hutool.core.util.StrUtil; import com.java3y.austin.domain.TaskInfo; +import com.java3y.austin.enums.DeduplicationType; import org.springframework.stereotype.Service; /** @@ -12,6 +13,10 @@ import org.springframework.stereotype.Service; @Service public class FrequencyDeduplicationService extends AbstractDeduplicationService { + public FrequencyDeduplicationService() { + deduplicationType = DeduplicationType.FREQUENCY.getCode(); + } + private static final String PREFIX = "FRE"; /**