重构去重服务

austin-common/src/main/java/com/java3y/austin/enums/DeduplicationType.java

@@ -0,0 +1,37 @@
+package com.java3y.austin.enums;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.ToString;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * 去重类型枚举
+ * @author 3y
+ */
+@Getter
+@ToString
+@AllArgsConstructor
+public enum DeduplicationType {
+
+    CONTENT(10, "N分钟相同内容去重"),
+    FREQUENCY(20, "一天内N次相同渠道去重"),
+    ;
+    private Integer code;
+    private String description;
+
+
+    /**
+     * 获取去重渠道的列表
+     * @return
+     */
+    public static List<Integer> getDeduplicationList() {
+        ArrayList<Integer> result = new ArrayList<>();
+        for (DeduplicationType value : DeduplicationType.values()) {
+            result.add(value.getCode());
+        }
+        return result;
+    }
+}

austin-handler/src/main/java/com/java3y/austin/domain/DeduplicationParam.java

@@ -2,8 +2,10 @@ package com.java3y.austin.domain;
 
 import com.alibaba.fastjson.annotation.JSONField;
 import com.java3y.austin.enums.AnchorState;
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 /**
  * @author 3y
@@ -12,6 +14,8 @@ import lombok.Data;
  */
 @Builder
 @Data
+@AllArgsConstructor
+@NoArgsConstructor
 public class DeduplicationParam {
     /**
      * TaskIno信息
@@ -32,7 +36,7 @@ public class DeduplicationParam {
     private Integer countNum;
 
     /**
-     * 标识属于哪种去重
+     * 标识属于哪种去重(数据埋点)
      */
     private AnchorState anchorState;
 }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationConstants.java

@@ -1,15 +0,0 @@
-package com.java3y.austin.service.deduplication;
-
-/**
- * @author huskey
- * @date 2022/1/18
- */
-public class DeduplicationConstants {
-
-    /**
-     * 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}}
-     */
-    public static final String DEDUPLICATION_RULE_KEY = "deduplication";
-    public static final String CONTENT_DEDUPLICATION = "contentDeduplication";
-    public static final String FREQUENCY_DEDUPLICATION = "frequencyDeduplication";
-}

austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationHolder.java

@@ -0,0 +1,36 @@
+package com.java3y.austin.service.deduplication;
+
+import com.java3y.austin.service.deduplication.build.Builder;
+import com.java3y.austin.service.deduplication.service.DeduplicationService;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * @author huskey
+ * @date 2022/1/18
+ */
+@Service
+public class DeduplicationHolder {
+
+    private Map<Integer, Builder> builderHolder = new HashMap<>(4);
+    private Map<Integer, DeduplicationService> serviceHolder = new HashMap<>(4);
+
+    public Builder selectBuilder(Integer key) {
+        return builderHolder.get(key);
+    }
+
+    public DeduplicationService selectService(Integer key) {
+        return serviceHolder.get(key);
+    }
+
+    public void putBuilder(Integer key, Builder builder) {
+        builderHolder.put(key, builder);
+    }
+
+    public void putService(Integer key, DeduplicationService service) {
+        serviceHolder.put(key, service);
+    }
+}

austin-handler/src/main/java/com/java3y/austin/service/deduplication/DeduplicationRuleService.java

@@ -2,13 +2,11 @@ package com.java3y.austin.service.deduplication;
 
 import com.ctrip.framework.apollo.Config;
 import com.ctrip.framework.apollo.spring.annotation.ApolloConfig;
-import com.google.common.collect.Lists;
 import com.java3y.austin.constant.AustinConstant;
 import com.java3y.austin.domain.DeduplicationParam;
 import com.java3y.austin.domain.TaskInfo;
-import com.java3y.austin.service.deduplication.build.BuilderFactory;
+import com.java3y.austin.enums.DeduplicationType;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.ApplicationContext;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -21,39 +19,27 @@ import java.util.List;
 @Service
 public class DeduplicationRuleService {
 
+    public static final String DEDUPLICATION_RULE_KEY = "deduplication";
 
-    private static final String SERVICE = "Service";
     @ApolloConfig("boss.austin")
     private Config config;
 
     @Autowired
-    private ApplicationContext applicationContext;
+    private DeduplicationHolder deduplicationHolder;
-    @Autowired
-    private BuilderFactory builderFactory;
-
-    //需要去重的服务
-    private static final List<String> DEDUPLICATION_LIST = Lists.newArrayList(DeduplicationConstants.CONTENT_DEDUPLICATION, DeduplicationConstants.FREQUENCY_DEDUPLICATION);
 
     public void duplication(TaskInfo taskInfo) {
-        // 配置样例：{"contentDeduplication":{"num":1,"time":300},"frequencyDeduplication":{"num":5}}
+        // 配置样例：{"deduplication_10":{"num":1,"time":300},"deduplication_20":{"num":5}}
-        String deduplication = config.getProperty(DeduplicationConstants.DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT);
+        String deduplicationConfig = config.getProperty(DEDUPLICATION_RULE_KEY, AustinConstant.APOLLO_DEFAULT_VALUE_JSON_OBJECT);
-        //去重
+
-        DEDUPLICATION_LIST.forEach(
+        // 去重
-                key -> {
+        List<Integer> deduplicationList = DeduplicationType.getDeduplicationList();
-                    DeduplicationParam deduplicationParam = builderFactory.select(key).build(deduplication, key);
+        for (Integer deduplicationType : deduplicationList) {
-                    if (deduplicationParam != null) {
+            DeduplicationParam deduplicationParam = deduplicationHolder.selectBuilder(deduplicationType).build(deduplicationConfig, taskInfo);
-                        deduplicationParam.setTaskInfo(taskInfo);
+            if (deduplicationParam != null) {
-                        DeduplicationService deduplicationService = findService(key + SERVICE);
+                deduplicationHolder.selectService(deduplicationType).deduplication(deduplicationParam);
-                        deduplicationService.deduplication(deduplicationParam);
+            }
-                    }
+        }
-                }
-        );
-
     }
 
-    private DeduplicationService findService(String beanName) {
-        return applicationContext.getBean(beanName, DeduplicationService.class);
-
-    }
 
 }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/AbstractDeduplicationBuilder.java

@@ -0,0 +1,40 @@
+package com.java3y.austin.service.deduplication.build;
+
+import com.alibaba.fastjson.JSONObject;
+import com.java3y.austin.domain.DeduplicationParam;
+import com.java3y.austin.domain.TaskInfo;
+import com.java3y.austin.service.deduplication.DeduplicationHolder;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.PostConstruct;
+
+/**
+ * @author 3y
+ * @date 2022/1/19
+ */
+public abstract class AbstractDeduplicationBuilder implements Builder {
+
+    protected Integer deduplicationType;
+
+    @Autowired
+    private DeduplicationHolder deduplicationHolder;
+
+    @PostConstruct
+    public void init() {
+        deduplicationHolder.putBuilder(deduplicationType, this);
+    }
+
+    public DeduplicationParam getParamsFromConfig(Integer key, String duplicationConfig, TaskInfo taskInfo) {
+        JSONObject object = JSONObject.parseObject(duplicationConfig);
+        if (object == null) {
+            return null;
+        }
+        DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(CONFIG_PRE + key), DeduplicationParam.class);
+        if (deduplicationParam == null) {
+            return null;
+        }
+        deduplicationParam.setTaskInfo(taskInfo);
+        return deduplicationParam;
+    }
+
+}

austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/Builder.java

@@ -1,6 +1,7 @@
 package com.java3y.austin.service.deduplication.build;
 
 import com.java3y.austin.domain.DeduplicationParam;
+import com.java3y.austin.domain.TaskInfo;
 
 /**
  * @author luohaojie
@@ -8,5 +9,14 @@ import com.java3y.austin.domain.DeduplicationParam;
  */
 public interface Builder {
 
-    DeduplicationParam build(String deduplication, String key);
+    String CONFIG_PRE = "deduplication_";
+
+    /**
+     * 根据配置构建去重参数
+     *
+     * @param deduplication
+     * @param taskInfo
+     * @return
+     */
+    DeduplicationParam build(String deduplication, TaskInfo taskInfo);
 }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/BuilderFactory.java

@@ -1,40 +0,0 @@
-package com.java3y.austin.service.deduplication.build;
-
-import com.java3y.austin.service.deduplication.DeduplicationConstants;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import javax.annotation.PostConstruct;
-import java.util.HashMap;
-import java.util.Map;
-
-
-/**
- * @author huskey
- * @date 2022/1/18
- */
-
-@Service
-public class BuilderFactory {
-
-
-    private Map<String, Builder> builderFactory = new HashMap<>(4);
-
-    @Autowired
-    private Builder contentDeduplicationBuilder;
-
-    @Autowired
-    private Builder frequencyDeduplicationBuilder;
-
-    @PostConstruct
-    public void init() {
-        builderFactory.put(DeduplicationConstants.CONTENT_DEDUPLICATION, contentDeduplicationBuilder);
-        builderFactory.put(DeduplicationConstants.FREQUENCY_DEDUPLICATION, frequencyDeduplicationBuilder);
-    }
-
-
-    public Builder select(String key) {
-        return builderFactory.get(key);
-    }
-
-}

austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/ContentDeduplicationBuilder.java

@@ -1,9 +1,9 @@
 package com.java3y.austin.service.deduplication.build;
 
-import com.alibaba.fastjson.JSONObject;
 import com.java3y.austin.domain.DeduplicationParam;
+import com.java3y.austin.domain.TaskInfo;
 import com.java3y.austin.enums.AnchorState;
-import com.java3y.austin.service.deduplication.build.Builder;
+import com.java3y.austin.enums.DeduplicationType;
 import org.springframework.stereotype.Service;
 
 
@@ -11,20 +11,20 @@ import org.springframework.stereotype.Service;
  * @author huskey
  * @date 2022/1/18
  */
-
 @Service
-public class ContentDeduplicationBuilder implements Builder {
+public class ContentDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder {
 
-    public DeduplicationParam build(String deduplication, String key) {
+    public ContentDeduplicationBuilder() {
-        JSONObject object = JSONObject.parseObject(deduplication);
+        deduplicationType = DeduplicationType.CONTENT.getCode();
-        if (object == null) {
+    }
-            return null;
+
-        }
+    @Override
-        DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class);
+    public DeduplicationParam build(String deduplication, TaskInfo taskInfo) {
-        if (deduplicationParam == null) {
+        DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo);
-            return null;
+        if (deduplication == null) {
+           return null;
         }
-        deduplicationParam.setAnchorState(AnchorState.RULE_DEDUPLICATION);
+        deduplicationParam.setAnchorState(AnchorState.CONTENT_DEDUPLICATION);
         return deduplicationParam;
 
     }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/build/FrequencyDeduplicationBuilder.java

@@ -1,10 +1,10 @@
 package com.java3y.austin.service.deduplication.build;
 
 import cn.hutool.core.date.DateUtil;
-import com.alibaba.fastjson.JSONObject;
 import com.java3y.austin.domain.DeduplicationParam;
+import com.java3y.austin.domain.TaskInfo;
 import com.java3y.austin.enums.AnchorState;
-import com.java3y.austin.service.deduplication.build.Builder;
+import com.java3y.austin.enums.DeduplicationType;
 import org.springframework.stereotype.Service;
 
 import java.util.Date;
@@ -15,15 +15,14 @@ import java.util.Date;
  */
 
 @Service
-public class FrequencyDeduplicationBuilder implements Builder {
+public class FrequencyDeduplicationBuilder extends AbstractDeduplicationBuilder implements Builder {
+    public FrequencyDeduplicationBuilder() {
+        deduplicationType = DeduplicationType.FREQUENCY.getCode();
+    }
 
     @Override
-    public DeduplicationParam build(String deduplication, String key) {
+    public DeduplicationParam build(String deduplication, TaskInfo taskInfo) {
-        JSONObject object = JSONObject.parseObject(deduplication);
+        DeduplicationParam deduplicationParam = getParamsFromConfig(deduplicationType, deduplication, taskInfo);
-        if (object == null) {
-            return null;
-        }
-        DeduplicationParam deduplicationParam = JSONObject.parseObject(object.getString(key), DeduplicationParam.class);
         if (deduplicationParam == null) {
             return null;
         }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/AbstractDeduplicationService.java

@@ -1,16 +1,17 @@
-package com.java3y.austin.service.deduplication;
+package com.java3y.austin.service.deduplication.service;
 
 import cn.hutool.core.collection.CollUtil;
 import com.java3y.austin.constant.AustinConstant;
 import com.java3y.austin.domain.AnchorInfo;
 import com.java3y.austin.domain.DeduplicationParam;
 import com.java3y.austin.domain.TaskInfo;
-import com.java3y.austin.enums.AnchorState;
+import com.java3y.austin.service.deduplication.DeduplicationHolder;
 import com.java3y.austin.utils.LogUtils;
 import com.java3y.austin.utils.RedisUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import javax.annotation.PostConstruct;
 import java.util.*;
 
 /**
@@ -20,10 +21,20 @@ import java.util.*;
  */
 @Slf4j
 public abstract class AbstractDeduplicationService implements DeduplicationService {
+    protected Integer deduplicationType;
+
+    @Autowired
+    private DeduplicationHolder deduplicationHolder;
+
+    @PostConstruct
+    private void init() {
+        deduplicationHolder.putService(deduplicationType, this);
+    }
 
     @Autowired
     private RedisUtils redisUtils;
 
+    @Override
     public void deduplication(DeduplicationParam param) {
         TaskInfo taskInfo = param.getTaskInfo();
         Set<String> filterReceiver = new HashSet<>(taskInfo.getReceiver().size());

austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/ContentDeduplicationService.java

@@ -1,8 +1,9 @@
-package com.java3y.austin.service.deduplication;
+package com.java3y.austin.service.deduplication.service;
 
 import cn.hutool.crypto.digest.DigestUtil;
 import com.alibaba.fastjson.JSON;
 import com.java3y.austin.domain.TaskInfo;
+import com.java3y.austin.enums.DeduplicationType;
 import org.springframework.stereotype.Service;
 
 /**
@@ -13,6 +14,10 @@ import org.springframework.stereotype.Service;
 @Service
 public class ContentDeduplicationService extends AbstractDeduplicationService {
 
+    public ContentDeduplicationService() {
+        deduplicationType = DeduplicationType.CONTENT.getCode();
+    }
+
     /**
      * 内容去重 构建key
      * <p>

austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/DeduplicationService.java

@@ -1,4 +1,4 @@
-package com.java3y.austin.service.deduplication;
+package com.java3y.austin.service.deduplication.service;
 
 import com.java3y.austin.domain.DeduplicationParam;
 
@@ -8,5 +8,9 @@ import com.java3y.austin.domain.DeduplicationParam;
  */
 public interface DeduplicationService {
 
+    /**
+     * 去重
+     * @param param
+     */
     void deduplication(DeduplicationParam param);
 }

austin-handler/src/main/java/com/java3y/austin/service/deduplication/service/FrequencyDeduplicationService.java

@@ -1,7 +1,8 @@
-package com.java3y.austin.service.deduplication;
+package com.java3y.austin.service.deduplication.service;
 
 import cn.hutool.core.util.StrUtil;
 import com.java3y.austin.domain.TaskInfo;
+import com.java3y.austin.enums.DeduplicationType;
 import org.springframework.stereotype.Service;
 
 /**
@@ -12,6 +13,10 @@ import org.springframework.stereotype.Service;
 @Service
 public class FrequencyDeduplicationService extends AbstractDeduplicationService {
 
+    public FrequencyDeduplicationService() {
+        deduplicationType = DeduplicationType.FREQUENCY.getCode();
+    }
+
     private static final String PREFIX = "FRE";
 
     /**