|
|
@ -1,9 +1,12 @@
|
|
|
|
package com.java3y.austin.handler.config;
|
|
|
|
package com.java3y.austin.handler.config;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import com.java3y.austin.common.constant.CommonConstant;
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
import org.springframework.context.annotation.Configuration;
|
|
|
|
import org.springframework.context.annotation.Configuration;
|
|
|
|
|
|
|
|
import org.springframework.core.io.Resource;
|
|
|
|
|
|
|
|
import org.springframework.core.io.ResourceLoader;
|
|
|
|
import org.springframework.core.task.TaskExecutor;
|
|
|
|
import org.springframework.core.task.TaskExecutor;
|
|
|
|
import org.springframework.data.redis.core.RedisTemplate;
|
|
|
|
import org.springframework.data.redis.core.RedisTemplate;
|
|
|
|
import org.springframework.util.ObjectUtils;
|
|
|
|
import org.springframework.util.ObjectUtils;
|
|
|
@ -11,8 +14,8 @@ import org.springframework.util.ObjectUtils;
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.nio.file.Files;
|
|
|
|
import java.io.InputStreamReader;
|
|
|
|
import java.nio.file.Paths;
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
@ -26,17 +29,21 @@ import java.util.stream.Collectors;
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
@Slf4j
|
|
|
|
@Slf4j
|
|
|
|
@Configuration
|
|
|
|
@Configuration
|
|
|
|
@ConfigurationProperties(prefix = "austin.senswords")
|
|
|
|
|
|
|
|
public class SensitiveWordsConfig {
|
|
|
|
public class SensitiveWordsConfig {
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 敏感词字典redis key
|
|
|
|
* 敏感词字典redis key
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
public static final String SENS_WORDS_DICT = "SENS_WORDS_DICT";
|
|
|
|
public static final String SENS_WORDS_DICT = "SENS_WORDS_DICT";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
|
|
* 文件前缀
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
private static final String FILE_PREFIX = "file:";
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 更新时间
|
|
|
|
* 更新时间
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
private static final long UPDATE_TIME = 10 * 60 * 1000;
|
|
|
|
private static final long UPDATE_TIME_SECONDS = 10 * 60;
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 敏感词字典
|
|
|
|
* 敏感词字典
|
|
|
|
*/
|
|
|
|
*/
|
|
|
@ -45,15 +52,20 @@ public class SensitiveWordsConfig {
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 是否开启敏感词过滤
|
|
|
|
* 是否开启敏感词过滤
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Value("${austin.senswords.filter.enabled}")
|
|
|
|
private boolean filterEnabled;
|
|
|
|
private boolean filterEnabled;
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 字典路径
|
|
|
|
* 字典路径
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Value("${austin.senswords.dict.path}")
|
|
|
|
private String dictPath;
|
|
|
|
private String dictPath;
|
|
|
|
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
@Autowired
|
|
|
|
private RedisTemplate<String, String> redisTemplate;
|
|
|
|
private RedisTemplate<String, String> redisTemplate;
|
|
|
|
@Autowired
|
|
|
|
@Autowired
|
|
|
|
private TaskExecutor taskExecutor;
|
|
|
|
private TaskExecutor taskExecutor;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
|
|
|
|
private ResourceLoader resourceLoader;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* 初始化敏感词字典
|
|
|
|
* 初始化敏感词字典
|
|
|
@ -65,9 +77,10 @@ public class SensitiveWordsConfig {
|
|
|
|
log.info("SensitiveWordConfig#loadSensitiveWords filterEnabled is false, return.");
|
|
|
|
log.info("SensitiveWordConfig#loadSensitiveWords filterEnabled is false, return.");
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 加载并存储
|
|
|
|
loadSensWords();
|
|
|
|
loadSensWords();
|
|
|
|
storeSensWords();
|
|
|
|
storeSensWords();
|
|
|
|
// 开启定时任务,每10分钟更新一次
|
|
|
|
// 定时更新
|
|
|
|
taskExecutor.execute(this::startScheduledUpdate);
|
|
|
|
taskExecutor.execute(this::startScheduledUpdate);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -79,7 +92,9 @@ public class SensitiveWordsConfig {
|
|
|
|
log.error("SensitiveWordConfig#loadSensWords dictPath is null or empty, skipping load.");
|
|
|
|
log.error("SensitiveWordConfig#loadSensWords dictPath is null or empty, skipping load.");
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
try (BufferedReader reader = Files.newBufferedReader(Paths.get(dictPath))) {
|
|
|
|
// 为直接路径,添加前缀
|
|
|
|
|
|
|
|
Resource resource = resourceLoader.getResource(dictPath.startsWith(CommonConstant.SLASH) ? FILE_PREFIX + dictPath : dictPath);
|
|
|
|
|
|
|
|
try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
|
|
|
|
sensitiveWords = reader.lines().map(String::trim).collect(Collectors.toSet());
|
|
|
|
sensitiveWords = reader.lines().map(String::trim).collect(Collectors.toSet());
|
|
|
|
} catch (IOException e) {
|
|
|
|
} catch (IOException e) {
|
|
|
|
log.error("SensitiveWordConfig#loadSensitiveWords Failed to load sensitive words from {}: {}",
|
|
|
|
log.error("SensitiveWordConfig#loadSensitiveWords Failed to load sensitive words from {}: {}",
|
|
|
@ -92,9 +107,13 @@ public class SensitiveWordsConfig {
|
|
|
|
* 存储敏感词字典
|
|
|
|
* 存储敏感词字典
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
private void storeSensWords() {
|
|
|
|
private void storeSensWords() {
|
|
|
|
|
|
|
|
redisTemplate.delete(SENS_WORDS_DICT);
|
|
|
|
|
|
|
|
if (ObjectUtils.isEmpty(sensitiveWords)) {
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
}
|
|
|
|
redisTemplate.opsForSet().add(SENS_WORDS_DICT, sensitiveWords.toArray(new String[0]));
|
|
|
|
redisTemplate.opsForSet().add(SENS_WORDS_DICT, sensitiveWords.toArray(new String[0]));
|
|
|
|
log.info("SensitiveWordConfig#storeSensWords {} sensitive words stored in Redis under key '{}'.",
|
|
|
|
log.debug("SensitiveWordConfig#storeSensWords sensitive words stored in Redis under key [{}], count [{}].",
|
|
|
|
sensitiveWords.size(), SENS_WORDS_DICT);
|
|
|
|
SENS_WORDS_DICT, sensitiveWords.size());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
@ -103,13 +122,11 @@ public class SensitiveWordsConfig {
|
|
|
|
private void startScheduledUpdate() {
|
|
|
|
private void startScheduledUpdate() {
|
|
|
|
while (true) {
|
|
|
|
while (true) {
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
// 每10分钟更新一次
|
|
|
|
TimeUnit.SECONDS.sleep(UPDATE_TIME_SECONDS);
|
|
|
|
TimeUnit.SECONDS.sleep(UPDATE_TIME);
|
|
|
|
log.debug("SensitiveWordConfig#startScheduledUpdate start update...");
|
|
|
|
log.info("SensitiveWordConfig#startScheduledUpdate start update...");
|
|
|
|
|
|
|
|
loadSensitiveWords();
|
|
|
|
loadSensitiveWords();
|
|
|
|
storeSensWords();
|
|
|
|
storeSensWords();
|
|
|
|
} catch (InterruptedException e) {
|
|
|
|
} catch (InterruptedException e) {
|
|
|
|
Thread.currentThread().interrupt();
|
|
|
|
|
|
|
|
log.error("SensitiveWordConfig#startScheduledUpdate interrupted: {}", e.getMessage());
|
|
|
|
log.error("SensitiveWordConfig#startScheduledUpdate interrupted: {}", e.getMessage());
|
|
|
|
break;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|