feat: 敏感词汇处理(二):修改文件加载方式,正常加载更新词典

pull/68/head
xiaoxiamo 4 months ago
parent 80590e0a87
commit 537f8ffdf6

@ -1,9 +1,12 @@
package com.java3y.austin.handler.config;
import com.java3y.austin.common.constant.CommonConstant;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.ObjectUtils;
@ -11,8 +14,8 @@ import org.springframework.util.ObjectUtils;
import javax.annotation.PostConstruct;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -26,17 +29,21 @@ import java.util.stream.Collectors;
*/
@Slf4j
@Configuration
@ConfigurationProperties(prefix = "austin.senswords")
public class SensitiveWordsConfig {
/**
* redis key
*/
public static final String SENS_WORDS_DICT = "SENS_WORDS_DICT";
/**
*
*/
private static final String FILE_PREFIX = "file:";
/**
*
*/
private static final long UPDATE_TIME = 10 * 60 * 1000;
private static final long UPDATE_TIME_SECONDS = 10 * 60;
/**
*
*/
@ -45,15 +52,20 @@ public class SensitiveWordsConfig {
/**
*
*/
@Value("${austin.senswords.filter.enabled}")
private boolean filterEnabled;
/**
*
*/
@Value("${austin.senswords.dict.path}")
private String dictPath;
@Autowired
private RedisTemplate<String, String> redisTemplate;
@Autowired
private TaskExecutor taskExecutor;
@Autowired
private ResourceLoader resourceLoader;
/**
*
@ -65,9 +77,10 @@ public class SensitiveWordsConfig {
log.info("SensitiveWordConfig#loadSensitiveWords filterEnabled is false, return.");
return;
}
// 加载并存储
loadSensWords();
storeSensWords();
// 开启定时任务每10分钟更新一次
// 定时更新
taskExecutor.execute(this::startScheduledUpdate);
}
@ -79,7 +92,9 @@ public class SensitiveWordsConfig {
log.error("SensitiveWordConfig#loadSensWords dictPath is null or empty, skipping load.");
return;
}
try (BufferedReader reader = Files.newBufferedReader(Paths.get(dictPath))) {
// 为直接路径,添加前缀
Resource resource = resourceLoader.getResource(dictPath.startsWith(CommonConstant.SLASH) ? FILE_PREFIX + dictPath : dictPath);
try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
sensitiveWords = reader.lines().map(String::trim).collect(Collectors.toSet());
} catch (IOException e) {
log.error("SensitiveWordConfig#loadSensitiveWords Failed to load sensitive words from {}: {}",
@ -92,9 +107,13 @@ public class SensitiveWordsConfig {
*
*/
private void storeSensWords() {
redisTemplate.delete(SENS_WORDS_DICT);
if (ObjectUtils.isEmpty(sensitiveWords)) {
return;
}
redisTemplate.opsForSet().add(SENS_WORDS_DICT, sensitiveWords.toArray(new String[0]));
log.info("SensitiveWordConfig#storeSensWords {} sensitive words stored in Redis under key '{}'.",
sensitiveWords.size(), SENS_WORDS_DICT);
log.debug("SensitiveWordConfig#storeSensWords sensitive words stored in Redis under key [{}], count [{}].",
SENS_WORDS_DICT, sensitiveWords.size());
}
/**
@ -103,13 +122,11 @@ public class SensitiveWordsConfig {
private void startScheduledUpdate() {
while (true) {
try {
// 每10分钟更新一次
TimeUnit.SECONDS.sleep(UPDATE_TIME);
log.info("SensitiveWordConfig#startScheduledUpdate start update...");
TimeUnit.SECONDS.sleep(UPDATE_TIME_SECONDS);
log.debug("SensitiveWordConfig#startScheduledUpdate start update...");
loadSensitiveWords();
storeSensWords();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("SensitiveWordConfig#startScheduledUpdate interrupted: {}", e.getMessage());
break;
}

Loading…
Cancel
Save