feat: 敏感词汇处理(二):修改文件加载方式,正常加载更新词典

pull/68/head
xiaoxiamo 4 months ago
parent 80590e0a87
commit 537f8ffdf6

@ -1,9 +1,12 @@
package com.java3y.austin.handler.config; package com.java3y.austin.handler.config;
import com.java3y.austin.common.constant.CommonConstant;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.core.task.TaskExecutor; import org.springframework.core.task.TaskExecutor;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.ObjectUtils; import org.springframework.util.ObjectUtils;
@ -11,8 +14,8 @@ import org.springframework.util.ObjectUtils;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.io.InputStreamReader;
import java.nio.file.Paths; import java.nio.charset.StandardCharsets;
import java.util.Collections; import java.util.Collections;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -26,17 +29,21 @@ import java.util.stream.Collectors;
*/ */
@Slf4j @Slf4j
@Configuration @Configuration
@ConfigurationProperties(prefix = "austin.senswords")
public class SensitiveWordsConfig { public class SensitiveWordsConfig {
/** /**
* redis key * redis key
*/ */
public static final String SENS_WORDS_DICT = "SENS_WORDS_DICT"; public static final String SENS_WORDS_DICT = "SENS_WORDS_DICT";
/**
*
*/
private static final String FILE_PREFIX = "file:";
/** /**
* *
*/ */
private static final long UPDATE_TIME = 10 * 60 * 1000; private static final long UPDATE_TIME_SECONDS = 10 * 60;
/** /**
* *
*/ */
@ -45,15 +52,20 @@ public class SensitiveWordsConfig {
/** /**
* *
*/ */
@Value("${austin.senswords.filter.enabled}")
private boolean filterEnabled; private boolean filterEnabled;
/** /**
* *
*/ */
@Value("${austin.senswords.dict.path}")
private String dictPath; private String dictPath;
@Autowired @Autowired
private RedisTemplate<String, String> redisTemplate; private RedisTemplate<String, String> redisTemplate;
@Autowired @Autowired
private TaskExecutor taskExecutor; private TaskExecutor taskExecutor;
@Autowired
private ResourceLoader resourceLoader;
/** /**
* *
@ -65,9 +77,10 @@ public class SensitiveWordsConfig {
log.info("SensitiveWordConfig#loadSensitiveWords filterEnabled is false, return."); log.info("SensitiveWordConfig#loadSensitiveWords filterEnabled is false, return.");
return; return;
} }
// 加载并存储
loadSensWords(); loadSensWords();
storeSensWords(); storeSensWords();
// 开启定时任务每10分钟更新一次 // 定时更新
taskExecutor.execute(this::startScheduledUpdate); taskExecutor.execute(this::startScheduledUpdate);
} }
@ -79,7 +92,9 @@ public class SensitiveWordsConfig {
log.error("SensitiveWordConfig#loadSensWords dictPath is null or empty, skipping load."); log.error("SensitiveWordConfig#loadSensWords dictPath is null or empty, skipping load.");
return; return;
} }
try (BufferedReader reader = Files.newBufferedReader(Paths.get(dictPath))) { // 为直接路径,添加前缀
Resource resource = resourceLoader.getResource(dictPath.startsWith(CommonConstant.SLASH) ? FILE_PREFIX + dictPath : dictPath);
try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) {
sensitiveWords = reader.lines().map(String::trim).collect(Collectors.toSet()); sensitiveWords = reader.lines().map(String::trim).collect(Collectors.toSet());
} catch (IOException e) { } catch (IOException e) {
log.error("SensitiveWordConfig#loadSensitiveWords Failed to load sensitive words from {}: {}", log.error("SensitiveWordConfig#loadSensitiveWords Failed to load sensitive words from {}: {}",
@ -92,9 +107,13 @@ public class SensitiveWordsConfig {
* *
*/ */
private void storeSensWords() { private void storeSensWords() {
redisTemplate.delete(SENS_WORDS_DICT);
if (ObjectUtils.isEmpty(sensitiveWords)) {
return;
}
redisTemplate.opsForSet().add(SENS_WORDS_DICT, sensitiveWords.toArray(new String[0])); redisTemplate.opsForSet().add(SENS_WORDS_DICT, sensitiveWords.toArray(new String[0]));
log.info("SensitiveWordConfig#storeSensWords {} sensitive words stored in Redis under key '{}'.", log.debug("SensitiveWordConfig#storeSensWords sensitive words stored in Redis under key [{}], count [{}].",
sensitiveWords.size(), SENS_WORDS_DICT); SENS_WORDS_DICT, sensitiveWords.size());
} }
/** /**
@ -103,13 +122,11 @@ public class SensitiveWordsConfig {
private void startScheduledUpdate() { private void startScheduledUpdate() {
while (true) { while (true) {
try { try {
// 每10分钟更新一次 TimeUnit.SECONDS.sleep(UPDATE_TIME_SECONDS);
TimeUnit.SECONDS.sleep(UPDATE_TIME); log.debug("SensitiveWordConfig#startScheduledUpdate start update...");
log.info("SensitiveWordConfig#startScheduledUpdate start update...");
loadSensitiveWords(); loadSensitiveWords();
storeSensWords(); storeSensWords();
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("SensitiveWordConfig#startScheduledUpdate interrupted: {}", e.getMessage()); log.error("SensitiveWordConfig#startScheduledUpdate interrupted: {}", e.getMessage());
break; break;
} }

Loading…
Cancel
Save