diff --git a/pom.xml b/pom.xml index 12e3199a..ce8abf72 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 3.0.3 5.7.17 3.4.3.4 + 3.6 diff --git a/ruoyi-common/ruoyi-common-core/pom.xml b/ruoyi-common/ruoyi-common-core/pom.xml index c7067c45..e5c2b14b 100644 --- a/ruoyi-common/ruoyi-common-core/pom.xml +++ b/ruoyi-common/ruoyi-common-core/pom.xml @@ -125,11 +125,20 @@ swagger-annotations + + + cn.hutool hutool-all ${hutool.version} + + + org.projectlombok + lombok + + diff --git a/ruoyi-modules/ruoyi-file/pom.xml b/ruoyi-modules/ruoyi-file/pom.xml index 349cfc91..9463b20e 100644 --- a/ruoyi-modules/ruoyi-file/pom.xml +++ b/ruoyi-modules/ruoyi-file/pom.xml @@ -16,56 +16,63 @@ - - + + com.alibaba.cloud spring-cloud-starter-alibaba-nacos-discovery - + com.alibaba.cloud spring-cloud-starter-alibaba-nacos-config - + com.alibaba.cloud spring-cloud-starter-alibaba-sentinel - + org.springframework.boot spring-boot-starter-actuator - + com.github.tobato fastdfs-client - + io.minio minio ${minio.version} - + + + + commons-net + commons-net + ${commonsNet.version} + + com.ruoyi ruoyi-api-system - + com.ruoyi ruoyi-common-swagger - + @@ -84,5 +91,5 @@ - + \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/FtpProperties.java b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/FtpProperties.java new file mode 100644 index 00000000..74545d6a --- /dev/null +++ b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/FtpProperties.java @@ -0,0 +1,52 @@ +package com.ruoyi.file.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * ftp配置信息 + * @author xiejs + * @since 2022-01-24 + */ +@Component +@ConfigurationProperties(prefix = "ftp") +@Data +public class FtpProperties { + + /** + * 端口 + */ + private String port; + + /** + * 用户名 + */ + private String user; + + /** + * 密码 + */ + private String password; + + + /** + * ftp服务ip + */ + private String serverIp; + + /** + * ftp文件路径 + */ + private String path; + + /** + * 项目地址 + */ + private String domain; + + /** + * 前缀 + */ + private String prefix; +} diff --git a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/ResourcesConfig.java b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/ResourcesConfig.java index 3bca6686..4cb15d02 100644 --- a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/ResourcesConfig.java +++ b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/config/ResourcesConfig.java @@ -1,12 +1,13 @@ package com.ruoyi.file.config; -import java.io.File; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import java.io.File; + /** * 通用映射配置 * @@ -27,12 +28,29 @@ public class ResourcesConfig implements WebMvcConfigurer @Value("${file.prefix}") public String localFilePrefix; + + /** + * ftp文件路径 + */ + @Value("${ftp.path}") + private String ftpPath; + + /** + * ftp资源映射路径 前缀 + */ + @Value("${ftp.prefix}") + public String ftpFilePrefix; + @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler(ftpFilePrefix + "/**") + .addResourceLocations("file:" + ftpPath + File.separator); /** 本地文件上传路径 */ registry.addResourceHandler(localFilePrefix + "/**") .addResourceLocations("file:" + localFilePath + File.separator); + + } /** diff --git a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/FtpSysFileServiceImpl.java b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/FtpSysFileServiceImpl.java new file mode 100644 index 00000000..e3a631c6 --- /dev/null +++ b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/FtpSysFileServiceImpl.java @@ -0,0 +1,41 @@ +package com.ruoyi.file.service; + +import cn.hutool.core.date.DateUtil; +import com.ruoyi.file.config.FtpProperties; +import com.ruoyi.file.utils.FileUploadUtils; +import com.ruoyi.file.utils.FtpUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.springframework.web.multipart.MultipartFile; + +import java.util.UUID; + +/** + * FTP 文件存储 + * @author xiejs + * @since 2022-01-24 + */ +@Service +@Primary +public class FtpSysFileServiceImpl implements ISysFileService{ + + @Autowired + private FtpUtils ftpUtils; + @Autowired + private FtpProperties ftpProperties; + + @Override + public String uploadFile(MultipartFile file) throws Exception { + Assert.notNull(file, "file is null"); + String extension = FileUploadUtils.getExtension(file); + String url = ""; + String fileName = UUID.randomUUID() + "." + extension; + boolean uploadResult = ftpUtils.uploadFile(String.valueOf(DateUtil.thisYear()),fileName , file.getInputStream()); + if (uploadResult) { + url = ftpProperties.getDomain() + ftpProperties.getPrefix() +"/"+DateUtil.thisYear()+"/"+ fileName; + } + return url; + } +} diff --git a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/LocalSysFileServiceImpl.java b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/LocalSysFileServiceImpl.java index c0e20681..f35a2711 100644 --- a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/LocalSysFileServiceImpl.java +++ b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/service/LocalSysFileServiceImpl.java @@ -1,17 +1,15 @@ package com.ruoyi.file.service; +import com.ruoyi.file.utils.FileUploadUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import com.ruoyi.file.utils.FileUploadUtils; /** * 本地文件存储 * * @author ruoyi */ -@Primary @Service public class LocalSysFileServiceImpl implements ISysFileService { diff --git a/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/utils/FtpUtils.java b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/utils/FtpUtils.java new file mode 100644 index 00000000..4b5ccd8b --- /dev/null +++ b/ruoyi-modules/ruoyi-file/src/main/java/com/ruoyi/file/utils/FtpUtils.java @@ -0,0 +1,118 @@ +package com.ruoyi.file.utils; + +import cn.hutool.extra.ftp.Ftp; +import com.ruoyi.file.config.FtpProperties; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; + +/** + * ftp工具类 + * @author xjs + * @since 2022-01-24 16:49:40 + */ +@Component +public class FtpUtils { + + private Ftp ftp = null; + + private static final Logger logger = LoggerFactory.getLogger(FtpUtils.class); + public static String LOCAL_CHARSET = "GBK"; + + @Autowired + private FtpProperties ftpProperties; + + public Ftp instance() { + if (ftp != null) { + ftp.reconnectIfTimeout(); + } else { + String ftpServerIp = ftpProperties.getServerIp(); + String ftpServerAccount = ftpProperties.getUser(); + String ftpServerPassword = ftpProperties.getPassword(); + String ftpServerPort = ftpProperties.getPort(); + synchronized (this) { + if (StringUtils.isBlank(ftpServerPassword) || StringUtils.isBlank(ftpServerAccount)) { + ftp = new Ftp(ftpServerIp, Integer.parseInt(ftpServerPort)); + } else { + ftp = new Ftp(ftpServerIp, Integer.parseInt(ftpServerPort), ftpServerAccount, ftpServerPassword); + } + ftp.setBackToPwd(true); + logger.info("login ftp server with {}:{}@{}:{}", ftpServerIp, Integer.parseInt(ftpServerPort), ftpServerAccount, ftpServerPassword); + setClientCharacter(); + } + } + return ftp; + } + + + + private void setClientCharacter() { + FTPClient ftpClient = ftp.getClient(); + ftpClient.setControlKeepAliveTimeout(60); + try { + int command = ftpClient.sendCommand("OPTS UTF8", "ON"); + if (FTPReply.isPositiveCompletion(command)) { + LOCAL_CHARSET = "UTF-8"; + } + ftpClient.setControlEncoding(LOCAL_CHARSET); + } catch (Exception exception) { + throw new RuntimeException("获取ftp服务编码格式异常：", exception.getCause()); + } + } + + public boolean existFile(String filePath) { + return instance().existFile(filePath); + } + + public boolean delFile(String filePath) { + return instance().delFile(filePath); + } + + public boolean delDir(String dirPath) { + return instance().delDir(dirPath); + } + + public boolean mkDir(String dirPath) { + return instance().mkdir(dirPath); + } + + public List lsFiles(String dirPath) { + FTPFile[] ftpFiles = instance().lsFiles(dirPath); + return Arrays.asList(ftpFiles); + } + + public boolean uploadFile(String path, String fileName, File file) { + return instance().upload(path, fileName, file); + } + + public boolean uploadFile(String path, String fileName, InputStream inputStream) { + return instance().upload(path, fileName, inputStream); + } + + public void download(String path, String fileName, OutputStream outputStream) { + instance().download(path, fileName, outputStream); + } + + public boolean close() { + try { + if (ftp != null) { + ftp.close(); + } + return true; + } catch (Exception exception) { + logger.error("error close ftp: " + exception.getCause()); + return false; + } + } +} diff --git a/xjs-business/xjs-business-webmagic/pom.xml b/xjs-business/xjs-business-webmagic/pom.xml index f063335c..17a7c72f 100644 --- a/xjs-business/xjs-business-webmagic/pom.xml +++ b/xjs-business/xjs-business-webmagic/pom.xml @@ -8,6 +8,7 @@ 3.3.0 4.0.0 + 业务模块-爬虫模块 xjs-business-webmagic diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/handler/MiHoYoRepoPageProcessor.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/handler/MiHoYoRepoPageProcessor.java new file mode 100644 index 00000000..4b7f4cd1 --- /dev/null +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/handler/MiHoYoRepoPageProcessor.java @@ -0,0 +1,30 @@ +package com.xjs.handler; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.processor.PageProcessor; + +/** + * 米哈游社区爬取资源 + * @author xiejs + * @since 2022-01-24 + */ +public class MiHoYoRepoPageProcessor implements PageProcessor { + + + private Site site = Site.me().setRetryTimes(3).setSleepTime(100); + + + @Override + public void process(Page page) { + + } + + @Override + public Site getSite() { + return site; + } + + + +}