diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagicCopyWritingNetworkFeign.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagicCopyWritingNetworkFeign.java new file mode 100644 index 00000000..15580e42 --- /dev/null +++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagicCopyWritingNetworkFeign.java @@ -0,0 +1,22 @@ +package com.xjs.business.webmagic; + +import com.ruoyi.common.core.constant.ServiceNameConstants; +import com.ruoyi.common.core.domain.R; +import com.xjs.business.webmagic.factory.RemoteWebmagicCopyWritingNetworkFactory; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; + +/** + * 内部 调用 文案网 爬虫定时任务 + * + * @author xiejs + * @since 2022-02-17 + */ +@FeignClient(contextId = "remoteWebmagicCopyWritingNetworkFeign", + value = ServiceNameConstants.BUSINESS_WEBMAGIC_SERVICE, + fallbackFactory = RemoteWebmagicCopyWritingNetworkFactory.class) +public interface RemoteWebmagicCopyWritingNetworkFeign { + + @GetMapping("copyWritingNetwork/taskForPRC") + R copyWritingNetworkTaskForPRC(); +} diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java new file mode 100644 index 00000000..3ce40662 --- /dev/null +++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java @@ -0,0 +1,27 @@ +package com.xjs.business.webmagic.factory; + +import com.ruoyi.common.core.domain.R; +import com.xjs.business.webmagic.RemoteWebmagicCopyWritingNetworkFeign; +import lombok.extern.log4j.Log4j2; +import org.springframework.cloud.openfeign.FallbackFactory; +import org.springframework.stereotype.Component; + +/** + * 内部 调用 文案网 爬虫定时任务 降级 + * @author xiejs + * @since 2022-02-17 + */ +@Component +@Log4j2 +public class RemoteWebmagicCopyWritingNetworkFactory implements FallbackFactory { + @Override + public RemoteWebmagicCopyWritingNetworkFeign create(Throwable cause) { + return new RemoteWebmagicCopyWritingNetworkFeign() { + @Override + public R copyWritingNetworkTaskForPRC() { + log.error("文案网 爬虫定时任务 降级"); + return R.fail("降级处理"); + } + }; + } +} diff --git a/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/CopyWritingNetworkTask.java b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/CopyWritingNetworkTask.java new file mode 100644 index 00000000..24807f5d --- /dev/null +++ b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/CopyWritingNetworkTask.java @@ -0,0 +1,40 @@ +package com.xjs.job.task.webmagic; + +import cn.hutool.core.date.DateUtil; +import com.ruoyi.common.core.domain.R; +import com.xjs.business.webmagic.RemoteWebmagicCopyWritingNetworkFeign; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +/** + * 爬虫 文案网 定时任务 + * @author xiejs + * @since 2022-02-17 + */ +@Component("CopyWritingNetworkTask") +@Log4j2 +public class CopyWritingNetworkTask { + + @Resource + private RemoteWebmagicCopyWritingNetworkFeign remoteWebmagicCopyWritingNetworkFeign; + + /** + * 爬虫 文案网 定时任务执行 + */ + public void copyWritingNetwork() { + log.info("---------------爬虫-文案网定时任务Start-------------------"); + LocalDateTime localDateTime1 = DateUtil.date().toLocalDateTime(); + + R r = remoteWebmagicCopyWritingNetworkFeign.copyWritingNetworkTaskForPRC(); + + log.info("爬虫-文案网定时任务结果:code={},msg={},data={}",r.getCode(),r.getMsg(),r.getData()); + LocalDateTime localDateTime2 = DateUtil.date().toLocalDateTime(); + long between = ChronoUnit.MILLIS.between(localDateTime1, localDateTime2); + log.info("爬虫-文案网定时任务Job耗费时间:{}ms", between); + log.info("---------------爬虫-文案网定时任务end---------------------"); + } +} diff --git a/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/SinaTask.java b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/SinaTask.java index 4f831407..95c024e6 100644 --- a/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/SinaTask.java +++ b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/SinaTask.java @@ -22,7 +22,7 @@ public class SinaTask { private RemoteWebmagicSinaFeign remoteWebmagicSinaFeign; /** - * 任务执行 + * 爬虫 新浪新闻 定时任务执行 */ public void sinaNews() { log.info("---------------爬虫-新浪新闻定时任务Start-------------------"); diff --git a/ruoyi-ui/src/api/business/webmagic/copywritingnetwork/copyWritingNetwork.js b/ruoyi-ui/src/api/business/webmagic/copywritingnetwork/copyWritingNetwork.js new file mode 100644 index 00000000..b4a57a79 --- /dev/null +++ b/ruoyi-ui/src/api/business/webmagic/copywritingnetwork/copyWritingNetwork.js @@ -0,0 +1,18 @@ +import request from '@/utils/request' + +// 查询文案网列表 +export function listCopyWritingNetwork(query) { + return request({ + url: '/webmagic/copyWritingNetwork/list', + method: 'get', + params: query + }) +} + +// 删除文案网 +export function delCopyWritingNetwork(id) { + return request({ + url: '/webmagic/copyWritingNetwork/' + id, + method: 'delete' + }) +} diff --git a/ruoyi-ui/src/views/business/log/apilog/index.vue b/ruoyi-ui/src/views/business/log/apilog/index.vue index eec3784f..c59f2c79 100644 --- a/ruoyi-ui/src/views/business/log/apilog/index.vue +++ b/ruoyi-ui/src/views/business/log/apilog/index.vue @@ -129,7 +129,11 @@ - + + + @@ -201,11 +205,11 @@ + + diff --git a/ruoyi-ui/src/views/business/webmagic/sina/sinaNews/index.vue b/ruoyi-ui/src/views/business/webmagic/sina/sinaNews/index.vue index ef14dad5..497b5156 100644 --- a/ruoyi-ui/src/views/business/webmagic/sina/sinaNews/index.vue +++ b/ruoyi-ui/src/views/business/webmagic/sina/sinaNews/index.vue @@ -20,7 +20,7 @@ @keyup.enter.native="handleQuery" /> - + 0){ + if (args.length > 0) { String jsonString = JSON.toJSONString(args[0]); entity.setRequest(jsonString); } } entity.setMethod(apiLog.method()); + String response = null; if (Objects.nonNull(jsonResult)) { - entity.setResponse(jsonResult.toString()); + response = jsonResult.toString(); + entity.setResponse(response); } - if (e != null) { + if (e != null || StringUtils.isEmpty(response)) { entity.setIsSuccess(ReqConst.ERROR); } else { entity.setIsSuccess(ReqConst.SUCCESS); diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/util/HttpUtils.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/util/HttpUtils.java index bf454a00..85c56e36 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/util/HttpUtils.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/util/HttpUtils.java @@ -1,6 +1,7 @@ package com.xjs.common.util; import com.ruoyi.common.core.constant.HttpStatus; +import lombok.extern.log4j.Log4j2; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -22,6 +23,7 @@ import java.util.UUID; * @since 2022-02-15 */ @Component +@Log4j2 public class HttpUtils { private PoolingHttpClientConnectionManager cm; @@ -30,9 +32,9 @@ public class HttpUtils { this.cm = new PoolingHttpClientConnectionManager(); //设置最大连接数 - this.cm.setMaxTotal(100); + this.cm.setMaxTotal(1000); //设置每个主机最大连接数 - this.cm.setDefaultMaxPerRoute(10); + this.cm.setDefaultMaxPerRoute(100); } /** @@ -67,14 +69,14 @@ public class HttpUtils { } } } catch (IOException e) { - e.printStackTrace(); + log.error(e.getMessage()); } finally { //关闭response if (response != null) { try { response.close(); } catch (IOException e) { - e.printStackTrace(); + log.error(e.getMessage()); } } } @@ -122,14 +124,14 @@ public class HttpUtils { } } } catch (IOException e) { - e.printStackTrace(); + log.error(e.getMessage()); } finally { //关闭response if (response != null) { try { response.close(); } catch (IOException e) { - e.printStackTrace(); + log.error(e.getMessage()); } } } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/controller/CopyWritingNetworkController.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/controller/CopyWritingNetworkController.java new file mode 100644 index 00000000..95c380c7 --- /dev/null +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/controller/CopyWritingNetworkController.java @@ -0,0 +1,83 @@ +package com.xjs.copywritingNetwork.controller; + +import com.ruoyi.common.core.domain.R; +import com.ruoyi.common.core.utils.poi.ExcelUtil; +import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.common.core.web.page.TableDataInfo; +import com.ruoyi.common.log.annotation.Log; +import com.ruoyi.common.log.enums.BusinessType; +import com.ruoyi.common.security.annotation.RequiresPermissions; +import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork; +import com.xjs.copywritingNetwork.service.CopyWritingNetworkService; +import com.xjs.copywritingNetwork.task.CopyWritingNetworkTask; +import com.xjs.web.MyBaseController; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; + +/** + * 文案网controller + * + * @author xiejs + * @since 2022-02-17 + */ +@RestController +@RequestMapping("copyWritingNetwork") +@Api(tags = "爬虫模块-文案网") +public class CopyWritingNetworkController extends MyBaseController { + + @Autowired + private CopyWritingNetworkService copyWritingNetworkService; + @Autowired + private CopyWritingNetworkTask copyWritingNetworkTask; + + //----------------------远程rpc调用--------------------------- + @GetMapping("taskForPRC") + @ApiOperation("供定时任务服务RPC远程调用") + public R copyWritingNetworkTaskForPRC() { + copyWritingNetworkTask.reptileCopyWriting(); + return R.ok(); + } + + + //----------------------------代码生成----------------------------- + + /** + * 查询文案网列表 + */ + @RequiresPermissions("webmagic:copyWritingNetwork:list") + @GetMapping("/list") + public TableDataInfo list(CopyWritingNetwork copyWritingNetwork) { + startPage(); + List list = copyWritingNetworkService.selectCopyWritingNetworkList(copyWritingNetwork); + return getDataTable(list); + } + + /** + * 导出文案网列表 + */ + @RequiresPermissions("webmagic:copyWritingNetwork:export") + @Log(title = "文案网", businessType = BusinessType.EXPORT) + @PostMapping("/export") + public void export(HttpServletResponse response, CopyWritingNetwork copyWritingNetwork) { + List list = copyWritingNetworkService.selectCopyWritingNetworkList(copyWritingNetwork); + ExcelUtil util = new ExcelUtil<>(CopyWritingNetwork.class); + util.exportExcel(response, list, "文案网数据"); + } + + /** + * 删除文案网 + */ + @RequiresPermissions("webmagic:copyWritingNetwork:remove") + @Log(title = "文案网", businessType = BusinessType.DELETE) + @DeleteMapping("/{ids}") + public AjaxResult remove(@PathVariable Long[] ids) { + return toAjax(copyWritingNetworkService.deleteCopyWritingNetworkByIds(ids)); + } + + +} diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/mapper/CopyWritingNetworkMapper.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/mapper/CopyWritingNetworkMapper.java index a7df6688..f1752673 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/mapper/CopyWritingNetworkMapper.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/mapper/CopyWritingNetworkMapper.java @@ -3,6 +3,8 @@ package com.xjs.copywritingNetwork.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork; +import java.util.List; + /** * 文案网mapper * @author xiejs @@ -15,4 +17,32 @@ public interface CopyWritingNetworkMapper extends BaseMapper * @return int */ int deleteRepeatData(); + + + //------------------------------代码生成----------------------------------- + + /** + * 查询文案网列表 + * + * @param copyWritingNetwork 文案网 + * @return 文案网集合 + */ + List selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork); + + /** + * 删除文案网 + * + * @param id 文案网主键 + * @return 结果 + */ + public int deleteCopyWritingNetworkById(Long id); + + /** + * 批量删除文案网 + * + * @param ids 需要删除的数据主键集合 + * @return 结果 + */ + public int deleteCopyWritingNetworkByIds(Long[] ids); + } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/pojo/CopyWritingNetwork.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/pojo/CopyWritingNetwork.java index be30f688..3ce6c737 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/pojo/CopyWritingNetwork.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/pojo/CopyWritingNetwork.java @@ -32,4 +32,11 @@ public class CopyWritingNetwork implements Serializable { /** 创建时间 */ @TableField(fill = FieldFill.INSERT) private Date createTime; + + + /** + * 查询条件的结束时间 + */ + @TableField(exist = false) + private Date endCreateTime; } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/CopyWritingNetworkService.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/CopyWritingNetworkService.java index 14fa2fb0..fc56c58f 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/CopyWritingNetworkService.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/CopyWritingNetworkService.java @@ -3,15 +3,45 @@ package com.xjs.copywritingNetwork.service; import com.baomidou.mybatisplus.extension.service.IService; import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork; +import java.util.List; + /** * 文案网service接口 + * * @author xiejs * @since 2022-02-16 */ public interface CopyWritingNetworkService extends IService { /** * 删除重复数据 + * * @return int */ int deleteRepeatData(); + + //---------------------------代码生成-------------------------------- + + /** + * 查询文案网列表 + * + * @param copyWritingNetwork 文案网 + * @return 文案网集合 + */ + List selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork); + + /** + * 批量删除文案网 + * + * @param ids 需要删除的文案网主键集合 + * @return 结果 + */ + int deleteCopyWritingNetworkByIds(Long[] ids); + + /** + * 删除文案网信息 + * + * @param id 文案网主键 + * @return 结果 + */ + int deleteCopyWritingNetworkById(Long id); } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/impl/CopyWritingNetworkServiceImpl.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/impl/CopyWritingNetworkServiceImpl.java index 826ec40b..8283f33e 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/impl/CopyWritingNetworkServiceImpl.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/impl/CopyWritingNetworkServiceImpl.java @@ -7,6 +7,7 @@ import com.xjs.copywritingNetwork.service.CopyWritingNetworkService; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.util.List; /** * @author xiejs @@ -22,4 +23,39 @@ public class CopyWritingNetworkServiceImpl extends ServiceImpl selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork) { + return copyWritingNetworkMapper.selectCopyWritingNetworkList(copyWritingNetwork); + } + + /** + * 批量删除文案网 + * + * @param ids 需要删除的文案网主键 + * @return 结果 + */ + @Override + public int deleteCopyWritingNetworkByIds(Long[] ids) { + return copyWritingNetworkMapper.deleteCopyWritingNetworkByIds(ids); + } + + /** + * 删除文案网信息 + * + * @param id 文案网主键 + * @return 结果 + */ + @Override + public int deleteCopyWritingNetworkById(Long id) { + return copyWritingNetworkMapper.deleteCopyWritingNetworkById(id); + } } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/task/CopyWritingNetworkTask.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/task/CopyWritingNetworkTask.java index a5f60d02..e06f717d 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/task/CopyWritingNetworkTask.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/task/CopyWritingNetworkTask.java @@ -10,7 +10,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import java.util.ArrayList; @@ -18,6 +17,8 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import static com.xjs.consts.RegexConst.NUMBER_REGEX; + /** * 文案网爬虫任务 url:https://www.wenanwang.com/ * @@ -36,9 +37,8 @@ public class CopyWritingNetworkTask { public static final String URL = "https://www.wenanwang.com/"; - private static Pattern pattern = Pattern.compile("[0-9]*"); + private static final Pattern pattern = Pattern.compile(NUMBER_REGEX); - @Scheduled(fixedDelay = 1000 * 5) public void reptileCopyWriting() { try { String html = httpUtils.doGetHtml(URL); @@ -46,11 +46,9 @@ public class CopyWritingNetworkTask { Document document = Jsoup.parse(html); this.parseHtmlGetUrl(document); + } catch (Exception e) { - e.printStackTrace(); - }finally { - int i = copyWritingNetworkService.deleteRepeatData(); - log.info("删除文案网数据重复数："+i); + log.error(e.getMessage()); } } @@ -118,6 +116,9 @@ public class CopyWritingNetworkTask { copyWritingNetworkService.saveBatch(copyWritingNetworks, 20); + int i = copyWritingNetworkService.deleteRepeatData(); + log.info("删除文案网数据重复数："+i); + } diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java index ae139f04..9ce5ea41 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java @@ -30,17 +30,18 @@ public class SinaNewsTask { @Autowired private SinaNewsService sinaNewsService; + public static final String URL = "https://news.sina.com.cn/"; + public void reptileSinaNews() { try { - String url = "https://news.sina.com.cn/"; - String html = httpUtils.doGetHtml(url); + String html = httpUtils.doGetHtml(URL); Document document = Jsoup.parse(html); this.parse(document); } catch (Exception e) { - e.printStackTrace(); + log.error(e.getMessage()); } } diff --git a/xjs-business/xjs-business-webmagic/src/main/resources/mapper/webmagic/CopyWritingNetworkMapper.xml b/xjs-business/xjs-business-webmagic/src/main/resources/mapper/webmagic/CopyWritingNetworkMapper.xml index 804bbe60..cfc151ea 100644 --- a/xjs-business/xjs-business-webmagic/src/main/resources/mapper/webmagic/CopyWritingNetworkMapper.xml +++ b/xjs-business/xjs-business-webmagic/src/main/resources/mapper/webmagic/CopyWritingNetworkMapper.xml @@ -4,6 +4,31 @@ "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> + + + + + + + + + + + select id, type, theme, content, create_time from webmagic_copywriting_network + + + + + delete from webmagic_copywriting_network where id not in ( SELECT @@ -12,4 +37,15 @@ ( SELECT min( id ) AS min_id FROM webmagic_copywriting_network GROUP BY content ) AS t ) + + + delete from webmagic_copywriting_network where id = #{id} + + + + delete from webmagic_copywriting_network where id in + + #{id} + + \ No newline at end of file