From 2f75b19ccd37301147fcdb344e0f9c8e7e281042 Mon Sep 17 00:00:00 2001 From: xjs <1294405880@qq.com> Date: Thu, 17 Feb 2022 21:28:23 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E7=88=AC=E8=99=AB=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=89=8D=E7=AB=AF=E9=A1=B5=E9=9D=A2=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=202=E3=80=81api=E6=97=A5=E5=BF=97=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=9D=A1=E4=BB=B6bug=E4=BF=AE=E6=94=B9=20?= =?UTF-8?q?=EF=BC=8Cinput=E6=A1=86=E7=9A=84model=E5=80=BC=E9=9C=80?= =?UTF-8?q?=E8=A6=81=E6=9C=89=E5=80=BC=EF=BC=88null=E4=B9=9F=E8=A1=8C?= =?UTF-8?q?=EF=BC=89,=E4=B8=8D=E7=84=B6=E6=97=A0=E6=B3=95=E8=BE=93?= =?UTF-8?q?=E5=85=A5=EF=BC=8C=E5=B7=A8=E5=9D=91=203=E3=80=81=E5=85=B6?= =?UTF-8?q?=E4=BB=96=E5=89=8D=E7=AB=AF=E9=A1=B5=E9=9D=A2=E7=BB=86=E5=BE=AE?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ruoyi-ui/src/api/business/log/reptilelog.js | 18 ++ .../src/views/business/log/apilog/index.vue | 55 +--- .../views/business/log/reptilelog/index.vue | 260 ++++++++++++++++++ .../controller/WebmagicLogController.java | 58 +++- .../xjs/reptileLog/domain/WebmagicLog.java | 15 + .../reptileLog/mapper/WebmagicLogMapper.java | 28 ++ .../service/WebmagicLogService.java | 28 ++ .../service/impl/WebmagicLogServiceImpl.java | 42 +++ .../mapper/log/WebmagicLogMapper.xml | 52 ++++ .../java/com/xjs/sina/task/SinaNewsTask.java | 57 ++-- 10 files changed, 535 insertions(+), 78 deletions(-) create mode 100644 ruoyi-ui/src/api/business/log/reptilelog.js create mode 100644 ruoyi-ui/src/views/business/log/reptilelog/index.vue create mode 100644 xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml diff --git a/ruoyi-ui/src/api/business/log/reptilelog.js b/ruoyi-ui/src/api/business/log/reptilelog.js new file mode 100644 index 00000000..bd6e0c3f --- /dev/null +++ b/ruoyi-ui/src/api/business/log/reptilelog.js @@ -0,0 +1,18 @@ +import request from '@/utils/request' + +// 查询爬虫日志列表 +export function listWebmagicLog(query) { + return request({ + url: '/log/reptileLog/list', + method: 'get', + params: query + }) +} + +// 删除爬虫日志 +export function delWebmagicLog(id) { + return request({ + url: '/log/reptileLog/' + id, + method: 'delete' + }) +} diff --git a/ruoyi-ui/src/views/business/log/apilog/index.vue b/ruoyi-ui/src/views/business/log/apilog/index.vue index c59f2c79..33a768db 100644 --- a/ruoyi-ui/src/views/business/log/apilog/index.vue +++ b/ruoyi-ui/src/views/business/log/apilog/index.vue @@ -131,7 +131,7 @@ @@ -205,7 +205,7 @@ diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java index 76ff5bc0..aa528c8d 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java @@ -1,38 +1,82 @@ package com.xjs.reptileLog.controller; import com.ruoyi.common.core.domain.R; +import com.ruoyi.common.core.utils.poi.ExcelUtil; +import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.common.core.web.page.TableDataInfo; +import com.ruoyi.common.log.annotation.Log; +import com.ruoyi.common.log.enums.BusinessType; +import com.ruoyi.common.security.annotation.RequiresPermissions; import com.xjs.reptileLog.domain.WebmagicLog; import com.xjs.reptileLog.service.WebmagicLogService; +import com.xjs.web.MyBaseController; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; /** * 爬虫日志控制器 + * * @author xiejs * @since 2022-02-17 */ @RestController @RequestMapping("reptileLog") @Api(tags = "业务模块-爬虫日志") -public class WebmagicLogController { +public class WebmagicLogController extends MyBaseController { @Autowired private WebmagicLogService webmagicLogService; - //-----------------------内部调用rpc------------------------ @PostMapping("saveForPRC") @ApiOperation("供AOP切面RPC远程调用") public R saveReptileLog(@RequestBody WebmagicLog webmagicLog) { boolean save = webmagicLogService.save(webmagicLog); - return save?R.ok():R.fail(); + return save ? R.ok() : R.fail(); + } + + + //------------------代码生成---------------------------- + + /** + * 查询爬虫日志列表 + */ + @RequiresPermissions("log:webmagicLog:list") + @GetMapping("/list") + public TableDataInfo list(WebmagicLog webmagicLog) { + startPage(); + List list = webmagicLogService.selectWebmagicLogList(webmagicLog); + return getDataTable(list); } + /** + * 导出爬虫日志列表 + */ + @RequiresPermissions("log:webmagicLog:export") + @Log(title = "爬虫日志", businessType = BusinessType.EXPORT) + @PostMapping("/export") + public void export(HttpServletResponse response, WebmagicLog webmagicLog) { + List list = webmagicLogService.selectWebmagicLogList(webmagicLog); + ExcelUtil util = new ExcelUtil<>(WebmagicLog.class); + util.exportExcel(response, list, "爬虫日志数据"); + } + + /** + * 删除爬虫日志 + */ + @RequiresPermissions("log:webmagicLog:remove") + @Log(title = "爬虫日志", businessType = BusinessType.DELETE) + @DeleteMapping("/{ids}") + public AjaxResult remove(@PathVariable Long[] ids) { + return toAjax(webmagicLogService.deleteWebmagicLogByIds(ids)); + } + + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java index 164961aa..fce709e4 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java @@ -47,4 +47,19 @@ public class WebmagicLog implements Serializable { @TableField(fill = FieldFill.INSERT) private Date createTime; + @TableField(exist = false) + private Date endCreateTime; + + /** + * 查询条件：耗费时间Start + */ + @TableField(exist = false) + private Long beginRequestTime; + + /** + * 查询条件：耗费时间End + */ + @TableField(exist = false) + private Long endRequestTime; + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java index a083ca3b..ff980f24 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java @@ -3,6 +3,8 @@ package com.xjs.reptileLog.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.xjs.reptileLog.domain.WebmagicLog; +import java.util.List; + /** * WebmagicLog mapper * @author xiejs @@ -10,4 +12,30 @@ import com.xjs.reptileLog.domain.WebmagicLog; */ public interface WebmagicLogMapper extends BaseMapper { + + //--------------------代码生成----------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志集合 + */ + public List selectWebmagicLogList(WebmagicLog webmagicLog); + + /** + * 删除爬虫日志 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + public int deleteWebmagicLogById(Long id); + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的数据主键集合 + * @return 结果 + */ + public int deleteWebmagicLogByIds(Long[] ids); } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java index a262f108..0ca46bc5 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java @@ -3,6 +3,8 @@ package com.xjs.reptileLog.service; import com.baomidou.mybatisplus.extension.service.IService; import com.xjs.reptileLog.domain.WebmagicLog; +import java.util.List; + /** * 爬虫日志 Service接口 * @author xiejs @@ -10,4 +12,30 @@ import com.xjs.reptileLog.domain.WebmagicLog; */ public interface WebmagicLogService extends IService { + //------------------------代码生成------------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志集合 + */ + public List selectWebmagicLogList(WebmagicLog webmagicLog); + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的爬虫日志主键集合 + * @return 结果 + */ + public int deleteWebmagicLogByIds(Long[] ids); + + /** + * 删除爬虫日志信息 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + public int deleteWebmagicLogById(Long id); + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java index 2515f945..58cce084 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java @@ -6,6 +6,9 @@ import com.xjs.reptileLog.mapper.WebmagicLogMapper; import com.xjs.reptileLog.service.WebmagicLogService; import org.springframework.stereotype.Service; +import javax.annotation.Resource; +import java.util.List; + /** * @author xiejs * @since 2022-02-17 @@ -13,4 +16,43 @@ import org.springframework.stereotype.Service; @Service public class WebmagicLogServiceImpl extends ServiceImpl implements WebmagicLogService { + @Resource + private WebmagicLogMapper webmagicLogMapper; + + + //------------------------代码生成----------------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志 + */ + @Override + public List selectWebmagicLogList(WebmagicLog webmagicLog) { + return webmagicLogMapper.selectWebmagicLogList(webmagicLog); + } + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的爬虫日志主键 + * @return 结果 + */ + @Override + public int deleteWebmagicLogByIds(Long[] ids) { + return webmagicLogMapper.deleteWebmagicLogByIds(ids); + } + + /** + * 删除爬虫日志信息 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + @Override + public int deleteWebmagicLogById(Long id) { + return webmagicLogMapper.deleteWebmagicLogById(id); + } + } diff --git a/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml b/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml new file mode 100644 index 00000000..5f341164 --- /dev/null +++ b/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + select id, name, url, request_time, create_time, complex_rate, status + from webmagic_log + + + + + + delete + from webmagic_log + where id = #{id} + + + + delete from webmagic_log where id in + + #{id} + + + + + \ No newline at end of file diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java index 2b3104f6..3efcfc7d 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; /** * 新浪新闻爬虫任务 + * * @author xiejs * @since 2022-02-15 */ @@ -44,7 +45,7 @@ public class SinaNewsTask { Document document = Jsoup.parse(html); - count = this.parse(document,count); + count = this.parse(document, count); } catch (Exception e) { log.error(e.getMessage()); } @@ -55,36 +56,32 @@ public class SinaNewsTask { * 解析dom * * @param document dom - * @param count 循环次数 + * @param count 循环次数 */ - private Long parse(Document document,Long count) { - try { - //获取子链接 - Elements nav_mod_1 = document.getElementsByClass("nav-mod-1"); - Elements link = nav_mod_1.select("ul > li > a"); - List> hrefList = link.stream().map(a -> { - String href = a.attr("href"); - String text = a.text(); - Map map = new HashMap<>(); - map.put(text, href); - return map; - }).collect(Collectors.toList()); - hrefList.removeIf(s -> s.containsKey("javascript:;")); - - for (Map map : hrefList) { - Set> entrySet = map.entrySet(); - for (Map.Entry entry : entrySet) { - String html = httpUtils.doGetHtml(entry.getValue()); - Document docChild = Jsoup.parse(html); - - //计数 - count++; - - count =this.parseChile(docChild, entry.getKey(),count); - } + private Long parse(Document document, Long count) { + //获取子链接 + Elements nav_mod_1 = document.getElementsByClass("nav-mod-1"); + Elements link = nav_mod_1.select("ul > li > a"); + List> hrefList = link.stream().map(a -> { + String href = a.attr("href"); + String text = a.text(); + Map map = new HashMap<>(); + map.put(text, href); + return map; + }).collect(Collectors.toList()); + hrefList.removeIf(s -> s.containsKey("javascript:;")); + + for (Map map : hrefList) { + Set> entrySet = map.entrySet(); + for (Map.Entry entry : entrySet) { + String html = httpUtils.doGetHtml(entry.getValue()); + Document docChild = Jsoup.parse(html); + + Long newCount = this.parseChile(docChild, entry.getKey(), count); + + count = count + newCount; } - } catch (Exception e) { - log.error(e.getMessage()); + } return count; } @@ -95,7 +92,7 @@ public class SinaNewsTask { * @param docChild 子 * @param key key */ - private Long parseChile(Document docChild, String key,Long count) { + private Long parseChile(Document docChild, String key, Long count) { try { Elements a = docChild.getElementsByTag("a"); ArrayList link = new ArrayList<>();