1、36壁纸网爬虫持久到数据库代码优化

ruoyi-ui/src/views/business/english/word/index.vue

@@ -397,9 +397,9 @@ export default {
         this.queryParams.endCreateTime = this.daterangeCreateTime[1];
       }
       listWord(this.queryParams).then(response => {
+        this.loading = false;
         this.wordList = response.rows;
         this.total = response.total;
-        this.loading = false;
       });
     },
     // 取消按钮

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/task/_36wallpaperTask.java

@@ -1,5 +1,6 @@
 package com.xjs._36wallpaper.task;
 
+import com.xjs._36wallpaper.service._36wallpaperService;
 import com.xjs._36wallpaper.webmagic._36wallpaperProcessor;
 import com.xjs.annotation.ReptileLog;
 import lombok.extern.log4j.Log4j2;
@@ -20,6 +21,9 @@ public class _36wallpaperTask {
     @Autowired
     private _36wallpaperProcessor wallpaperProcessor;
 
+    @Autowired
+    private _36wallpaperService wallpaperService;
+
 
     /**
      * 提供定时任务调取
@@ -27,7 +31,12 @@ public class _36wallpaperTask {
      */
     @ReptileLog(name = "36壁纸网", url = _36_WALLPAPER_URL)
     public Long reptileWallpaper() {
-        return wallpaperProcessor.run();
+        Long run = wallpaperProcessor.run();
+        //删除重复数据
+        int count = wallpaperService.deleteRepeatData();
+        log.info("36壁纸删除重复数据数：" + count);
+
+        return run;
     }
 
 }

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperPipeline.java

@@ -0,0 +1,37 @@
+package com.xjs._36wallpaper.webmagic;
+
+import cn.hutool.core.collection.CollUtil;
+import com.xjs._36wallpaper.pojo._36wallpaper;
+import com.xjs._36wallpaper.service._36wallpaperService;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.ResultItems;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.pipeline.Pipeline;
+
+import java.util.List;
+
+/**
+ * webmagic持久化保存
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@Component
+@Log4j2
+public class _36wallpaperPipeline implements Pipeline {
+
+    @Autowired
+    private  _36wallpaperService wallpaperService;
+
+    @Override
+    public void process(ResultItems resultItems, Task task) {
+
+        List<_36wallpaper> wallpaperData = resultItems.get("_36wallpaperData");
+
+        if (CollUtil.isNotEmpty(wallpaperData)) {
+            wallpaperService.saveBatch(wallpaperData, 25);
+        }
+
+    }
+}

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java

@@ -2,7 +2,6 @@ package com.xjs._36wallpaper.webmagic;
 
 import com.ruoyi.common.redis.service.RedisService;
 import com.xjs._36wallpaper.pojo._36wallpaper;
-import com.xjs._36wallpaper.service._36wallpaperService;
 import lombok.extern.log4j.Log4j2;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -44,6 +43,9 @@ public class _36wallpaperProcessor implements PageProcessor {
     private static final String headerValue = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
 
 
+    @Autowired
+    private _36wallpaperPipeline wallpaperPipeline;
+
     private static RedisService redisService;
 
     @Autowired
@@ -52,12 +54,12 @@ public class _36wallpaperProcessor implements PageProcessor {
     }
 
 
-    private static _36wallpaperService wallpaperService;
+    /*private static _36wallpaperService wallpaperService;
 
     @Autowired
     public void setWallpaperService(_36wallpaperService wallpaperService) {
         _36wallpaperProcessor.wallpaperService = wallpaperService;
-    }
+    }*/
 
     /**
      * 图片保存到磁盘的路径
@@ -159,8 +161,11 @@ public class _36wallpaperProcessor implements PageProcessor {
                 }
             }
 
-            //持久化
+            //持久化  --使用Pipeline实现持久化了
-            wallpaperService.saveBatch(wallpapers, 25);
+            //wallpaperService.saveBatch(wallpapers, 25);
+
+            //暂时保存到内存中，后续实现Pipeline接口保存到数据库
+            page.putField("_36wallpaperData",wallpapers);
 
             //循环次数存入redis中
             Integer count = redisService.getCacheObject(REPTILE_COUNT);
@@ -191,11 +196,10 @@ public class _36wallpaperProcessor implements PageProcessor {
     public Long run() {
         Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(20)
                 .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(110000)))
+                .addPipeline(wallpaperPipeline)
                 .run();
 
-        //删除重复数据
+
-        int count = wallpaperService.deleteRepeatData();
-        log.info("36壁纸删除重复数据数：" + count);
 
         //从redis中获取循环次数
         Integer cache = redisService.getCacheObject(REPTILE_COUNT);

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java

@@ -112,7 +112,7 @@ public class reptileLogAspect {
      * 持久化保存数据
      */
     private void saveData(WebmagicLog webmagicLog) {
-        if (webmagicLog.getComplexRate() != null && webmagicLog.getComplexRate() == 0L) {
+        if (webmagicLog.getComplexRate() != null && webmagicLog.getComplexRate() == 0L || webmagicLog.getComplexRate() == 1L) {
             webmagicLog.setStatus(ERROR);
         } else {
             webmagicLog.setStatus(SUCCESS);