1、36壁纸网爬虫定时任务实现

ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java

@@ -0,0 +1,24 @@
+package com.xjs.business.webmagic;
+
+import com.ruoyi.common.core.constant.ServiceNameConstants;
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.factory.RemoteWebmagic36wallpaperFactory;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.web.bind.annotation.GetMapping;
+
+/**
+ * 内部 调用 36壁纸网 爬虫定时任务
+ *
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@FeignClient(contextId = "remoteWebmagic36wallpaperFeign",
+        value = ServiceNameConstants.BUSINESS_WEBMAGIC_SERVICE,
+        fallbackFactory = RemoteWebmagic36wallpaperFactory.class)
+public interface RemoteWebmagic36wallpaperFeign {
+
+    @GetMapping("/_36wallpaper/taskForPRC")
+    public R _36wallpaperControllerTaskForPRC();
+
+
+}

ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java

@@ -0,0 +1,27 @@
+package com.xjs.business.webmagic.factory;
+
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.cloud.openfeign.FallbackFactory;
+import org.springframework.stereotype.Component;
+
+/**
+ * 内部 调用 36壁纸网 爬虫定时任务降级
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@Component
+@Log4j2
+public class RemoteWebmagic36wallpaperFactory implements FallbackFactory<RemoteWebmagic36wallpaperFeign> {
+    @Override
+    public RemoteWebmagic36wallpaperFeign create(Throwable cause) {
+        return new RemoteWebmagic36wallpaperFeign() {
+            @Override
+            public R _36wallpaperControllerTaskForPRC() {
+                log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
+            }
+        };
+    }
+}

ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java

@@ -19,8 +19,8 @@ public class RemoteWebmagicCopyWritingNetworkFactory implements FallbackFactory<
         return new RemoteWebmagicCopyWritingNetworkFeign() {
             @Override
             public R copyWritingNetworkTaskForPRC() {
-                log.error("文案网 爬虫定时任务 降级");
-                return R.fail("降级处理");
+                log.error("文案网 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
             }
         };
     }

ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java

@@ -20,8 +20,8 @@ public class RemoteWebmagicSinaFactory implements FallbackFactory<RemoteWebmagic
         return new RemoteWebmagicSinaFeign() {
             @Override
             public R sinaTaskForPRC() {
-                log.error("新浪 爬虫定时任务 降级");
-                return R.fail("降级处理");
+                log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
             }
         };
     }

ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java

@@ -0,0 +1,42 @@
+package com.xjs.job.task.webmagic;
+
+import cn.hutool.core.date.DateUtil;
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * 爬虫 36壁纸网 定时任务
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@Component("Wallpaper_36Task")
+@Log4j2
+public class _36wallpaperTask {
+
+    @Resource
+    private RemoteWebmagic36wallpaperFeign remoteWebmagic36wallpaperFeign;
+
+    /**
+     * 爬虫 36壁纸网 定时任务执行
+     */
+    public void _36wallpaper() {
+        log.info("---------------爬虫-36壁纸网定时任务Start-------------------");
+        LocalDateTime localDateTime1 = DateUtil.date().toLocalDateTime();
+
+        R r = remoteWebmagic36wallpaperFeign._36wallpaperControllerTaskForPRC();
+
+        log.info("爬虫-36壁纸网定时任务结果:code={},msg={},data={}",r.getCode(),r.getMsg(),r.getData());
+        LocalDateTime localDateTime2 = DateUtil.date().toLocalDateTime();
+        long between = ChronoUnit.MILLIS.between(localDateTime1, localDateTime2);
+        log.info("爬虫-36壁纸网定时任务Job耗费时间:{}ms", between);
+        log.info("---------------爬虫-36壁纸网定时任务end---------------------");
+
+    }
+
+}

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java

@@ -40,6 +40,9 @@ public class _36wallpaperProcessor implements PageProcessor {
      */
     private static boolean init = false;
 
+    private static final String headerKey = "User-Agent";
+    private static final String headerValue = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
+
 
     private static RedisService redisService;
 
@@ -63,6 +66,7 @@ public class _36wallpaperProcessor implements PageProcessor {
 
 
     private Site site = Site.me()
+            .addHeader(headerKey,headerValue)
             .setCharset("utf8")//设置字符编码
             .setTimeOut(2000)//设置超时时间
             .setRetrySleepTime(200)//设置重试间隔时间
@@ -160,6 +164,9 @@ public class _36wallpaperProcessor implements PageProcessor {
 
             //循环次数存入redis中
             Integer count = redisService.getCacheObject(REPTILE_COUNT);
+            if (count == null) {
+                count=0;
+            }
             redisService.setCacheObject(REPTILE_COUNT, count+1);
 
         } catch (Exception e) {
@@ -239,8 +246,4 @@ public class _36wallpaperProcessor implements PageProcessor {
     }
 
 
-    public static void main(String[] args) {
-        Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(15).run();
-
-    }
 }

xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java

@@ -97,6 +97,9 @@ public class reptileLogAspect {
                         if (obj instanceof Long) {
                             webmagicLog.setComplexRate((Long) obj);
                         }
+                        if (obj instanceof Integer) {
+                            webmagicLog.setComplexRate(Long.parseLong(obj.toString()));
+                        }
                         this.saveData(webmagicLog);
                     }
                 }