From 8f36a77d2bd41d69e479ec1347d0a20f1d59eb05 Mon Sep 17 00:00:00 2001
From: xjs <1294405880@qq.com>
Date: Sun, 20 Feb 2022 18:48:30 +0800
Subject: [PATCH] =?UTF-8?q?1=E3=80=8136=E5=A3=81=E7=BA=B8=E7=BD=91?=
 =?UTF-8?q?=E7=88=AC=E8=99=AB=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E5=AE=9E?=
 =?UTF-8?q?=E7=8E=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../RemoteWebmagic36wallpaperFeign.java       | 24 +++++++++++
 .../RemoteWebmagic36wallpaperFactory.java     | 27 ++++++++++++
 ...moteWebmagicCopyWritingNetworkFactory.java |  4 +-
 .../factory/RemoteWebmagicSinaFactory.java    |  4 +-
 .../job/task/webmagic/_36wallpaperTask.java   | 42 +++++++++++++++++++
 .../webmagic/_36wallpaperProcessor.java       | 11 +++--
 .../com/xjs/common/aop/reptileLogAspect.java  |  3 ++
 7 files changed, 107 insertions(+), 8 deletions(-)
 create mode 100644 ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java
 create mode 100644 ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java
 create mode 100644 ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java

diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java
new file mode 100644
index 00000000..5b01a81b
--- /dev/null
+++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java
@@ -0,0 +1,24 @@
+package com.xjs.business.webmagic;
+
+import com.ruoyi.common.core.constant.ServiceNameConstants;
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.factory.RemoteWebmagic36wallpaperFactory;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.web.bind.annotation.GetMapping;
+
+/**
+ * 内部 调用 36壁纸网 爬虫定时任务
+ *
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@FeignClient(contextId = "remoteWebmagic36wallpaperFeign",
+        value = ServiceNameConstants.BUSINESS_WEBMAGIC_SERVICE,
+        fallbackFactory = RemoteWebmagic36wallpaperFactory.class)
+public interface RemoteWebmagic36wallpaperFeign {
+
+    @GetMapping("/_36wallpaper/taskForPRC")
+    public R _36wallpaperControllerTaskForPRC();
+
+
+}
diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java
new file mode 100644
index 00000000..05a08aeb
--- /dev/null
+++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java
@@ -0,0 +1,27 @@
+package com.xjs.business.webmagic.factory;
+
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.cloud.openfeign.FallbackFactory;
+import org.springframework.stereotype.Component;
+
+/**
+ * 内部 调用 36壁纸网 爬虫定时任务降级
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@Component
+@Log4j2
+public class RemoteWebmagic36wallpaperFactory implements FallbackFactory<RemoteWebmagic36wallpaperFeign> {
+    @Override
+    public RemoteWebmagic36wallpaperFeign create(Throwable cause) {
+        return new RemoteWebmagic36wallpaperFeign() {
+            @Override
+            public R _36wallpaperControllerTaskForPRC() {
+                log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
+            }
+        };
+    }
+}
diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java
index 3ce40662..c98e20bd 100644
--- a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java
+++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java
@@ -19,8 +19,8 @@ public class RemoteWebmagicCopyWritingNetworkFactory implements FallbackFactory<
         return new RemoteWebmagicCopyWritingNetworkFeign() {
             @Override
             public R copyWritingNetworkTaskForPRC() {
-                log.error("文案网 爬虫定时任务 降级");
-                return R.fail("降级处理");
+                log.error("文案网 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
             }
         };
     }
diff --git a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java
index c598d223..f6758c79 100644
--- a/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java
+++ b/ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java
@@ -20,8 +20,8 @@ public class RemoteWebmagicSinaFactory implements FallbackFactory<RemoteWebmagic
         return new RemoteWebmagicSinaFeign() {
             @Override
             public R sinaTaskForPRC() {
-                log.error("新浪 爬虫定时任务 降级");
-                return R.fail("降级处理");
+                log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
+                return R.fail("降级处理------服务可能正在运行");
             }
         };
     }
diff --git a/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java
new file mode 100644
index 00000000..00afb66d
--- /dev/null
+++ b/ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java
@@ -0,0 +1,42 @@
+package com.xjs.job.task.webmagic;
+
+import cn.hutool.core.date.DateUtil;
+import com.ruoyi.common.core.domain.R;
+import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+
+/**
+ * 爬虫 36壁纸网 定时任务
+ * @author xiejs
+ * @since 2022-02-20
+ */
+@Component("Wallpaper_36Task")
+@Log4j2
+public class _36wallpaperTask {
+
+    @Resource
+    private RemoteWebmagic36wallpaperFeign remoteWebmagic36wallpaperFeign;
+
+    /**
+     * 爬虫 36壁纸网 定时任务执行
+     */
+    public void _36wallpaper() {
+        log.info("---------------爬虫-36壁纸网定时任务Start-------------------");
+        LocalDateTime localDateTime1 = DateUtil.date().toLocalDateTime();
+
+        R r = remoteWebmagic36wallpaperFeign._36wallpaperControllerTaskForPRC();
+
+        log.info("爬虫-36壁纸网定时任务结果:code={},msg={},data={}",r.getCode(),r.getMsg(),r.getData());
+        LocalDateTime localDateTime2 = DateUtil.date().toLocalDateTime();
+        long between = ChronoUnit.MILLIS.between(localDateTime1, localDateTime2);
+        log.info("爬虫-36壁纸网定时任务Job耗费时间:{}ms", between);
+        log.info("---------------爬虫-36壁纸网定时任务end---------------------");
+
+    }
+
+}
diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java
index 759377f9..97aae974 100644
--- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java
+++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java
@@ -40,6 +40,9 @@ public class _36wallpaperProcessor implements PageProcessor {
      */
     private static boolean init = false;
 
+    private static final String headerKey = "User-Agent";
+    private static final String headerValue = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
+
 
     private static RedisService redisService;
 
@@ -63,6 +66,7 @@ public class _36wallpaperProcessor implements PageProcessor {
 
 
     private Site site = Site.me()
+            .addHeader(headerKey,headerValue)
             .setCharset("utf8")//设置字符编码
             .setTimeOut(2000)//设置超时时间
             .setRetrySleepTime(200)//设置重试间隔时间
@@ -160,6 +164,9 @@ public class _36wallpaperProcessor implements PageProcessor {
 
             //循环次数存入redis中
             Integer count = redisService.getCacheObject(REPTILE_COUNT);
+            if (count == null) {
+                count=0;
+            }
             redisService.setCacheObject(REPTILE_COUNT, count+1);
 
         } catch (Exception e) {
@@ -239,8 +246,4 @@ public class _36wallpaperProcessor implements PageProcessor {
     }
 
 
-    public static void main(String[] args) {
-        Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(15).run();
-
-    }
 }
diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java
index 4ceae502..2b91ba8e 100644
--- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java
+++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java
@@ -97,6 +97,9 @@ public class reptileLogAspect {
                         if (obj instanceof Long) {
                             webmagicLog.setComplexRate((Long) obj);
                         }
+                        if (obj instanceof Integer) {
+                            webmagicLog.setComplexRate(Long.parseLong(obj.toString()));
+                        }
                         this.saveData(webmagicLog);
                     }
                 }