壳牌取数据逻辑初始化

wanhua-executor-plugins/pom.xml

@@ -13,6 +13,7 @@
     <packaging>pom</packaging>
     <modules>
         <module>wanhua-frameless-stationv3-collector-plugin</module>
+        <module>wanhua-frameless-qp-stationv2-collector-plugin</module>
     </modules>
 
     <properties>

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/pom.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>wanhua-executor-plugins</artifactId>
+        <groupId>com.xuxueli</groupId>
+        <version>2.4.2-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>wanhua-frameless-qp-stationv2-collector-plugin</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+    <dependencies>
+        <!-- slf4j -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-reload4j</artifactId>
+            <version>${slf4j-api.version}</version>
+        </dependency>
+        <!-- junit -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit-jupiter.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- xxl-job-core -->
+        <dependency>
+            <groupId>com.xuxueli</groupId>
+            <artifactId>xxl-job-core</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+
+        <dependency>
+            <groupId>com.wanhua</groupId>
+            <artifactId>stationv3-base-dao-service</artifactId>
+            <version>1.0-QP-SNAPSHOT</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <!-- 生成的jar中，不要包含pom.xml和pom.properties这两个文件 -->
+                        <addMavenDescriptor>false</addMavenDescriptor>
+                        <manifest>
+                            <!-- 是否要把第三方jar放到manifest的classpath中 -->
+                            <addClasspath>true</addClasspath>
+                            <!-- 外部依赖jar包的最终位置 -->
+                            <classpathPrefix>lib/</classpathPrefix>
+                            <!-- 项目启动类 -->
+                            <mainClass>com.wanhua.frameless.stationv3.FramelessStationV3CollectorApplication</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <!-- 使用assembly打包 -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <!-- assembly配置文件位置 -->
+                        <descriptor>src/main/assembly/assembly.xml</descriptor>
+                    </descriptors>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/assembly/assembly.xml

@@ -0,0 +1,45 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id>release</id>
+    <formats>
+        <format>zip</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+<!--    <dependencySets>-->
+<!--        <dependencySet>-->
+<!--            <outputDirectory>/</outputDirectory>-->
+<!--            <useProjectArtifact>true</useProjectArtifact>-->
+<!--            <unpack>false</unpack>-->
+<!--            <scope>runtime</scope>-->
+<!--        </dependencySet>-->
+<!--        <dependencySet>-->
+<!--            <outputDirectory>/</outputDirectory>-->
+<!--            <useProjectArtifact>true</useProjectArtifact>-->
+<!--            <unpack>false</unpack>-->
+<!--            <scope>system</scope>-->
+<!--        </dependencySet>-->
+<!--    </dependencySets>-->
+    <dependencySets>
+        <dependencySet>
+            <!-- 不使用项目的artifact，第三方jar不要解压，打包进zip文件的lib目录 -->
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>lib</outputDirectory>
+            <unpack>false</unpack>
+        </dependencySet>
+    </dependencySets>
+    <fileSets>
+        <!-- 把项目的配置文件，打包进zip文件的config目录 -->
+        <fileSet>
+            <directory>${project.basedir}/src/main/resources</directory>
+            <outputDirectory>config</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}</directory>
+            <outputDirectory></outputDirectory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</assembly>

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/java/com/wanhua/frameless/qp/stationv2/FramelessQpStationV2CollectorApplication.java

@@ -0,0 +1,38 @@
+package com.wanhua.frameless.qp.stationv2;
+
+import com.wanhua.frameless.qp.stationv2.config.QpCollectorJobConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author xuxueli 2018-10-31 19:05:43
+ */
+public class FramelessQpStationV2CollectorApplication {
+    private static Logger logger = LoggerFactory.getLogger(FramelessQpStationV2CollectorApplication.class);
+
+    public static void main(String[] args) {
+
+        try {
+            // start
+            QpCollectorJobConfig.getInstance().initXxlJobExecutor();
+
+            // Blocks until interrupted
+            while (true) {
+                try {
+                    TimeUnit.HOURS.sleep(1);
+                } catch (InterruptedException e) {
+                    break;
+                }
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            // destroy
+            QpCollectorJobConfig.getInstance().destroyXxlJobExecutor();
+        }
+
+    }
+
+}

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/java/com/wanhua/frameless/qp/stationv2/config/QpCollectorJobConfig.java

@@ -0,0 +1,94 @@
+package com.wanhua.frameless.qp.stationv2.config;
+
+
+import com.wanhua.frameless.qp.stationv2.jobhandler.QpStationV2CollectorJob;
+import com.xxl.job.core.executor.impl.XxlJobSimpleExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Properties;
+
+/**
+ * @author xuxueli 2018-10-31 19:05:43
+ */
+public class QpCollectorJobConfig {
+    private static Logger logger = LoggerFactory.getLogger(QpCollectorJobConfig.class);
+
+
+    private static QpCollectorJobConfig instance = new QpCollectorJobConfig();
+    public static QpCollectorJobConfig getInstance() {
+        return instance;
+    }
+
+
+    private XxlJobSimpleExecutor xxlJobExecutor = null;
+
+    /**
+     * init
+     */
+    public void initXxlJobExecutor() {
+
+        // load executor prop
+        Properties xxlJobProp = loadProperties("xxl-job-executor.properties");
+
+        // init executor
+        xxlJobExecutor = new XxlJobSimpleExecutor();
+        xxlJobExecutor.setAdminAddresses(xxlJobProp.getProperty("xxl.job.admin.addresses"));
+        xxlJobExecutor.setAccessToken(xxlJobProp.getProperty("xxl.job.accessToken"));
+        xxlJobExecutor.setAppname(xxlJobProp.getProperty("xxl.job.executor.appname"));
+        xxlJobExecutor.setAddress(xxlJobProp.getProperty("xxl.job.executor.address"));
+        xxlJobExecutor.setIp(xxlJobProp.getProperty("xxl.job.executor.ip"));
+        xxlJobExecutor.setPort(Integer.valueOf(xxlJobProp.getProperty("xxl.job.executor.port")));
+        xxlJobExecutor.setLogPath(xxlJobProp.getProperty("xxl.job.executor.logpath"));
+        xxlJobExecutor.setLogRetentionDays(Integer.valueOf(xxlJobProp.getProperty("xxl.job.executor.logretentiondays")));
+
+        // registry job bean
+        xxlJobExecutor.setXxlJobBeanList(Arrays.asList(new QpStationV2CollectorJob()));
+
+        // start executor
+        try {
+            xxlJobExecutor.start();
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * destroy
+     */
+    public void destroyXxlJobExecutor() {
+        if (xxlJobExecutor != null) {
+            xxlJobExecutor.destroy();
+        }
+    }
+
+
+    public static Properties loadProperties(String propertyFileName) {
+        InputStreamReader in = null;
+        try {
+            ClassLoader loder = Thread.currentThread().getContextClassLoader();
+
+            in = new InputStreamReader(loder.getResourceAsStream(propertyFileName), "UTF-8");;
+            if (in != null) {
+                Properties prop = new Properties();
+                prop.load(in);
+                return prop;
+            }
+        } catch (IOException e) {
+            logger.error("load {} error!", propertyFileName);
+        } finally {
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (IOException e) {
+                    logger.error("close {} error!", propertyFileName);
+                }
+            }
+        }
+        return null;
+    }
+
+}

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/java/com/wanhua/frameless/qp/stationv2/jobhandler/QpStationV2CollectorJob.java

@@ -0,0 +1,251 @@
+package com.wanhua.frameless.qp.stationv2.jobhandler;
+
+import com.xxl.job.core.context.XxlJobHelper;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * XxlJob开发示例（Bean模式）
+ *
+ * 开发步骤：
+ *      1、任务开发：在Spring Bean实例中，开发Job方法；
+ *      2、注解配置：为Job方法添加注解 "@XxlJob(value="自定义jobhandler名称", init = "JobHandler初始化方法", destroy = "JobHandler销毁方法")"，注解value值对应的是调度中心新建任务的JobHandler属性的值。
+ *      3、执行日志：需要通过 "XxlJobHelper.log" 打印执行日志；
+ *      4、任务结果：默认任务结果为 "成功" 状态，不需要主动设置；如有诉求，比如设置任务结果为失败，可以通过 "XxlJobHelper.handleFail/handleSuccess" 自主设置任务结果；
+ *
+ * @author xuxueli 2019-12-11 21:52:51
+ */
+public class QpStationV2CollectorJob {
+    private static Logger logger = LoggerFactory.getLogger(QpStationV2CollectorJob.class);
+
+
+    /**
+     * 1、简单任务示例（Bean模式）
+     */
+    @XxlJob("demoJobHandler")
+    public void demoJobHandler() throws Exception {
+        XxlJobHelper.log("XXL-JOB, Hello World.");
+
+        for (int i = 0; i < 5; i++) {
+            XxlJobHelper.log("beat at:" + i);
+            TimeUnit.SECONDS.sleep(2);
+        }
+        // default success
+    }
+
+
+    /**
+     * 2、分片广播任务
+     */
+    @XxlJob("shardingJobHandler")
+    public void shardingJobHandler() throws Exception {
+
+        // 分片参数
+        int shardIndex = XxlJobHelper.getShardIndex();
+        int shardTotal = XxlJobHelper.getShardTotal();
+
+        XxlJobHelper.log("分片参数：当前分片序号 = {}, 总分片数 = {}", shardIndex, shardTotal);
+
+        // 业务逻辑
+        for (int i = 0; i < shardTotal; i++) {
+            if (i == shardIndex) {
+                XxlJobHelper.log("第 {} 片, 命中分片开始处理", i);
+            } else {
+                XxlJobHelper.log("第 {} 片, 忽略", i);
+            }
+        }
+
+    }
+
+
+    /**
+     * 3、命令行任务
+     */
+    @XxlJob("commandJobHandler")
+    public void commandJobHandler() throws Exception {
+        String command = XxlJobHelper.getJobParam();
+        int exitValue = -1;
+
+        BufferedReader bufferedReader = null;
+        try {
+            // command process
+            ProcessBuilder processBuilder = new ProcessBuilder();
+            processBuilder.command(command);
+            processBuilder.redirectErrorStream(true);
+
+            Process process = processBuilder.start();
+            //Process process = Runtime.getRuntime().exec(command);
+
+            BufferedInputStream bufferedInputStream = new BufferedInputStream(process.getInputStream());
+            bufferedReader = new BufferedReader(new InputStreamReader(bufferedInputStream));
+
+            // command log
+            String line;
+            while ((line = bufferedReader.readLine()) != null) {
+                XxlJobHelper.log(line);
+            }
+
+            // command exit
+            process.waitFor();
+            exitValue = process.exitValue();
+        } catch (Exception e) {
+            XxlJobHelper.log(e);
+        } finally {
+            if (bufferedReader != null) {
+                bufferedReader.close();
+            }
+        }
+
+        if (exitValue == 0) {
+            // default success
+        } else {
+            XxlJobHelper.handleFail("command exit value("+exitValue+") is failed");
+        }
+
+    }
+
+
+    /**
+     * 4、跨平台Http任务
+     *  参数示例：
+     *      "url: http://www.baidu.com\n" +
+     *      "method: get\n" +
+     *      "data: content\n";
+     */
+    @XxlJob("httpJobHandler")
+    public void httpJobHandler() throws Exception {
+
+        // param parse
+        String param = XxlJobHelper.getJobParam();
+        if (param==null || param.trim().length()==0) {
+            XxlJobHelper.log("param["+ param +"] invalid.");
+
+            XxlJobHelper.handleFail();
+            return;
+        }
+
+        String[] httpParams = param.split("\n");
+        String url = null;
+        String method = null;
+        String data = null;
+        for (String httpParam: httpParams) {
+            if (httpParam.startsWith("url:")) {
+                url = httpParam.substring(httpParam.indexOf("url:") + 4).trim();
+            }
+            if (httpParam.startsWith("method:")) {
+                method = httpParam.substring(httpParam.indexOf("method:") + 7).trim().toUpperCase();
+            }
+            if (httpParam.startsWith("data:")) {
+                data = httpParam.substring(httpParam.indexOf("data:") + 5).trim();
+            }
+        }
+
+        // param valid
+        if (url==null || url.trim().length()==0) {
+            XxlJobHelper.log("url["+ url +"] invalid.");
+
+            XxlJobHelper.handleFail();
+            return;
+        }
+        if (method==null || !Arrays.asList("GET", "POST").contains(method)) {
+            XxlJobHelper.log("method["+ method +"] invalid.");
+
+            XxlJobHelper.handleFail();
+            return;
+        }
+        boolean isPostMethod = method.equals("POST");
+
+        // request
+        HttpURLConnection connection = null;
+        BufferedReader bufferedReader = null;
+        try {
+            // connection
+            URL realUrl = new URL(url);
+            connection = (HttpURLConnection) realUrl.openConnection();
+
+            // connection setting
+            connection.setRequestMethod(method);
+            connection.setDoOutput(isPostMethod);
+            connection.setDoInput(true);
+            connection.setUseCaches(false);
+            connection.setReadTimeout(5 * 1000);
+            connection.setConnectTimeout(3 * 1000);
+            connection.setRequestProperty("connection", "Keep-Alive");
+            connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
+            connection.setRequestProperty("Accept-Charset", "application/json;charset=UTF-8");
+
+            // do connection
+            connection.connect();
+
+            // data
+            if (isPostMethod && data!=null && data.trim().length()>0) {
+                DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream());
+                dataOutputStream.write(data.getBytes("UTF-8"));
+                dataOutputStream.flush();
+                dataOutputStream.close();
+            }
+
+            // valid StatusCode
+            int statusCode = connection.getResponseCode();
+            if (statusCode != 200) {
+                throw new RuntimeException("Http Request StatusCode(" + statusCode + ") Invalid.");
+            }
+
+            // result
+            bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
+            StringBuilder result = new StringBuilder();
+            String line;
+            while ((line = bufferedReader.readLine()) != null) {
+                result.append(line);
+            }
+            String responseMsg = result.toString();
+
+            XxlJobHelper.log(responseMsg);
+
+            return;
+        } catch (Exception e) {
+            XxlJobHelper.log(e);
+
+            XxlJobHelper.handleFail();
+            return;
+        } finally {
+            try {
+                if (bufferedReader != null) {
+                    bufferedReader.close();
+                }
+                if (connection != null) {
+                    connection.disconnect();
+                }
+            } catch (Exception e2) {
+                XxlJobHelper.log(e2);
+            }
+        }
+
+    }
+
+    /**
+     * 5、生命周期任务示例：任务初始化与销毁时，支持自定义相关逻辑；
+     */
+    @XxlJob(value = "demoJobHandler2", init = "init", destroy = "destroy")
+    public void demoJobHandler2() throws Exception {
+        XxlJobHelper.log("XXL-JOB, Hello World.");
+    }
+    public void init(){
+        logger.info("init");
+    }
+    public void destroy(){
+        logger.info("destroy");
+    }
+
+
+}

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/resources/log4j.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE log4j:configuration PUBLIC "-//log4j/log4j Configuration//EN" "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" threshold="null" debug="null">
+
+	<appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+		<param name="Target" value="System.out" />
+		<layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss} xxl-job-executor-sample-frameless [%c]-[%t]-[%M]-[%L]-[%p] %m%n"/>
+		</layout>
+	</appender>
+	
+    <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="file" value="./logs/xxl-job/xxl-job-executor-sample-frameless.log"/>
+        <param name="append" value="true"/>
+        <param name="encoding" value="UTF-8"/>
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss} xxl-job-executor-sample-frameless [%c]-[%t]-[%M]-[%L]-[%p] %m%n"/>
+        </layout>
+    </appender>
+
+    <root>
+        <level value="INFO" />
+        <appender-ref ref="CONSOLE" />
+        <appender-ref ref="FILE" />
+    </root>
+
+</log4j:configuration>

wanhua-executor-plugins/wanhua-frameless-qp-stationv2-collector-plugin/src/main/resources/xxl-job-executor.properties

@@ -0,0 +1,17 @@
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+xxl.job.admin.addresses=http://172.22.62.105/internal/etl/xxl-job-admin
+
+### xxl-job, access token
+xxl.job.accessToken=default_token
+
+### xxl-job executor appname
+xxl.job.executor.appname=qpsy-interbase2
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
+xxl.job.executor.address=http://172.22.62.105
+### xxl-job executor server-info
+xxl.job.executor.ip=
+xxl.job.executor.port=19998
+### xxl-job executor log-path
+xxl.job.executor.logpath=logs/xxl-job/jobhandler
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays=30