2023-09-08更新

Test/Hadoop/pom.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>2.6.7</version>
+        <relativePath/> <!-- lookup parent from repository -->
+    </parent>
+    <groupId>com.renchao</groupId>
+    <artifactId>Hadoop</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>Hadoop</name>
+    <description>Demo project for Spring Boot</description>
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+    <dependencies>
+<!--        <dependency>-->
+<!--            <groupId>org.springframework.boot</groupId>-->
+<!--            <artifactId>spring-boot-starter</artifactId>-->
+<!--        </dependency>-->
+
+        <!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-web -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc -->
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-jdbc</artifactId>
+            <version>3.1.2</version>
+
+            <exclusions>
+
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-framework</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-upgrade-acid</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-shims</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-metastore</artifactId>
+                </exclusion>
+
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-runner</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

Test/Hadoop/src/main/java/com/renchao/HadoopApplication.java

@@ -0,0 +1,14 @@
+package com.renchao;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class HadoopApplication {
+
+    public static void main(String[] args) {
+
+        SpringApplication.run(HadoopApplication.class, args);
+    }
+
+}

Test/Hadoop/src/main/java/com/renchao/controller/HiveController.java

@@ -0,0 +1,17 @@
+package com.renchao.controller;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+public class HiveController {
+
+
+    @GetMapping("/test")
+    public String test() {
+        return "hiveService.test()";
+    }
+}

Test/Hadoop/src/main/java/com/renchao/hive/HiveTest.java

@@ -0,0 +1,43 @@
+package com.renchao.hive;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+public class HiveTest {
+    public static void main(String[] args) {
+        // HiveServer2的JDBC连接URL
+        String hiveServerURL = "jdbc:hive2://172.16.12.101:10000/hive;socketTimeout=12000;";
+        String hiveUser = "flink";
+        String hivePassword = "flink";
+
+        try {
+            // 加载Hive JDBC驱动程序
+            Class.forName("org.apache.hive.jdbc.HiveDriver");
+
+            // 连接到HiveServer2
+            Connection connection = DriverManager.getConnection(hiveServerURL, hiveUser, hivePassword);
+
+            // 创建一个Hive语句对象
+            Statement statement = connection.createStatement();
+
+            // 执行Hive查询
+            String sql = "select * from student2";
+            ResultSet resultSet = statement.executeQuery(sql);
+
+            // 处理查询结果
+            while (resultSet.next()) {
+                System.out.println(resultSet.getInt("id"));
+                System.out.println(resultSet.getString("name"));
+            }
+
+            // 关闭资源
+            resultSet.close();
+            statement.close();
+            connection.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

Test/MyMaven/src/main/java/com/renchao/RestTemplate/RestDemo.java

@@ -10,8 +10,10 @@ import org.springframework.web.client.RestTemplate;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 public class RestDemo {
     public static void main(String[] args) throws UnsupportedEncodingException {
@@ -25,6 +27,7 @@ public class RestDemo {
         ResponseEntity<String> exchange = restTemplate.postForEntity("http://localhost:8889/test02", request, String.class);
 //        ResponseEntity<String> exchange = restTemplate.getForEntity("https://www.baidu.com/", String.class, request);
         System.out.println(exchange.getBody());
+        List<byte[]> bytes = new ArrayList<>();
         test();
     }
 

Test/MyMaven/src/main/java/com/renchao/spring/AnnotationUtilsTest.java

@@ -2,8 +2,10 @@ package com.renchao.spring;
 
 import com.renchao.spring.bean.Anonymous;
 import com.renchao.spring.bean.TestController;
+import org.junit.Test;
 import org.springframework.core.annotation.AnnotatedElementUtils;
 import org.springframework.core.annotation.AnnotationUtils;
+import org.springframework.http.ContentDisposition;
 
 public class AnnotationUtilsTest {
     public static void main(String[] args) {
@@ -11,4 +13,9 @@ public class AnnotationUtilsTest {
         System.out.println(annotation);
         System.out.println(AnnotatedElementUtils.isAnnotated(TestController.class, Anonymous.class));
     }
+
+    @Test
+    public void test01() {
+        System.out.println(ContentDisposition.attachment().filename("55.txt").build());
+    }
 }

Test/src/com/renchao/RSADemo.java

@@ -2,14 +2,24 @@ package com.renchao;
 
 import org.junit.Test;
 
+import javax.crypto.BadPaddingException;
 import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.OAEPParameterSpec;
+import javax.crypto.spec.PSource;
 import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
 import java.security.KeyFactory;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
 import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
 import java.security.spec.KeySpec;
+import java.security.spec.MGF1ParameterSpec;
+import java.security.spec.PKCS8EncodedKeySpec;
 import java.security.spec.X509EncodedKeySpec;
 import java.util.Base64;
 
@@ -25,7 +35,7 @@ public class RSADemo {
 
         // 加密消息
         String message = "Hello RSA";
-        Cipher cipher = Cipher.getInstance("RSA");
+        Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
         cipher.init(Cipher.ENCRYPT_MODE, publicKey);
         byte[] encryptedMessage = cipher.doFinal(message.getBytes());
         System.out.println("Encrypted message: " + Base64.getEncoder().encodeToString(encryptedMessage));
@@ -39,7 +49,7 @@ public class RSADemo {
     @Test
     public void test01() throws Exception {
 //        String publicKeyStr = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDG03gR1w6i3E6h6+N9F2///BnRrkzPc7RT4qZKKl2b/rolym0EYl3QZTsIV5oQngT93TLtld7EK5svdwUabX6kzqd8yDDChZXS/E7/FrufN6Hwf9S3O3ZzkhEyd45HmRHV4aNRFsS/NviEZx83D6FR94l0SPnomvPkVqM8UnafnQIDAQAB";
-        String publicKeyStr = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCUAPS4EiEkyKAay4dRY7hWuyTSewj3X1g9NXj6832Eup0VE+xxGfsDiU5xlZBenFcLT8nn88q3mYit5DowuwxTCmem2TIAfkxdAnZ4vm7ndVbugQTu3TDB5R7LIGRjNF62lfwzYc7ywJFHVH/7dVfh4/uaijjQeDhznlBxM57NgwIDAQAB";
+        String publicKeyStr = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC5Kfw/IMSSsGOKhN08Vmf4ztuvnBrfm8EYJ7pdBmeh64RDSd4t5QGeH086ExXfFKCHyiF2ryfMY/rknictvBSMTomD4U8JwKEgIKQcukHaFnyEpHmZalqSUuWjZJyE6Ru6O8CLM9yMwQHKW7H4vSxrw3MM4acgytPuTGgaC6oeaQIDAQAB";
         String privateKeyStr = "MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMbTeBHXDqLcTqHr430Xb//8GdGuTM9ztFPipkoqXZv+uiXKbQRiXdBlOwhXmhCeBP3dMu2V3sQrmy93BRptfqTOp3zIMMKFldL8Tv8Wu583ofB/1Lc7dnOSETJ3jkeZEdXho1EWxL82+IRnHzcPoVH3iXRI+eia8+RWozxSdp+dAgMBAAECgYAJjtfqT6LR/HJBQXQ9qrdFIIrjNBRYMrE8CRzCWvgGDEBJmcoU2F+3KW6lj4SGAPqvc4dDuZ0sZAZBSWDy7MmWL+Zz2z44sulxsOsb3DJqIyBSAr5D6mhrRmu7MJA5AGgDHo/2gn+9Cji2JQBHBFe18BzJdr2tIM4uAYTVB6EW8QJBAPCrnHohSDtgLSmHrbORP/cIS8OOF/M3PsYfHZ3cpdrKk2zs1rXAHJq80GlmhSQx8tezx6wt63Cph0reiHbOMRkCQQDTfYqahFR0NTFFfTBfSJKQEqoiRYMnOrjkkOOgFv6cBwYd16pnqTfNISSYkBsOcDO09qiMILW96MoJONCV458lAkEAmMrqueK9X+zMX0xjK9hwOp5Ks2lXrTKKqO+CNwGpTkFD3WhzW8oOnvJ2giPzLSqE2QqrHpW8nrcSTKcBDiQTqQJABORmjGR7P6TrWtwmfk3Ddim4XcqV2hZ1qHPhkBZ4FUvkTFRs0LENZWVa31yWA6N8zrbV90fabGYyJjx2NsFpMQJARtRflzJjWc/49nzu+om41bz9Ngg07/S8Rxe8AlZbSlCxggmp/KUBcoVgNJCa5qGsX2AvTOCXaHngp+YLtHHPBQ==";
 
         KeyFactory keyFactory = KeyFactory.getInstance("RSA");
@@ -50,7 +60,8 @@ public class RSADemo {
         System.out.println(publicKey);
         System.out.println(System.currentTimeMillis());
         rsa.init(Cipher.ENCRYPT_MODE, publicKey);
-        String str = "Admin1234," + System.currentTimeMillis();
+        String str = "111111," + System.currentTimeMillis();
+//        String str = "Admin1234," + System.currentTimeMillis();
         byte[] bytes = rsa.doFinal(str.getBytes(StandardCharsets.UTF_8));
         String s = Base64.getEncoder().encodeToString(bytes);
         System.out.println(s);
@@ -66,5 +77,12 @@ public class RSADemo {
 
     }
 
+    @Test
+    public void test02() throws InterruptedException {
+        System.out.println(System.currentTimeMillis());
+        Thread.sleep(1000);
+        System.out.println(System.currentTimeMillis());
 
+        System.out.println(1000 << 9);
+    }
 }

Test/src/com/renchao/Test01.java

@@ -1,6 +1,18 @@
 package com.renchao;
 
+import org.junit.Test;
+import sun.net.www.http.HttpClient;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
@@ -16,5 +28,48 @@ public class Test01 {
         System.out.println(TimeUnit.NANOSECONDS.toMillis(duration));
     }
 
+
+    @Test
+    public void test02() {
+        List<String> ALLOW_PYTHON_EXT = Arrays.asList("zip", "tar", "gz", "bz2");
+        List<String> ALLOW_DATA_EXT = Arrays.asList("zip", "tar", "gz", "csv", "txt", "xls", "xlsx");
+
+        String fileType = "PYTHONs";
+        String fileExt = "zip";
+
+        boolean isPython = "PYTHON".equals(fileType) && ALLOW_PYTHON_EXT.contains(fileExt);
+        boolean isData = "DATA".equals(fileType) && ALLOW_DATA_EXT.contains(fileExt);
+        if (!isPython && !isData) {
+            System.out.println("文件类型错误");
+        } else {
+            System.out.println("文件类型正确====");
+        }
+    }
+
+    @Test
+    public void test03() throws IOException {
+        String ipAddress = "8.8.8.8"; // 要查询的IP地址
+        String apiKey = "YOUR_API_KEY"; // 在https://ipinfo.io/signup获取您的免费API密钥
+
+        String apiUrl = "http://whois.pconline.com.cn/ipJson.jsp?ip=112.64.187.2&json=true";
+
+        URL url = new URL(apiUrl);
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+        BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8.name()));
+        StringBuilder response = new StringBuilder();
+        String line;
+
+        while ((line = reader.readLine()) != null) {
+            response.append(line);
+        }
+        reader.close();
+
+        String jsonResponse = response.toString();
+        System.out.println(jsonResponse); // 打印响应，包含地理位置等信息
+
+        // 在这里您可以解析jsonResponse并提取所需的地理位置信息
+    }
+
 }
 

agile-bacth/agile-batch-api/pom.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>agile-bacth</artifactId>
+        <groupId>com.jiuyv.sptcc.agile.batch</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>agile-batch-api</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+</project>

agile-bacth/agile-batch-dws/pom.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>agile-bacth</artifactId>
+        <groupId>com.jiuyv.sptcc.agile.batch</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>agile-batch-dws</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc -->
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-jdbc</artifactId>
+            <version>3.1.2</version>
+            <exclusions>
+
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.curator</groupId>
+                    <artifactId>curator-framework</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-upgrade-acid</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-shims</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-metastore</artifactId>
+                </exclusion>
+
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-runner</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+</project>

agile-bacth/agile-batch-service/.gitignore

@@ -0,0 +1,33 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/

agile-bacth/agile-batch-service/pom.xml

@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <parent>
+        <artifactId>agile-bacth</artifactId>
+        <groupId>com.jiuyv.sptcc.agile.batch</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>agile-batch-service</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>agile-batch-service</name>
+    <description>agile-batch-service</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-starter</artifactId>
+        </dependency>
+
+        <dependency>
+          <groupId>org.springframework.cloud</groupId>
+           <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
+       </dependency>
+        <dependency>
+           <groupId>org.springframework.cloud</groupId>
+           <artifactId>spring-cloud-starter-loadbalancer</artifactId>
+         </dependency>
+       <dependency>
+           <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-config</artifactId>
+       </dependency>
+       <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-bootstrap</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.springframework.boot</groupId>
+          <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <!-- spring security 安全认证 -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-security</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-devtools</artifactId>
+            <scope>runtime</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+        
+        
+        <!-- hive库有一些包特殊的需要用huawei的-->
+        
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-client</artifactId>
+            <version>2.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-framework</artifactId>
+            <version>2.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.woodstox</groupId>
+            <artifactId>woodstox-core</artifactId>
+            <version>6.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.codehaus.woodstox</groupId>
+            <artifactId>woodstox-core-asl</artifactId>
+            <version>4.4.1</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+            <version>3.2.2</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-configuration</groupId>
+            <artifactId>commons-configuration</artifactId>
+            <version>1.10</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-configuration2</artifactId>
+            <version>2.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        
+        <!-- json logstash encoder -->
+        <dependency>
+            <groupId>net.logstash.logback</groupId>
+            <artifactId>logstash-logback-encoder</artifactId>
+            <version>6.4</version>
+        </dependency>
+
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>2.1.1.RELEASE</version>
+                <configuration>
+                    <fork>true</fork> <!-- 如果没有该配置，devtools不会生效 -->
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <encoding>UTF-8</encoding>
+                    <compilerArguments>
+                        <extdirs>${project.basedir}/src/main/resources/libx</extdirs>
+                    </compilerArguments>
+                </configuration>
+            </plugin>
+        </plugins>
+        
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*</include>
+                </includes>
+                <excludes>
+                    <exclude>libx/**</exclude>
+                    <exclude>config/hiveJsy/**</exclude>
+                </excludes>
+                <filtering>false</filtering>
+            </resource>
+            <!--打成JAR包时 合并 额外引入的JAR包库 -->
+            <resource>
+                <directory>src/main/resources/libx</directory>
+                <targetPath>BOOT-INF/lib/</targetPath>
+                <includes>
+                    <include>**/*.jar</include>
+                </includes>
+            </resource>
+       </resources>
+    </build>
+
+</project>

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/AgileBatchServiceApplication.java

@@ -0,0 +1,17 @@
+package com.jiuyv.sptcc.agile.batch;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+@EnableScheduling
+@SpringBootApplication
+@EnableTransactionManagement
+public class AgileBatchServiceApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(AgileBatchServiceApplication.class, args);
+    }
+
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/common/TblBatchTaskEnum.java

@@ -0,0 +1,58 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.common;
+
+/**
+ * 批处理任务表枚举
+ * @author zhouliang
+ *
+ */
+public class TblBatchTaskEnum {
+    
+    /** 业务状态*/
+    public enum BUS_STATUS {
+        RUNING("runing", "任务运行中"),
+        //三个结束都等价任务未运行
+        END("end", "强制结束"),//如果任务运行中项目重启，那么会更新为此状态.
+        FINISH("finish", "正常结束"),
+        UNFINISH("unfinish", "异常结束"),
+        
+        ;
+        private String code;
+        private String msg;
+
+        BUS_STATUS(String code, String msg) {
+            this.code = code;
+            this.msg = msg;
+        }
+        
+        public String getCode() {
+            return code;
+        }
+        
+        public String getMsg() {
+            return msg;
+        }
+    }
+    
+    /** 数据状态*/
+    public enum DATA_STATUS {
+        NORMAL("00", "正常"),
+        DELETED("99", "删除"),
+        
+        ;
+        private String code;
+        private String msg;
+
+        DATA_STATUS(String code, String msg) {
+            this.code = code;
+            this.msg = msg;
+        }
+        
+        public String getCode() {
+            return code;
+        }
+        
+        public String getMsg() {
+            return msg;
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/entity/TblBatchTableMapping.java

@@ -0,0 +1,274 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.entity;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+
+/**
+ * 批处理同步表映射表
+ * @author zhouliang
+ * @date 2023-07-24
+ */
+public class TblBatchTableMapping implements java.io.Serializable {
+
+    private static final long serialVersionUID = 1L;
+    
+    /** 任务编号 */
+    private String taskNo;
+    
+    /** 版本号 */
+    private Long versionNum;
+    
+    /** 随机码 */
+    private String recToken;
+    
+    /** 远程表查询sql */
+    private String remoteTableSql;
+    
+    /** 远程数据库 */
+    private String remoteDbName;
+    
+    /** 远程前推天数 */
+    private Integer remoteDays;
+    
+    /** 本地表编码 */
+    private String localTable;
+    
+    /** 本地数据库 */
+    private String localDbName;
+    
+    /** 本地前置sql */
+    private String localPreSql;
+    
+    
+    /** 映射关系 */
+    private String mappingJson;
+    
+    /** 备注 */
+    private String remarks;
+    
+    /** 数据状态 */
+    private String dataStatus;
+    
+    /** 更新时间 */
+    private Date updateTime;
+    
+    /** 备用字段1 */
+    private String rsv1;
+    
+    /** 备用字段2 */
+    private String rsv2;
+    
+    /** 备用字段3 */
+    private String rsv3;
+    
+    
+    
+    /**
+     * Get任务编号
+     */
+    public String getTaskNo(){
+        return taskNo;
+    }
+    /**
+     * Set任务编号
+     */
+    public void setTaskNo(String taskNo){
+        this.taskNo = taskNo;
+    }
+    
+    /**
+     * Get版本号
+     */
+    public Long getVersionNum(){
+        return versionNum;
+    }
+    /**
+     * Set版本号
+     */
+    public void setVersionNum(Long versionNum){
+        this.versionNum = versionNum;
+    }
+    
+    /**
+     * Get随机码
+     */
+    public String getRecToken(){
+        return recToken;
+    }
+    /**
+     * Set随机码
+     */
+    public void setRecToken(String recToken){
+        this.recToken = recToken;
+    }
+    
+    /**
+     * Get远程表查询sql
+     */
+    public String getRemoteTableSql(){
+        return remoteTableSql;
+    }
+    /**
+     * Set远程表查询sql
+     */
+    public void setRemoteTableSql(String remoteTableSql){
+        this.remoteTableSql = remoteTableSql;
+    }
+    
+    /**
+     * Get远程数据库
+     */
+    public String getRemoteDbName(){
+        return remoteDbName;
+    }
+    /**
+     * Set远程数据库
+     */
+    public void setRemoteDbName(String remoteDbName){
+        this.remoteDbName = remoteDbName;
+    }
+
+    /**
+     * Get远程前推天数
+     */
+    public Integer getRemoteDays() {
+        return remoteDays;
+    }
+    /**
+     * Set远程前推天数
+     */
+    public void setRemoteDays(Integer remoteDays) {
+        this.remoteDays = remoteDays;
+    }
+    
+    /**
+     * Get本地表编码
+     */
+    public String getLocalTable(){
+        return localTable;
+    }
+    /**
+     * Set本地表编码
+     */
+    public void setLocalTable(String localTable){
+        this.localTable = localTable;
+    }
+    
+    /**
+     * Get本地数据库
+     */
+    public String getLocalDbName(){
+        return localDbName;
+    }
+    /**
+     * Set本地数据库
+     */
+    public void setLocalDbName(String localDbName){
+        this.localDbName = localDbName;
+    }
+    
+    /**
+     * Get本地前置sql
+     */
+    public String getLocalPreSql() {
+        return localPreSql;
+    }
+    /**
+     * Set本地前置sql
+     */
+    public void setLocalPreSql(String localPreSql) {
+        this.localPreSql = localPreSql;
+    }
+    
+    /**
+     * Get映射关系
+     */
+    public String getMappingJson(){
+        return mappingJson;
+    }
+    /**
+     * Set映射关系
+     */
+    public void setMappingJson(String mappingJson){
+        this.mappingJson = mappingJson;
+    }
+    
+    /**
+     * Get备注
+     */
+    public String getRemarks(){
+        return remarks;
+    }
+    /**
+     * Set备注
+     */
+    public void setRemarks(String remarks){
+        this.remarks = remarks;
+    }
+    
+    /**
+     * Get数据状态
+     */
+    public String getDataStatus(){
+        return dataStatus;
+    }
+    /**
+     * Set数据状态
+     */
+    public void setDataStatus(String dataStatus){
+        this.dataStatus = dataStatus;
+    }
+    
+    /**
+     * Get更新时间
+     */
+    public Date getUpdateTime(){
+        return updateTime;
+    }
+    /**
+     * Set更新时间
+     */
+    public void setUpdateTime(Date updateTime){
+        this.updateTime = updateTime;
+    }
+    
+    /**
+     * Get备用字段1
+     */
+    public String getRsv1(){
+        return rsv1;
+    }
+    /**
+     * Set备用字段1
+     */
+    public void setRsv1(String rsv1){
+        this.rsv1 = rsv1;
+    }
+    
+    /**
+     * Get备用字段2
+     */
+    public String getRsv2(){
+        return rsv2;
+    }
+    /**
+     * Set备用字段2
+     */
+    public void setRsv2(String rsv2){
+        this.rsv2 = rsv2;
+    }
+    
+    /**
+     * Get备用字段3
+     */
+    public String getRsv3(){
+        return rsv3;
+    }
+    /**
+     * Set备用字段3
+     */
+    public void setRsv3(String rsv3){
+        this.rsv3 = rsv3;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/entity/TblBatchTask.java

@@ -0,0 +1,258 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.entity;
+
+import java.math.BigDecimal;
+import java.util.Date;
+
+
+/**
+ * 批处理任务表
+ * @author zhouliang
+ * @date 2023-07-05
+ */
+public class TblBatchTask implements java.io.Serializable {
+
+    private static final long serialVersionUID = 1L;
+    
+    /** 任务编号 */
+    private String taskNo;
+    
+    /** 版本号 */
+    private Long versionNum;
+    
+    /** 随机码 */
+    private String recToken;
+    
+    /** 任务名称 */
+    private String taskTitle;
+    
+    /** 上次开始时间 */
+    private Date preStartDate;
+    
+    /** 上次结束时间 */
+    private Date preEndDate;
+    
+    /** 上次耗时 */
+    private String preTotalTime;
+    
+    /** 当前开始时间 */
+    private Date currStartDate;
+    
+    /** 失败数据条件 */
+    private String failureConditions;
+    
+    /** 任务状态 */
+    private String busStatus;
+    
+    /** 数据状态 */
+    private String dataStatus;
+    
+    /** 更新时间 */
+    private Date updateTime;
+    
+    /** 备用字段1 */
+    private String rsv1;
+    
+    /** 备用字段2 */
+    private String rsv2;
+    
+    /** 备用字段3 */
+    private String rsv3;
+    
+    
+    
+    /**
+     * Get任务编号
+     */
+    public String getTaskNo(){
+        return taskNo;
+    }
+    /**
+     * Set任务编号
+     */
+    public void setTaskNo(String taskNo){
+        this.taskNo = taskNo;
+    }
+    
+    /**
+     * Get版本号
+     */
+    public Long getVersionNum(){
+        return versionNum;
+    }
+    /**
+     * Set版本号
+     */
+    public void setVersionNum(Long versionNum){
+        this.versionNum = versionNum;
+    }
+    
+    /**
+     * Get随机码
+     */
+    public String getRecToken(){
+        return recToken;
+    }
+    /**
+     * Set随机码
+     */
+    public void setRecToken(String recToken){
+        this.recToken = recToken;
+    }
+    
+    /**
+     * Get任务名称
+     */
+    public String getTaskTitle(){
+        return taskTitle;
+    }
+    /**
+     * Set任务名称
+     */
+    public void setTaskTitle(String taskTitle){
+        this.taskTitle = taskTitle;
+    }
+    
+    /**
+     * Get上次开始时间
+     */
+    public Date getPreStartDate(){
+        return preStartDate;
+    }
+    /**
+     * Set上次开始时间
+     */
+    public void setPreStartDate(Date preStartDate){
+        this.preStartDate = preStartDate;
+    }
+    
+    /**
+     * Get上次结束时间
+     */
+    public Date getPreEndDate(){
+        return preEndDate;
+    }
+    /**
+     * Set上次结束时间
+     */
+    public void setPreEndDate(Date preEndDate){
+        this.preEndDate = preEndDate;
+    }
+    
+    /**
+     * Get上次耗时
+     */
+    public String getPreTotalTime(){
+        return preTotalTime;
+    }
+    /**
+     * Set上次耗时
+     */
+    public void setPreTotalTime(String preTotalTime){
+        this.preTotalTime = preTotalTime;
+    }
+    
+    /**
+     * Get当前开始时间
+     */
+    public Date getCurrStartDate(){
+        return currStartDate;
+    }
+    /**
+     * Set当前开始时间
+     */
+    public void setCurrStartDate(Date currStartDate){
+        this.currStartDate = currStartDate;
+    }
+    
+    /**
+     * Get失败数据条件
+     */
+    public String getFailureConditions(){
+        return failureConditions;
+    }
+    /**
+     * Set失败数据条件
+     */
+    public void setFailureConditions(String failureConditions){
+        this.failureConditions = failureConditions;
+    }
+    
+    /**
+     * Get任务状态
+     */
+    public String getBusStatus(){
+        return busStatus;
+    }
+    /**
+     * Set任务状态
+     */
+    public void setBusStatus(String busStatus){
+        this.busStatus = busStatus;
+    }
+    
+    /**
+     * Get数据状态
+     */
+    public String getDataStatus(){
+        return dataStatus;
+    }
+    /**
+     * Set数据状态
+     */
+    public void setDataStatus(String dataStatus){
+        this.dataStatus = dataStatus;
+    }
+    
+    /**
+     * Get更新时间
+     */
+    public Date getUpdateTime(){
+        return updateTime;
+    }
+    /**
+     * Set更新时间
+     */
+    public void setUpdateTime(Date updateTime){
+        this.updateTime = updateTime;
+    }
+    
+    /**
+     * Get备用字段1
+     */
+    public String getRsv1(){
+        return rsv1;
+    }
+    /**
+     * Set备用字段1
+     */
+    public void setRsv1(String rsv1){
+        this.rsv1 = rsv1;
+    }
+    
+    /**
+     * Get备用字段2
+     */
+    public String getRsv2(){
+        return rsv2;
+    }
+    /**
+     * Set备用字段2
+     */
+    public void setRsv2(String rsv2){
+        this.rsv2 = rsv2;
+    }
+    
+    /**
+     * Get备用字段3
+     */
+    public String getRsv3(){
+        return rsv3;
+    }
+    /**
+     * Set备用字段3
+     */
+    public void setRsv3(String rsv3){
+        this.rsv3 = rsv3;
+    }
+    
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/entity/vo/TblBatchTableMappingVO.java

@@ -0,0 +1,16 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.entity.vo;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+
+ /**
+ * 批处理同步表映射表，扩展
+ * @author zhouliang
+ * @date 2023-07-24
+ */
+public class TblBatchTableMappingVO extends TblBatchTableMapping implements java.io.Serializable {
+
+    private static final long serialVersionUID = 1L;
+    
+    
+    
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/entity/vo/TblBatchTaskVO.java

@@ -0,0 +1,38 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.entity.vo;
+
+import java.util.List;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTask;
+
+ /**
+ * 批处理任务表，扩展
+ * @author zhouliang
+ * @date 2023-07-05
+ */
+public class TblBatchTaskVO extends TblBatchTask implements java.io.Serializable {
+
+    private static final long serialVersionUID = 1L;
+    
+    /** 任务状态集合 */
+    private List<String> busStatuss;
+
+    private TblBatchTableMapping mappingInfo;
+    
+    
+    public List<String> getBusStatuss() {
+        return busStatuss;
+    }
+
+    public void setBusStatuss(List<String> busStatuss) {
+        this.busStatuss = busStatuss;
+    }
+
+    public TblBatchTableMapping getMappingInfo() {
+        return mappingInfo;
+    }
+
+    public void setMappingInfo(TblBatchTableMapping mappingInfo) {
+        this.mappingInfo = mappingInfo;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/mapper/TblBatchTableMappingMapper.java

@@ -0,0 +1,20 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTableMappingVO;
+
+
+/**
+ * 批处理同步表映射表
+ * @author zhouliang
+ * @date 2023-07-24
+ */
+@Mapper
+public interface TblBatchTableMappingMapper{
+
+    /** 查询单条 */
+    TblBatchTableMapping selectOneByMap(TblBatchTableMappingVO paramMap);
+   
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/mapper/TblBatchTaskMapper.java

@@ -0,0 +1,28 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTask;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
+
+
+/**
+ * 批处理任务表
+ * @author zhouliang
+ * @date 2023-07-05
+ */
+@Mapper
+public interface TblBatchTaskMapper{
+
+    /** 查询单条 */
+    TblBatchTaskVO selectOneByMap(TblBatchTaskVO paramMap);
+    
+    
+    /** 更新记录 */
+    int updateByMap(@Param("vo") TblBatchTask record,@Param("map") TblBatchTaskVO paramMap);
+
+    /** 重置全部任务 */
+    void updateResetAllBusStatus(@Param("vo") TblBatchTask record,@Param("map") TblBatchTaskVO paramMap);
+    
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/service/BatchTaskServiceImpl.java

@@ -0,0 +1,174 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.service;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.common.TblBatchTaskEnum;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTask;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTableMappingVO;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
+import com.jiuyv.sptcc.agile.batch.batchTask.mapper.TblBatchTableMappingMapper;
+import com.jiuyv.sptcc.agile.batch.batchTask.mapper.TblBatchTaskMapper;
+import com.jiuyv.sptcc.agile.batch.common.BaseTime;
+import com.jiuyv.sptcc.agile.batch.dao.ISysTimeBaseMapper;
+
+
+/**
+ * 批处理任务表
+ * @author zhouliang
+ * @date 2023-07-05
+ */
+@Service("batchTaskService")
+public class BatchTaskServiceImpl implements IBatchTaskService {
+    @Autowired
+    private TblBatchTaskMapper tblBatchTaskMapper;
+    @Autowired
+    private TblBatchTableMappingMapper tblBatchTableMappingMapper;
+    @Autowired
+    private ISysTimeBaseMapper sysTimeBaseMapper;
+    
+    @Override
+    public TblBatchTaskVO getDetailBatchTask(String taskNo) throws Exception {
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setTaskNo(taskNo);
+        TblBatchTaskVO batchTaskRecord = tblBatchTaskMapper.selectOneByMap(batchTaskParamMap);
+        if(batchTaskRecord!=null) {
+            //查询同步配置,这个有没有取决于业务自己的实现
+            TblBatchTableMappingVO batchTaskMappingParamMap = new TblBatchTableMappingVO();
+            batchTaskMappingParamMap.setTaskNo(taskNo);
+            TblBatchTableMapping batchTaskMappingRecord = tblBatchTableMappingMapper.selectOneByMap(batchTaskMappingParamMap);
+            batchTaskRecord.setMappingInfo(batchTaskMappingRecord);
+        }
+        return batchTaskRecord;
+    }
+
+    @Override
+    public void doBatchTaskReset(String taskNo) throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        TblBatchTask batchTaskRecord = new TblBatchTask();
+        batchTaskRecord.setBusStatus(TblBatchTaskEnum.BUS_STATUS.END.getCode());
+        batchTaskRecord.setUpdateTime(timeVO.getDate());
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setTaskNo(taskNo);
+        batchTaskParamMap.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
+        tblBatchTaskMapper.updateByMap(batchTaskRecord, batchTaskParamMap);
+    }
+    
+    @Override
+    public void doBatchTaskReset(List<String> excludedTaskNos) throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        TblBatchTask batchTaskRecord = new TblBatchTask();
+        batchTaskRecord.setBusStatus(TblBatchTaskEnum.BUS_STATUS.END.getCode());
+        batchTaskRecord.setUpdateTime(timeVO.getDate());
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
+        tblBatchTaskMapper.updateResetAllBusStatus(batchTaskRecord, batchTaskParamMap);
+    }
+
+    @Override
+    public boolean doBatchTaskStart(TblBatchTask task) throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        task.setCurrStartDate(timeVO.getDate());//当前开始时间
+        TblBatchTask batchTaskRecord = new TblBatchTask();
+        batchTaskRecord.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
+        batchTaskRecord.setRecToken(getNewRecToken());
+        batchTaskRecord.setCurrStartDate(timeVO.getDate());//当前开始时间
+        batchTaskRecord.setUpdateTime(timeVO.getDate());
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setTaskNo(task.getTaskNo());
+        batchTaskParamMap.setVersionNum(task.getVersionNum());//避免开始多次任务
+        batchTaskParamMap.setRecToken(task.getRecToken());//避免开始多次任务
+        List<String> busStatuss=new ArrayList<>();//非运行状态
+        busStatuss.add(TblBatchTaskEnum.BUS_STATUS.FINISH.getCode());
+        busStatuss.add(TblBatchTaskEnum.BUS_STATUS.UNFINISH.getCode());
+        busStatuss.add(TblBatchTaskEnum.BUS_STATUS.END.getCode());
+        batchTaskParamMap.setBusStatuss(busStatuss);
+        int num = tblBatchTaskMapper.updateByMap(batchTaskRecord, batchTaskParamMap);
+        return num!=0;
+    }
+
+    @Override
+    public void doBatchTaskFinish(TblBatchTask task) throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        TblBatchTask batchTaskRecord = new TblBatchTask();
+        batchTaskRecord.setBusStatus(TblBatchTaskEnum.BUS_STATUS.FINISH.getCode());
+        batchTaskRecord.setRecToken(getNewRecToken());
+        //结束时
+        batchTaskRecord.setFailureConditions("");
+        batchTaskRecord.setPreStartDate(task.getCurrStartDate());
+        batchTaskRecord.setPreEndDate(timeVO.getDate());
+        batchTaskRecord.setPreTotalTime(getTotalTime(batchTaskRecord.getPreStartDate(),batchTaskRecord.getPreEndDate()));
+        batchTaskRecord.setUpdateTime(timeVO.getDate());
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setTaskNo(task.getTaskNo());
+        batchTaskParamMap.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
+        tblBatchTaskMapper.updateByMap(batchTaskRecord, batchTaskParamMap);
+    }
+
+    @Override
+    public void doBatchTaskUnFinish(TblBatchTask task) throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        TblBatchTask batchTaskRecord = new TblBatchTask();
+        batchTaskRecord.setBusStatus(TblBatchTaskEnum.BUS_STATUS.UNFINISH.getCode());
+        batchTaskRecord.setRecToken(getNewRecToken());
+        //结束时，如果需要根据条件重新跑就存到FailureConditions,比如标志、日期、id等等
+        batchTaskRecord.setFailureConditions(StringUtils.isNotBlank(task.getFailureConditions())?task.getFailureConditions():"");
+        batchTaskRecord.setPreStartDate(task.getCurrStartDate());
+        batchTaskRecord.setPreEndDate(timeVO.getDate());
+        batchTaskRecord.setPreTotalTime(getTotalTime(batchTaskRecord.getPreStartDate(),batchTaskRecord.getPreEndDate()));
+        batchTaskRecord.setUpdateTime(timeVO.getDate());
+        TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
+        batchTaskParamMap.setTaskNo(task.getTaskNo());
+        batchTaskParamMap.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
+        tblBatchTaskMapper.updateByMap(batchTaskRecord, batchTaskParamMap);
+    }
+    
+    private String getNewRecToken() {
+       return UUID.randomUUID().toString().substring(0,8);
+    }
+    /**
+     * 计算任务耗时
+     * @param start
+     * @param end
+     * @return
+     */
+    private String getTotalTime(Date start,Date end) {
+        // 获取日期间的时间差
+        long diff = end.getTime() - start.getTime();
+        // 计算小时、分钟和秒
+        long totaltime=(diff / 1000);
+        String totaltimeUnit="秒";
+        if(totaltime>=60) {
+            totaltime=totaltime/60;
+            totaltimeUnit="分";
+        }
+        if(totaltime>=60) {
+            totaltime=totaltime/60;
+            totaltimeUnit="小时";
+        }
+        return totaltime+totaltimeUnit;
+     }
+    
+    
+    
+    @PostConstruct
+    public void taskInit() {
+        //默认项目重启就应该重置任务状态，没完成的任务状态肯定有问题
+        List<String> excludedTaskNos=new ArrayList<>();
+        //如果不需要重置的自行排除
+        try {
+            this.doBatchTaskReset(excludedTaskNos);
+        } catch (Exception e) {
+            //不报错
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/batchTask/service/IBatchTaskService.java

@@ -0,0 +1,57 @@
+package com.jiuyv.sptcc.agile.batch.batchTask.service;
+
+import java.util.List;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTask;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
+
+
+/**
+ * 批处理任务表
+ * @author zhouliang
+ * @date 2023-07-05
+ */
+public interface IBatchTaskService {
+
+    /** 获取任务详情*/
+    public TblBatchTaskVO getDetailBatchTask(String taskNo) throws Exception;
+
+    /** 
+     * 重置任务 
+     * 让任务回到未运行状态
+     * @param taskNo
+     * @throws Exception
+     */
+    public void doBatchTaskReset(String taskNo) throws Exception;
+    /** 
+     * 重置全部任务 
+     * 让任务回到未运行状态
+     * @param excludedTaskNos 排除任务
+     * @throws Exception
+     */
+    public void doBatchTaskReset(List<String> excludedTaskNos) throws Exception;
+    /** 
+     * 开始任务
+     * 任务如何重跑，由业务代码实现
+     * @param taskNo
+     * @param versionNum
+     * @param recToken
+     * @return 如果任务已在运行则返回false，反之true
+     * @throws Exception
+     */
+    public boolean doBatchTaskStart(TblBatchTask task) throws Exception;
+    
+    /**
+     * 正常结束任务
+     * 任务如何重跑，由业务代码实现
+     */
+    public void doBatchTaskFinish(TblBatchTask task) throws Exception;
+    
+    /** 
+     * 异常结束任务
+     * 任务如何重跑，由业务代码实现
+     * @param taskNo
+     * @throws Exception
+     */
+    public void doBatchTaskUnFinish(TblBatchTask task) throws Exception;
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/common/BaseTime.java

@@ -0,0 +1,85 @@
+package com.jiuyv.sptcc.agile.batch.common;
+
+import java.time.Instant;
+import java.util.Date;
+
+/**
+ * 时间对象
+ * @author zhouliang
+ *
+ */
+public class BaseTime implements java.io.Serializable {
+
+    /** default Serial Version UID*/
+    private static final long serialVersionUID = 1L;
+    
+    /** 当前时区  */
+    private String timeZone ="+08:00";
+    
+    /** 当前时区 YYYY-MM-DD  */
+    private String dateDay;
+    
+    /** 当前时区 YYYY-MM-DD HH:MM:SS */
+    private String dateTime;
+    
+    /** 当前时区日期  */
+    private Date date;
+    
+    /** UTC-0 带时区时间  */
+    private Instant utcTime;
+    
+    /** UTC-0 带时区时间  */
+    private String utcTimeStr;
+
+    public String getTimeZone() {
+        return timeZone;
+    }
+
+    public void setTimeZone(String timeZone) {
+        this.timeZone = timeZone;
+    }
+
+    public String getDateDay() {
+        return dateDay;
+    }
+
+    public void setDateDay(String dateDay) {
+        this.dateDay = dateDay;
+    }
+
+    public String getDateTime() {
+        return dateTime;
+    }
+
+    public void setDateTime(String dateTime) {
+        this.dateTime = dateTime;
+    }
+
+    public Date getDate() {
+        return date;
+    }
+
+    public void setDate(Date date) {
+        this.date = date;
+    }
+
+    public Instant getUtcTime() {
+        return utcTime;
+    }
+
+    public void setUtcTime(Instant utcTime) {
+        this.utcTime = utcTime;
+    }
+
+    public String getUtcTimeStr() {
+        return utcTimeStr;
+    }
+
+    public void setUtcTimeStr(String utcTimeStr) {
+        this.utcTimeStr = utcTimeStr;
+    }
+    
+    public String getYearMonth() {
+        return dateDay.substring(0,7);
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/common/JsonUtil.java

@@ -0,0 +1,153 @@
+package com.jiuyv.sptcc.agile.batch.common;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.core.JsonGenerator.Feature;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+
+/**
+ * 
+ * @author zhouliang
+ *
+ */
+public abstract class JsonUtil {
+
+    /**
+     * The Constant LOGGER.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(JsonUtil.class);
+
+    private JsonUtil() {
+        throw new IllegalStateException("Utility class");
+    }
+    
+    /**
+     * The object mapper.
+     */
+    private static ObjectMapper objectMapper = new ObjectMapper();
+
+    static {
+        objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        objectMapper.setSerializationInclusion(Include.NON_NULL);
+        objectMapper.configure(Feature.WRITE_BIGDECIMAL_AS_PLAIN, true);
+        objectMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
+        objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);//允许单引号
+    }
+    public static ObjectMapper JsonMapper(){
+        return objectMapper;
+    }
+
+    /**
+     * 转为json字符串
+     *
+     * @param object the object
+     * @return the string
+     */
+    public static String toJSONString(Object object) {
+        try {
+            return objectMapper.writeValueAsString(object);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return "";
+        }
+    }
+
+    /**
+     * json字符串转为对象
+     * @param <T>
+     * @param json
+     * @param clz
+     * @return
+     */
+    public static <T> T json2Bean(String json, Class<T> clz) {
+        try {
+            return objectMapper.readValue(json, clz);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return null;
+        }
+    }
+    /**
+     * json字符串转为对象
+     * @param <T>
+     * @param json
+     * @param clz
+     * @return
+     */
+    public static <T> T json2Bean(String json, JavaType clz) {
+        try {
+            return objectMapper.readValue(json, clz);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return null;
+        }
+    }
+
+    /**
+     * 转为对象集合
+     * @param <T>
+     * @param json
+     * @param clz
+     * @return
+     */
+    public static <T> List<T> json2List(String json, Class<T> clz) {
+        try {
+            JavaType javaType = getCollectionType(ArrayList.class, clz);
+            return objectMapper.readValue(json, javaType);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return new ArrayList<>();
+        }
+    }
+
+    /**
+     * 获取泛型的Collection Type
+     *
+     * @param collectionClass 泛型的Collection
+     * @param elementClasses  元素类
+     * @return JavaType Java类型
+     * @since 1.0
+     */
+    public static JavaType getCollectionType(Class<?> collectionClass, Class<?>... elementClasses) {
+        return objectMapper.getTypeFactory().constructParametricType(collectionClass, elementClasses);
+    }
+    
+    /**
+     * json字符转ArrayNode
+     * @param json
+     * @return
+     */
+    public static ArrayNode parseArray(String json) {
+        try {
+            return (ArrayNode) objectMapper.readTree(json);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return null;
+        }
+    }
+    /**
+     * json字符转ObjectNode
+     * @param json
+     * @return
+     */
+    public static ObjectNode parseObject(String json) {
+        try {
+            return (ObjectNode) objectMapper.readTree(json);
+        } catch (Exception e) {
+            LOGGER.error("convert failed", e);
+            return null;
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/common/R.java

@@ -0,0 +1,135 @@
+package com.jiuyv.sptcc.agile.batch.common;
+
+import java.io.Serializable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpStatus;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+/**
+ * 响应信息主体
+ *
+ * @author admin
+ */
+public class R<T> implements Serializable
+{
+    private static final Logger log = LoggerFactory.getLogger(R.class);
+    
+    private static final long serialVersionUID = 1L;
+
+    /** 成功 */
+    public static final int SUCCESS = 200;
+
+    /** 失败 */
+    public static final int FAIL = 500;
+
+    /**
+     * 应答码,默认200为成功
+     */
+    private int code;
+
+    /**
+     * 应答描述
+     */
+    private String msg;
+
+    /**
+     * 返回数据
+     */
+    private T data;
+
+    public static <T> R<T> ok()
+    {
+        return restResult(null, SUCCESS, "操作成功");
+    }
+
+    public static <T> R<T> ok(T data)
+    {
+        return restResult(data, SUCCESS, "操作成功");
+    }
+
+    public static <T> R<T> ok(T data, String msg)
+    {
+        return restResult(data, SUCCESS, msg);
+    }
+
+    public static <T> R<T> fail()
+    {
+        return restResult(null, FAIL, "操作失败");
+    }
+
+    public static <T> R<T> fail(String msg)
+    {
+        return restResult(null, FAIL, msg);
+    }
+
+    public static <T> R<T> fail(T data)
+    {
+        return restResult(data, FAIL, "操作失败");
+    }
+
+    public static <T> R<T> fail(T data, String msg)
+    {
+        return restResult(data, FAIL, msg);
+    }
+
+    public static <T> R<T> fail(int code, String msg)
+    {
+        return restResult(null, code, msg);
+    }
+
+    private static <T> R<T> restResult(T data, int code, String msg)
+    {
+        if(SUCCESS!=code) {
+            //异常要输出
+            log.info("Return Business Exception >> code={}, msg={}",code, msg);
+        }
+        R<T> apiResult = new R<>();
+        apiResult.setCode(code);
+        apiResult.setData(data);
+        apiResult.setMsg(msg);
+        return apiResult;
+    }
+
+    public int getCode()
+    {
+        return code;
+    }
+
+    public void setCode(int code)
+    {
+        this.code = code;
+    }
+
+    public String getMsg()
+    {
+        return msg;
+    }
+
+    public void setMsg(String msg)
+    {
+        this.msg = msg;
+    }
+
+    public T getData()
+    {
+        return data;
+    }
+
+    public void setData(T data)
+    {
+        this.data = data;
+    }
+    
+    /**
+     * 这里只处理200成功，如果不是则那么不要使用，自行单独判断
+     * @return
+     */
+    @JsonIgnore
+    public boolean isSuccess()
+    {
+        return code==SUCCESS;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/dao/BaseDao.java

@@ -0,0 +1,10 @@
+package com.jiuyv.sptcc.agile.batch.dao;
+
+/**
+ * @ClassName : BaseDao
+ * @Description : 公告类
+ * @Author : sky
+ * @Date: 2023-06-07 15:27
+ */
+public interface BaseDao {
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/dao/ISysTimeBaseMapper.java

@@ -0,0 +1,15 @@
+package com.jiuyv.sptcc.agile.batch.dao;
+
+import org.apache.ibatis.annotations.Mapper;
+
+import com.jiuyv.sptcc.agile.batch.common.BaseTime;
+
+@Mapper
+public interface ISysTimeBaseMapper {
+    
+    /**
+     * 获取系统当前时间-yyyyMMddHHmmss
+     * @return
+     */
+    BaseTime selectSysCurrentTime();
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/framework/GlobalExceptionHandler.java

@@ -0,0 +1,43 @@
+package com.jiuyv.sptcc.agile.batch.framework;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.RestControllerAdvice;
+
+import com.jiuyv.sptcc.agile.batch.common.R;
+
+/**
+ * 全局异常处理器
+ *
+ * @author admin
+ */
+@RestControllerAdvice
+public class GlobalExceptionHandler {
+    private static final Logger log = LoggerFactory.getLogger(GlobalExceptionHandler.class);
+    
+    /**
+     * 拦截未知的运行时异常
+     */
+    @ExceptionHandler({RuntimeException.class})
+    public R<Object> handleRuntimeException(RuntimeException e, HttpServletRequest request, HttpServletResponse response) {
+        String requestURI = request.getRequestURI();
+        log.error("请求地址'{}',发生未知异常.", requestURI, e);
+
+        return R.fail("系统忙，请稍后再试");
+    }
+
+    /**
+     * 系统异常
+     */
+    @ExceptionHandler(Exception.class)
+    public R<Object> handleException(Exception e, HttpServletRequest request, HttpServletResponse response) {
+        response.setStatus(301);
+        String requestURI = request.getRequestURI();
+        log.error("请求地址'{}',发生系统异常.", requestURI, e);
+        return R.fail(e.getMessage());
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/framework/SecurityConfig.java

@@ -0,0 +1,47 @@
+package com.jiuyv.sptcc.agile.batch.framework;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+
+/**
+ * spring security配置
+ * 
+ * @author admin
+ */
+@EnableGlobalMethodSecurity(prePostEnabled = true, securedEnabled = true)
+public class SecurityConfig extends WebSecurityConfigurerAdapter
+{
+    @Autowired
+    private Environment env;
+
+    @Override
+    protected void configure(HttpSecurity httpSecurity) throws Exception
+    {
+        String actuatorPath = env.getProperty("management.endpoints.web.base-path");
+        if(StringUtils.isBlank(actuatorPath)) {
+            actuatorPath = "/actuator" ;
+        }
+        //优化一下根路径
+        String servletPath = env.getProperty("server.servlet.context-path");
+        if(StringUtils.isNotBlank(servletPath)) {
+            if(servletPath.endsWith("/")) {
+                servletPath=servletPath.substring(0,servletPath.length()-1);
+            }
+            servletPath=servletPath.replaceAll("[^/]+", "**");
+        }else {
+            servletPath="";
+        }
+        httpSecurity
+               .csrf().disable()
+                // 过滤请求
+                .authorizeRequests()
+                .antMatchers(servletPath+actuatorPath+"/shutdown").access("hasIpAddress(\"127.0.0.1\")")
+                .antMatchers(servletPath+actuatorPath+"/**").authenticated()
+                .anyRequest().permitAll()
+                .and().httpBasic();
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/DDsProperties.java

@@ -0,0 +1,119 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common;
+
+/**
+ * 数据库基础配置类
+ * @author zhouliang
+ *
+ */
+public class DDsProperties {
+    
+    /**
+     * url地址
+     */
+    private String url;
+    /**
+     * 账户
+     */
+    private String username;
+    /**
+     * 密码
+     */
+    private String password;
+    /**
+     * 驱动类名
+     */
+    private String driverClassName;
+    /**
+     * 驱动类名
+     */
+    private String confPath;
+    /**
+     * 一次写入数量，文件/数据库都是一样
+     */
+    private Integer singleWriteNumber;
+    
+    /**
+     * 本地存放路径
+     */
+    private String readWritePath;
+    
+    /**
+     * 字段分隔符号
+     */
+    private String fieldSeparator;
+    
+    
+    /**
+     * @return the url
+     */
+    public String getUrl() {
+        return url;
+    }
+    /**
+     * @param url the url to set
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+    /**
+     * @return the username
+     */
+    public String getUsername() {
+        return username;
+    }
+    /**
+     * @param username the username to set
+     */
+    public void setUsername(String username) {
+        this.username = username;
+    }
+    /**
+     * @return the password
+     */
+    public String getPassword() {
+        return password;
+    }
+    /**
+     * @param password the password to set
+     */
+    public void setPassword(String password) {
+        this.password = password;
+    }
+    /**
+     * @return the driverClassName
+     */
+    public String getDriverClassName() {
+        return driverClassName;
+    }
+    /**
+     * @param driverClassName the driverClassName to set
+     */
+    public void setDriverClassName(String driverClassName) {
+        this.driverClassName = driverClassName;
+    }
+    public String getConfPath() {
+        return confPath;
+    }
+    public void setConfPath(String confPath) {
+        this.confPath = confPath;
+    }
+    public Integer getSingleWriteNumber() {
+        return singleWriteNumber;
+    }
+    public void setSingleWriteNumber(Integer singleWriteNumber) {
+        this.singleWriteNumber = singleWriteNumber;
+    }
+    
+    public String getFieldSeparator() {
+        return fieldSeparator;
+    }
+    public void setFieldSeparator(String fieldSeparator) {
+        this.fieldSeparator = fieldSeparator;
+    }
+    public String getReadWritePath() {
+        return readWritePath;
+    }
+    public void setReadWritePath(String readWritePath) {
+        this.readWritePath = readWritePath;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/SqlHandlerUtilx.java

@@ -0,0 +1,326 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common;
+
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ibatis.type.BigDecimalTypeHandler;
+import org.apache.ibatis.type.LongTypeHandler;
+import org.apache.ibatis.type.SqlDateTypeHandler;
+import org.apache.ibatis.type.SqlTimestampTypeHandler;
+import org.apache.ibatis.type.StringTypeHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.jiuyv.sptcc.agile.batch.common.JsonUtil;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model.SqlHandlerResultVO;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model.SqlHandlerTypeVO;
+
+/**
+ * 动态拼接处理SQL
+ * @author zhouliang
+ *
+ */
+public class SqlHandlerUtilx {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(SqlHandlerUtilx.class);
+    
+    private SqlHandlerUtilx() {
+        throw new IllegalStateException("Utility class");
+    }
+    //把SQL关系配置转换为实际的sql语句
+    public static SqlHandlerResultVO convertJsonToSql(String fromQuerySql, String toTableName, String cfgJson) throws Exception {
+        SqlHandlerResultVO vo=new SqlHandlerResultVO();
+        
+        fromQuerySql=fromQuerySql.replace("\n", " ");
+        //有" from "则认为是完整的sql
+        boolean customSelect=fromQuerySql.toLowerCase().contains(" from ");
+        
+        Map<String,SqlHandlerTypeVO> fieldMapping=new LinkedHashMap<>();
+        List<String> values = new ArrayList<>();
+        //查询sql
+        StringBuilder sqlSelectBuilder = new StringBuilder();
+        sqlSelectBuilder.append("SELECT ");
+        //插入sql
+        StringBuilder sqlInsertBuilder = new StringBuilder();
+        sqlInsertBuilder.append("INSERT INTO ").append(toTableName).append(" (");
+        
+        //*表示所有的字段和类型完全一致
+        boolean allIgnoreFlag ="*".equals(cfgJson);
+
+        if(!allIgnoreFlag) {
+            ObjectNode obj = JsonUtil.parseObject(cfgJson);
+            Iterator<Entry<String, JsonNode>> fields = obj.fields();
+            while (fields.hasNext()) {
+                Entry<String, JsonNode> entry = fields.next();
+                String key = entry.getKey().trim();
+                String value = entry.getValue().asText();
+                if(!customSelect) {
+                    sqlSelectBuilder.append(key+",");
+                }
+                
+                String[] vals=value.trim().split("\\|");
+                sqlInsertBuilder.append(vals[0]+",");
+                SqlHandlerTypeVO convertVO=new SqlHandlerTypeVO();
+                convertVO.setColumnCode(vals[0]);
+                if(vals.length>1) {
+                    convertVO.setConvertType(vals[1]);
+                }
+                fieldMapping.put(key, convertVO);
+                values.add("?");
+            }
+        }
+        
+        if(!customSelect) {
+            if(!allIgnoreFlag) {
+                sqlSelectBuilder.deleteCharAt(sqlSelectBuilder.length() - 1);//去除多余的逗号
+                sqlSelectBuilder.append(" FROM " + fromQuerySql);
+                vo.setSelectSql(sqlSelectBuilder.toString());
+            }else {
+                vo.setSelectSql("SELECT * FROM " + fromQuerySql);
+            }
+        }else {
+            vo.setSelectSql(fromQuerySql);
+        }
+        if(!allIgnoreFlag) {
+            sqlInsertBuilder.deleteCharAt(sqlInsertBuilder.length() - 1);//去除多余的逗号
+            sqlInsertBuilder.append(") VALUES (");
+            sqlInsertBuilder.append(StringUtils.join(values,",")).append(")");
+            vo.setInsertSql(sqlInsertBuilder.toString());
+        }
+        
+        vo.setFieldMapping(fieldMapping);
+        return vo;
+    }
+    
+    /**
+     * 字段转换为insert语句
+     * @param list
+     * @param toTableName
+     * @return
+     */
+    public static String convertInsertSql(Collection<String> cols, String toTableName,Integer number) {
+        if(number==null || number.intValue()==0) {
+            number=1;
+        }
+        //插入sql
+        StringBuilder sqlInsertBuilder = new StringBuilder();
+        sqlInsertBuilder.append("INSERT INTO ").append(toTableName).append(" (");
+        List<String> values = new ArrayList<>();
+        for (String col:cols) {
+            sqlInsertBuilder.append(col+",");
+            values.add("?");
+        }
+        sqlInsertBuilder.deleteCharAt(sqlInsertBuilder.length() - 1);//去除多余的逗号
+        sqlInsertBuilder.append(") VALUES ");
+//        System.out.print(sqlInsertBuilder.toString());
+        while(number>0) {
+            sqlInsertBuilder.append("("+StringUtils.join(values,",")).append(")");
+            number--;
+            if(number>0) {
+                sqlInsertBuilder.append(",");
+            }
+        }
+        return sqlInsertBuilder.toString();
+    }
+    
+    /**
+     *替换自定义参数{XXXX}
+     * @param sql
+     * @param params
+     * @return
+     * @throws Exception
+     */
+    public static String replaceSqlCustomParams(String sql,Map<String,String> params) throws Exception {
+        if(StringUtils.isBlank(sql)) {
+            return null;
+        }
+        for(Entry<String, String> ex:params.entrySet()) {
+            sql=sql.replaceAll("\\{"+ex.getKey()+"\\}", ex.getValue());
+        }
+        return sql;
+    }
+    
+    /**
+     * 转换数据库字段类型，一般的也不用转（只是为了特殊类型，比如Date转string可能需要）
+     * @param resultSet
+     * @param colname 字段名
+     * @param type 转换类型
+     * @param stringFlag  true全部转为String
+     * @return
+     * @throws Exception
+     */
+    public static Object convertType(ResultSet resultSet,String colname, String type,boolean stringFlag) throws Exception {
+        if(StringUtils.isBlank(colname)) {
+            return null;
+        }
+        Object value=resultSet.getObject(colname);
+        if(value!=null) {
+            try {
+//                System.out.println("colname>>"+colname+"="+value.getClass());
+                //不同数据库之间才涉及类型需要统一，基本只管日期
+                if(!stringFlag && SyncDataConstants.CONVERT_TYPE_TO_DATE.equalsIgnoreCase(type)) {
+                    value = new SqlDateTypeHandler().getResult(resultSet,colname);
+                }
+                else if(!stringFlag && SyncDataConstants.CONVERT_TYPE_TO_INSTANT.equalsIgnoreCase(type)) {
+                    value = new SqlTimestampTypeHandler().getResult(resultSet,colname);
+                }
+                else if(!stringFlag && SyncDataConstants.CONVERT_TYPE_TO_INTEGER.equalsIgnoreCase(type)) {
+                    value = new SqlTimestampTypeHandler().getResult(resultSet,colname);
+                }
+                else if(!stringFlag && SyncDataConstants.CONVERT_TYPE_TO_BIGDECIMAL.equalsIgnoreCase(type)) {
+                    value = new BigDecimalTypeHandler().getResult(resultSet,colname);
+                }
+                else if(!stringFlag && SyncDataConstants.CONVERT_TYPE_TO_LONG.equalsIgnoreCase(type)) {
+                    value = new LongTypeHandler().getResult(resultSet,colname);
+                }
+                else if(stringFlag || SyncDataConstants.CONVERT_TYPE_TO_STRING.equalsIgnoreCase(type)){
+                    value = new StringTypeHandler().getResult(resultSet,colname);
+                }
+                //System.out.println(type);//测试看
+            }catch (Exception e) {
+                if(LOGGER.isDebugEnabled()) {
+                    LOGGER.debug("convertType error>>colname={},{}",colname,e.getMessage());
+                }
+                //如果报错，表示类型不能相互转换或数据有问题
+                value = value.toString();
+            }
+        }
+        return value;
+    }
+    
+    /*
+    public static void main(String[] args) throws Exception {
+       SqlHandlerResultVO vo = convertJsonToSql("SELECT * FROM TBL_USER WHERE ID='{currDate}'", "TBL_USER1"
+               , "{'field1':'fieldA','field2':'fieldB|String'}");
+       
+//       Map<String,String> sqlParams=new HashMap<>();
+//       sqlParams.put("currDate", "2022-02-12");//目前都是按天处理,如果有他条件再加
+//       
+//       System.out.println(JsonUtil.toJSONString(vo.getFieldMapping()));
+//       System.out.println(SqlHandlerUtilx.replaceSqlCustomParams(vo.getSelectSql(),sqlParams));
+//       System.out.println();
+//       System.out.println(SqlHandlerUtilx.replaceSqlCustomParams(vo.getInsertSql(),sqlParams));
+//       System.out.println();
+
+//       Connection connection = null;
+//       try {
+//           // 加载JDBC驱动
+//           Class.forName("org.postgresql.Driver");
+//           // 获取JDBC连接
+//           connection = DriverManager.getConnection("jdbc:postgresql://172.16.12.105:5432/keliubao", "postgres","postgres");
+//           Statement st = connection.createStatement();
+//           st.execute("SELECT * FROM tbl_sys_user where user_id=4 LIMIT 1");
+//           ResultSet resultSet = st.getResultSet();
+//           if(resultSet!=null) {
+//               while(resultSet.next()) {
+//                   System.out.println(resultSet.getString("email"));
+//                   System.out.println(convertType(resultSet, "email", "Instant", false));
+//               }
+//           }
+//           st.close();
+//       }catch (Exception e) {
+//           LOGGER.info("Create connection failed : "  + e.getMessage());
+//       }finally {
+//           if(null!=connection) {
+//               connection.close();
+//           }
+//       }
+       
+       
+       String task="INSERT INTO tbl_batch_task (task_no, task_title, bus_status, data_status, update_time) "
+               + "VALUES ('%s', '%s', 'end', '00', '2023-07-26 13:34:52');";
+        
+       String taskstr="INSERT INTO tbl_batch_table_mapping (task_no, remote_table_sql, remote_db_name, remote_days, local_table, local_db_name, local_pre_sql, mapping_json, data_status, update_time) "
+               + "VALUES ('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s', '00', '2023-07-26 11:08:08');"
+               + "";
+       String title="久事云";
+       String taskNo="SYNC_SJZTHIVE_dws_product_consum_month";
+       String readSql="SELECT * FROM sptcc.dws_product_consum_month where month_name={currDate} " ;//where workdate={currDate}
+       String readDB="sjztHiveDs";
+       String days="1";
+       String writeTable="ods_product_consum_month";
+       String writeDB="klbHiveDs";
+       String writePreSql="";
+       ObjectNode conf=JsonUtil.JsonMapper().createObjectNode();
+       
+       //转换
+       String[] fields1="card_no,rechamt_range_first,pro_name_hfreq,city_name_hfreq,bus_hfreq,metro_hfreq,freq_range,month_name"
+               .trim().split("( +)|(,)");
+       String[] fieldstype1="string,string,string,string,string,string,string,string"
+               .trim().split("( +)|(,)");
+       
+       String[] fields2="card_no,rechamt_range_first,pro_name_hfreq,city_name_hfreq,bus_hfreq,metro_hfreq,freq_range,month_name"
+               .trim().split("( +)|(,)");
+       String[] fieldstype2="string,string,string,string,string,string,string,string"
+               .trim().split("( +)|(,)");
+       
+       Map<String,String> map=new HashMap<>();
+       Map<String,String> maptype=new HashMap<>();
+       for(int i=0;i<fields2.length;i++) {
+           map.put(fields2[i].toLowerCase(), fields2[i].toLowerCase());
+           map.put(fields2[i].replace("_", "").toLowerCase(), fields2[i].toLowerCase());
+           maptype.put(fields2[i].toLowerCase(), fieldstype2[i].toLowerCase());
+           maptype.put(fields2[i].replace("_", "").toLowerCase(), fieldstype2[i].toLowerCase());
+       }
+       List<String> haslist=new ArrayList<>();
+       for(int i=0;i<fields1.length;i++) {
+           String t = map.get(fields1[i].toLowerCase());
+           if(t==null) {
+               conf.put(fields1[i], "未找到");
+           }else {
+               haslist.add(t);
+               String type1 =fieldstype1[i];
+               String type2 = maptype.get(fields1[i].toLowerCase());
+               String typex = type2;
+               if(typex.toLowerCase().startsWith("varchar") || typex.toLowerCase().startsWith("char") ) {
+                   typex="String";
+               }
+               if(typex.toLowerCase().startsWith("int")) {
+                   typex="Integer";
+               }
+               if(type1.toLowerCase().startsWith("varchar")  || type1.toLowerCase().startsWith("char") ) {
+                   type1="String";
+               }
+               if(type1.toLowerCase().startsWith("int")) {
+                   type1="Integer";
+               }
+//               System.out.println("类型>>"+type1 +"   "+typex);
+               if(!type1.equalsIgnoreCase(typex)){
+                   if(typex.equalsIgnoreCase("timestamp")) {
+                       typex="Instant";
+                   }else if(typex.equalsIgnoreCase("date")) {
+                       typex="Date";
+                   }else if(typex.equalsIgnoreCase("decimal")) {
+                       typex="BigDecimal";
+                   }else if(typex.equalsIgnoreCase("int")) {
+                       typex="Integer";
+                   }
+               }else {
+                   typex=null;
+               }
+               conf.put(fields1[i], typex==null?t:(t+"|"+typex));
+           }
+       }
+       for(String x:fields2) {
+           if(!haslist.contains(x.toLowerCase())) {
+               System.out.println("本地不匹配字段>>"+x);
+           }
+       }
+       task =String.format(task, taskNo, "同步"+title+"表");
+       taskstr =String.format(taskstr, taskNo,readSql,readDB
+               ,days,writeTable,writeDB, writePreSql, conf.toString());
+       System.out.println(task);
+       System.out.println();
+       System.out.println(taskstr);
+    }*/
+    
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/SyncDataConstants.java

@@ -0,0 +1,55 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ *  数据同步相关常量
+ */
+public class SyncDataConstants
+{
+    private SyncDataConstants() {
+        throw new IllegalStateException("Utility class");
+    }
+    
+    //这个主要用于把数据直接写到文件，配置方式和写到数据库一致
+    /** 数据源名称-txt文件 */
+    public static final String DB_NAME_FILE_TXT="txt";
+    /** 数据源名称-csv文件(和txt其实一样) */
+    public static final String DB_NAME_FILE_CSV="csv";
+    
+    /** 数据源名称-久事云，要特殊处理 */
+    public static final String DB_NAME_JSY_HIVE_DS="jsyHiveDs";
+    
+    /** Kerberos的执行命令 */
+    public static final String SHELL_KERBEROS_KINIT="kinit %s";
+    
+    /** 特殊标志,写文件需要标题 */
+    public static final String FLAG_NEED_TITLE="title";
+    
+    /** 临时文件后缀 */
+    public static final String TEMP_FILE_EXTENSION ="tmp";
+    
+    
+    /** 获取sql中的表*/
+    public static final Pattern FROM_TABLE_RULE = Pattern.compile("\\b(FROM|DESC)\\b ([A-Z0-9_]+\\.){0,2}([A-Z]+[A-Z0-9_]*)\\b", Pattern.CASE_INSENSITIVE);
+    public static String getTablecode(String sql) {
+        String tablecode="";
+        Matcher m = SyncDataConstants.FROM_TABLE_RULE.matcher(sql.replace("\n", " "));
+        if(m.find()) {
+           tablecode= m.group(3)+"-";
+        }
+        return tablecode;
+    }
+    /** 获取文件名*/
+    public static final Pattern FILE_NAME_RULE = Pattern.compile("(^.*)(\\.[A-Z0-9]+$)", Pattern.CASE_INSENSITIVE);
+    
+    
+    //类型转换
+    public static final String CONVERT_TYPE_TO_DATE="Date";
+    public static final String CONVERT_TYPE_TO_INSTANT="Instant";
+    public static final String CONVERT_TYPE_TO_INTEGER="Integer";
+    public static final String CONVERT_TYPE_TO_LONG="Long";
+    public static final String CONVERT_TYPE_TO_BIGDECIMAL="BigDecimal";
+    public static final String CONVERT_TYPE_TO_STRING="String";
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/jsydb/JsyHiveJDBCBuilder.java

@@ -0,0 +1,179 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.jsydb;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class JsyHiveJDBCBuilder {
+    private static final Logger LOGGER = LoggerFactory.getLogger(JsyHiveJDBCBuilder.class);
+    
+  private static final String ZOOKEEPER_DEFAULT_LOGIN_CONTEXT_NAME = "Client";
+  private static final String ZOOKEEPER_SERVER_PRINCIPAL_KEY = "zookeeper.server.principal";    
+  private static final String ZOOKEEPER_DEFAULT_SERVER_PRINCIPAL = "zookeeper/";
+
+  public static String getUserRealm() {
+    String serverRealm = System.getProperty("SERVER_REALM");
+    if (StringUtils.isNotBlank(serverRealm)) {
+        serverRealm = "hadoop." + serverRealm.toLowerCase();
+    } else {
+      serverRealm = KerberosUtilx.getKrb5DomainRealm();
+      if(StringUtils.isNotBlank(serverRealm)) {
+          serverRealm = "hadoop." + serverRealm.toLowerCase();
+      } else {
+          serverRealm = "hadoop";
+      }
+    }
+    LOGGER.info("getUserRealm>>"+serverRealm);
+    return serverRealm;
+  }
+   
+  public static String urlBuilder(String username,String password,String confPath) throws IOException{
+      Properties clientInfo = null;
+//      String userdir = System.getProperty("user.dir") + File.separator + "conf" + File.separator;
+      String userdir = confPath.endsWith(File.separator)?confPath:confPath + File.separator;
+      
+      clientInfo = new Properties();
+      //"hiveclient.properties"为客户端配置文件，如果使用多实例特性，需要把该文件换成对应实例客户端下的"hiveclient.properties"
+      //"hiveclient.properties"文件位置在对应实例客户端安裝包解压目录下的config目录下
+      String hiveclientProp = userdir + "hiveclient.properties" ;
+      
+      File propertiesFile = new File(hiveclientProp);
+      try(InputStream fileInputStream = new FileInputStream(propertiesFile);){
+        clientInfo.load(fileInputStream);
+      }catch (Exception e) {
+        throw new IOException(e);
+      }
+      //zkQuorum获取后的格式为"xxx.xxx.xxx.xxx:24002,xxx.xxx.xxx.xxx:24002,xxx.xxx.xxx.xxx:24002";
+      //"xxx.xxx.xxx.xxx"为集群中ZooKeeper所在节点的业务IP，端口默认是24002
+      String zkQuorum =  clientInfo.getProperty("zk.quorum");
+      String auth = clientInfo.getProperty("auth");
+      String saslQop = clientInfo.getProperty("sasl.qop");
+      String zooKeeperNamespace = clientInfo.getProperty("zooKeeperNamespace");
+      String serviceDiscoveryMode = clientInfo.getProperty("serviceDiscoveryMode");
+      String principal = clientInfo.getProperty("principal"); 
+      
+      String userKeytabFile=null;
+      String krb5File=null;
+      if ("KERBEROS".equalsIgnoreCase(auth)) {
+        // 设置客户端的keytab和krb5文件路径
+        userKeytabFile = "conf/user.keytab";
+        krb5File = userdir + "krb5.conf";
+        System.setProperty("java.security.krb5.conf", krb5File);
+        String serverPrincipal = ZOOKEEPER_DEFAULT_SERVER_PRINCIPAL + getUserRealm();
+        System.setProperty(ZOOKEEPER_SERVER_PRINCIPAL_KEY, serverPrincipal);
+      }
+      
+      // 拼接JDBC URL
+      StringBuilder sBuilder = new StringBuilder("jdbc:hive2://").append(zkQuorum).append("/"+username);
+      if ("KERBEROS".equalsIgnoreCase(auth)) {
+        sBuilder.append(";serviceDiscoveryMode=") 
+                .append(serviceDiscoveryMode)
+                .append(";zooKeeperNamespace=")
+                .append(zooKeeperNamespace)
+                .append(";saslQop=")
+                .append(saslQop)
+                .append(";auth=")
+                .append(auth)
+                .append(";principal=")
+                .append(principal)
+                .append(";user.principal=")
+                .append(username) // 设置新建用户的USER_NAME，例如创建的用户为user，则USER_NAME为user
+                .append(";user.keytab=")
+                .append(userKeytabFile)
+                .append(";");
+      } else {
+          //普通模式
+        sBuilder.append(";serviceDiscoveryMode=") 
+                .append(serviceDiscoveryMode)
+                .append(";zooKeeperNamespace=")
+                .append(zooKeeperNamespace)
+                .append(";auth=none");
+      }
+      String url = sBuilder.toString();
+//      System.out.print(url);
+      return url;
+    }
+  
+  public static String urlPreBuilder(String username,String password,String confPath) throws IOException{
+      Properties clientInfo = null;
+      String userdir = confPath.endsWith(File.separator)?confPath:confPath + File.separator;
+      
+      Configuration conf = new Configuration();
+      
+      clientInfo = new Properties();
+      //"hiveclient.properties"为客户端配置文件，如果使用多实例特性，需要把该文件换成对应实例客户端下的"hiveclient.properties"
+      //"hiveclient.properties"文件位置在对应实例客户端安裝包解压目录下的config目录下
+      String hiveclientProp = userdir + "hiveclient.properties" ;
+      
+      File propertiesFile = new File(hiveclientProp);
+      try(InputStream fileInputStream = new FileInputStream(propertiesFile);){
+        clientInfo.load(fileInputStream);
+      }catch (Exception e) {
+        throw new IOException(e);
+      }
+      //zkQuorum获取后的格式为"xxx.xxx.xxx.xxx:24002,xxx.xxx.xxx.xxx:24002,xxx.xxx.xxx.xxx:24002";
+      //"xxx.xxx.xxx.xxx"为集群中ZooKeeper所在节点的业务IP，端口默认是24002
+      String zkQuorum =  clientInfo.getProperty("zk.quorum");
+      String auth = clientInfo.getProperty("auth");
+      String saslQop = clientInfo.getProperty("sasl.qop");
+      String zooKeeperNamespace = clientInfo.getProperty("zooKeeperNamespace");
+      String serviceDiscoveryMode = clientInfo.getProperty("serviceDiscoveryMode");
+      String principal = clientInfo.getProperty("principal"); 
+
+      String userKeytabFile=null;
+      String krb5File=null;
+      if ("KERBEROS".equalsIgnoreCase(auth)) {
+        // 设置客户端的keytab和krb5文件路径
+        userKeytabFile = userdir + "user.keytab";
+        krb5File = userdir + "krb5.conf";
+        System.setProperty("java.security.krb5.conf", krb5File);
+        LoginUtil.setJaasConf(ZOOKEEPER_DEFAULT_LOGIN_CONTEXT_NAME, username, userKeytabFile);
+        String serverPrincipal = ZOOKEEPER_DEFAULT_SERVER_PRINCIPAL + getUserRealm();
+        LoginUtil.setZookeeperServerPrincipal(ZOOKEEPER_SERVER_PRINCIPAL_KEY, serverPrincipal);
+
+        // 安全模式
+        // Zookeeper登录认证
+        LoginUtil.login(username, userKeytabFile, krb5File, conf);
+      }
+      
+      // 拼接JDBC URL
+      StringBuilder sBuilder = new StringBuilder(
+          "jdbc:hive2://").append(zkQuorum).append("/"+username);
+
+      if ("KERBEROS".equalsIgnoreCase(auth)) {
+        sBuilder.append(";serviceDiscoveryMode=") 
+                .append(serviceDiscoveryMode)
+                .append(";zooKeeperNamespace=")
+                .append(zooKeeperNamespace)
+                .append(";saslQop=")
+                .append(saslQop)
+                .append(";auth=")
+                .append(auth)
+                .append(";user.principal=")
+                .append(username) // 设置新建用户的USER_NAME，例如创建的用户为user，则USER_NAME为user
+                .append(";principal=")
+                .append(principal)
+                .append(";");
+      } else {
+        // 普通模式
+        sBuilder.append(";serviceDiscoveryMode=") 
+                .append(serviceDiscoveryMode)
+                .append(";zooKeeperNamespace=")
+                .append(zooKeeperNamespace)
+                .append(";auth=none");
+      }
+      
+      String url = sBuilder.toString();
+//      System.out.print(url);
+      return url;
+    }
+  
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/jsydb/KerberosUtilx.java

@@ -0,0 +1,45 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.jsydb;
+
+import java.lang.reflect.Method;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KerberosUtilx {
+
+    private static Logger logger = LoggerFactory.getLogger(KerberosUtilx.class);
+
+    public static final String JAVA_VENDER = "java.vendor";
+    public static final String IBM_FLAG = "IBM";
+    public static final String CONFIG_CLASS_FOR_IBM = "com.ibm.security.krb5.internal.Config";
+    public static final String CONFIG_CLASS_FOR_SUN = "sun.security.krb5.Config";
+    public static final String METHOD_GET_INSTANCE = "getInstance";
+    public static final String METHOD_GET_DEFAULT_REALM = "getDefaultRealm";
+    public static final String DEFAULT_REALM = "HADOOP.COM";
+
+    public static String getKrb5DomainRealm() {
+        Class<?> krb5ConfClass;
+        String peerRealm;
+        try {
+            if (System.getProperty(JAVA_VENDER).contains(IBM_FLAG)) {
+                krb5ConfClass = Class.forName(CONFIG_CLASS_FOR_IBM);
+            } else {
+                krb5ConfClass = Class.forName(CONFIG_CLASS_FOR_SUN);
+            }
+
+            Method getInstanceMethod = krb5ConfClass.getMethod(METHOD_GET_INSTANCE);
+            Object kerbConf = getInstanceMethod.invoke(krb5ConfClass);
+
+            Method getDefaultRealmMethod = krb5ConfClass.getDeclaredMethod(METHOD_GET_DEFAULT_REALM);
+            peerRealm = (String)getDefaultRealmMethod.invoke(kerbConf);
+            logger.info("Get default realm successfully, the realm is : " + peerRealm);
+
+        } catch (Exception e) {
+            //e.printStackTrace();
+            peerRealm = DEFAULT_REALM;
+            logger.warn("Get default realm failed, use default value : " + DEFAULT_REALM);
+        }
+
+        return peerRealm;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/jsydb/LoginUtil.java

@@ -0,0 +1,457 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.jsydb;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LoginUtil
+{
+    private static final Logger LOGGER = LoggerFactory.getLogger(LoginUtil.class);
+    
+    private static final String JAVA_SECURITY_KRB5_CONF_KEY = "java.security.krb5.conf";
+    
+    private static final String JAVA_SECURITY_AUTH_USE_SUBJECT_CREDS_ONLY = "javax.security.auth.useSubjectCredsOnly";
+       
+    private static final String LOGIN_FAILED_CAUSE_PASSWORD_WRONG =
+        "(wrong password) keytab file and user not match, you can kinit -k -t keytab user in client server to check";
+    
+    private static final String LOGIN_FAILED_CAUSE_TIME_WRONG =
+        "(clock skew) time of local server and remote server not match, please check ntp to remote server";
+    
+    private static final String LOGIN_FAILED_CAUSE_AES256_WRONG =
+        "(aes256 not support) aes256 not support by default jdk/jre, need copy local_policy.jar and US_export_policy.jar from remote server in path /opt/huawei/Bigdata/jdk/jre/lib/security";
+    
+    private static final String LOGIN_FAILED_CAUSE_PRINCIPAL_WRONG =
+        "(no rule) principal format not support by default, need add property hadoop.security.auth_to_local(in core-site.xml) value RULE:[1:$1] RULE:[2:$1]";
+    
+    private static final String LOGIN_FAILED_CAUSE_TIME_OUT =
+        "(time out) can not connect to kdc server or there is fire wall in the network";
+    
+    private static final boolean IS_IBM_JDK = System.getProperty("java.vendor").contains("IBM");
+    
+    public synchronized static void login(String userPrincipal, String userKeytabPath, String krb5ConfPath, Configuration conf)
+        throws IOException
+    {
+        // 1.check input parameters
+        if ((userPrincipal == null) || (userPrincipal.length() <= 0))
+        {
+            LOGGER.error("input userPrincipal is invalid.");
+            throw new IOException("input userPrincipal is invalid.");
+        }
+        
+        if ((userKeytabPath == null) || (userKeytabPath.length() <= 0))
+        {
+            LOGGER.error("input userKeytabPath is invalid.");
+            throw new IOException("input userKeytabPath is invalid.");
+        }
+                
+        if ((krb5ConfPath == null) || (krb5ConfPath.length() <= 0))
+        {
+            LOGGER.error("input krb5ConfPath is invalid.");
+            throw new IOException("input krb5ConfPath is invalid.");
+        }
+        
+        if ((conf == null))
+        {
+            LOGGER.error("input conf is invalid.");
+            throw new IOException("input conf is invalid.");
+        }
+        
+        // 2.check file exsits
+        File userKeytabFile = new File(userKeytabPath);
+        if (!userKeytabFile.exists())
+        {
+            LOGGER.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
+            throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
+        }
+        if (!userKeytabFile.isFile())
+        {
+            LOGGER.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
+            throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
+        }
+        
+        File krb5ConfFile = new File(krb5ConfPath);
+        if (!krb5ConfFile.exists())
+        {
+            LOGGER.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
+            throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
+        }
+        if (!krb5ConfFile.isFile())
+        {
+            LOGGER.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
+            throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
+        }
+        
+        // 3.set and check krb5config
+        setKrb5Config(krb5ConfFile.getAbsolutePath());        
+        setConfiguration(conf);
+        
+        // 4.login and check for hadoop
+        loginHadoop(userPrincipal, userKeytabFile.getAbsolutePath());
+        
+        LOGGER.info("Login success!!!!!!!!!!!!!!");
+    }
+    
+    private static void setConfiguration(Configuration conf) throws IOException {
+	    UserGroupInformation.setConfiguration(conf);
+    }
+        
+    private static boolean checkNeedLogin(String principal)
+        throws IOException
+    {
+        if (!UserGroupInformation.isSecurityEnabled())
+        {
+            LOGGER.error("UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath.");
+            throw new IOException(
+                "UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath.");
+        }
+        UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+        if ((currentUser != null) && (currentUser.hasKerberosCredentials()))
+        {
+            if (checkCurrentUserCorrect(principal))
+            {
+                LOGGER.info("current user is " + currentUser + "has logined.");
+                if (!currentUser.isFromKeytab())
+                {
+                    LOGGER.error("current user is not from keytab.");
+                    throw new IOException("current user is not from keytab.");
+                }
+                return false;
+            }
+            else
+            {
+                LOGGER.error("current user is " + currentUser + "has logined. please check your enviroment , especially when it used IBM JDK or kerberos for OS count login!!");
+                throw new IOException("current user is " + currentUser + " has logined. And please check your enviroment!!");
+            }
+        }
+        
+        return true;
+    }
+    
+    private static void setKrb5Config(String krb5ConfFile)
+        throws IOException
+    {
+        System.setProperty(JAVA_SECURITY_KRB5_CONF_KEY, krb5ConfFile);
+        String ret = System.getProperty(JAVA_SECURITY_KRB5_CONF_KEY);
+        if (ret == null)
+        {
+            LOGGER.error(JAVA_SECURITY_KRB5_CONF_KEY + " is null.");
+            throw new IOException(JAVA_SECURITY_KRB5_CONF_KEY + " is null.");
+        }
+        if (!ret.equals(krb5ConfFile))
+        {
+            LOGGER.error(JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + ".");
+            throw new IOException(JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + ".");
+        }
+    }
+    
+    public static void setJaasConf(String loginContextName, String principal, String keytabFile)
+        throws IOException
+    {
+        if ((loginContextName == null) || (loginContextName.length() <= 0))
+        {
+            LOGGER.error("input loginContextName is invalid.");
+            throw new IOException("input loginContextName is invalid.");
+        }
+        
+        if ((principal == null) || (principal.length() <= 0))
+        {
+            LOGGER.error("input principal is invalid.");
+            throw new IOException("input principal is invalid.");
+        }
+        
+        if ((keytabFile == null) || (keytabFile.length() <= 0))
+        {
+            LOGGER.error("input keytabFile is invalid.");
+            throw new IOException("input keytabFile is invalid.");
+        }
+        
+        File userKeytabFile = new File(keytabFile);
+        if (!userKeytabFile.exists())
+        {
+            LOGGER.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
+            throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
+        }
+    	
+        javax.security.auth.login.Configuration.setConfiguration(new JaasConfiguration(loginContextName, principal,
+        		userKeytabFile.getAbsolutePath()));       
+        
+        javax.security.auth.login.Configuration conf = javax.security.auth.login.Configuration.getConfiguration();
+        if (!(conf instanceof JaasConfiguration))
+        {
+            LOGGER.error("javax.security.auth.login.Configuration is not JaasConfiguration.");
+            throw new IOException("javax.security.auth.login.Configuration is not JaasConfiguration.");
+        }
+        
+    	AppConfigurationEntry[] entrys = conf.getAppConfigurationEntry(loginContextName);
+        if (entrys == null)
+        {
+            LOGGER.error("javax.security.auth.login.Configuration has no AppConfigurationEntry named " + loginContextName
+                + ".");
+            throw new IOException("javax.security.auth.login.Configuration has no AppConfigurationEntry named "
+                + loginContextName + ".");
+        }
+        
+        boolean checkPrincipal = false;
+        boolean checkKeytab = false;        
+        for (int i = 0; i < entrys.length; i++)
+        {
+            if (entrys[i].getOptions().get("principal").equals(principal))
+            {
+                checkPrincipal = true;
+            }
+            
+            if (IS_IBM_JDK)
+            {
+                if (entrys[i].getOptions().get("useKeytab").equals(keytabFile))
+                {
+                    checkKeytab = true;
+                }
+            }
+            else 
+            {
+                if (entrys[i].getOptions().get("keyTab").equals(keytabFile))
+                {
+                    checkKeytab = true;
+                }
+            }
+
+        }
+        
+        if (!checkPrincipal)
+        {
+            LOGGER.error("AppConfigurationEntry named " + loginContextName + " does not have principal value of "
+                + principal + ".");
+            throw new IOException("AppConfigurationEntry named " + loginContextName
+                + " does not have principal value of " + principal + ".");
+        }
+        
+        if (!checkKeytab)
+        {
+            LOGGER.error("AppConfigurationEntry named " + loginContextName + " does not have keyTab value of "
+                + keytabFile + ".");
+            throw new IOException("AppConfigurationEntry named " + loginContextName + " does not have keyTab value of "
+                + keytabFile + ".");
+        }
+        
+    }
+        
+    public static void setZookeeperServerPrincipal(String zkServerPrincipalKey, String zkServerPrincipal)
+            throws IOException
+        {
+            System.setProperty(zkServerPrincipalKey, zkServerPrincipal);
+            String ret = System.getProperty(zkServerPrincipalKey);
+            if (ret == null)
+            {
+                LOGGER.error(zkServerPrincipalKey + " is null.");
+                throw new IOException(zkServerPrincipalKey + " is null.");
+            }
+            if (!ret.equals(zkServerPrincipal))
+            {
+                LOGGER.error(zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal
+                    + ".");
+                throw new IOException(zkServerPrincipalKey + " is " + ret + " is not "
+                    + zkServerPrincipal + ".");
+            }
+        }
+    
+    private static void loginHadoop(String principal, String keytabFile)
+        throws IOException
+    {
+        System.setProperty(JAVA_SECURITY_AUTH_USE_SUBJECT_CREDS_ONLY, "false");
+        try
+        {
+            UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
+            LOGGER.info("loginUserFromKeytab finished");
+        }
+        catch (IOException e)
+        {
+            LOGGER.error("login failed with " + principal + " and " + keytabFile + ".");
+            LOGGER.error("perhaps cause 1 is " + LOGIN_FAILED_CAUSE_PASSWORD_WRONG + ".");
+            LOGGER.error("perhaps cause 2 is " + LOGIN_FAILED_CAUSE_TIME_WRONG + ".");
+            LOGGER.error("perhaps cause 3 is " + LOGIN_FAILED_CAUSE_AES256_WRONG + ".");
+            LOGGER.error("perhaps cause 4 is " + LOGIN_FAILED_CAUSE_PRINCIPAL_WRONG + ".");
+            LOGGER.error("perhaps cause 5 is " + LOGIN_FAILED_CAUSE_TIME_OUT + ".");
+            
+            throw e;
+        }
+    }
+    
+    private static void checkAuthenticateOverKrb()
+        throws IOException
+    {
+        UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+        UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+        if (loginUser == null)
+        {
+            LOGGER.error("current user is " + currentUser + ", but loginUser is null.");
+            throw new IOException("current user is " + currentUser + ", but loginUser is null.");
+        }
+        if (!loginUser.equals(currentUser))
+        {
+            LOGGER.error("current user is " + currentUser + ", but loginUser is " + loginUser + ".");
+            throw new IOException("current user is " + currentUser + ", but loginUser is " + loginUser + ".");
+        }
+        if (!loginUser.hasKerberosCredentials())
+        {
+            LOGGER.error("current user is " + currentUser + " has no Kerberos Credentials.");
+            throw new IOException("current user is " + currentUser + " has no Kerberos Credentials.");
+        }
+        if (!UserGroupInformation.isLoginKeytabBased())
+        {
+            LOGGER.error("current user is " + currentUser + " is not Login Keytab Based.");
+            throw new IOException("current user is " + currentUser + " is not Login Keytab Based.");
+        }
+    }
+    
+    private static boolean checkCurrentUserCorrect(String principal)
+        throws IOException
+    {
+        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+        if (ugi == null)
+        {
+            LOGGER.error("current user still null.");
+            throw new IOException("current user still null.");
+        }
+        
+        String defaultRealm = null;
+		try {
+			defaultRealm = KerberosUtil.getDefaultRealm();
+		} catch (Exception e) {
+			LOGGER.warn("getDefaultRealm failed.");
+			throw new IOException(e);
+		}
+
+        if ((defaultRealm != null) && (defaultRealm.length() > 0))
+        {
+            StringBuilder realm = new StringBuilder();
+            StringBuilder principalWithRealm = new StringBuilder();
+            realm.append("@").append(defaultRealm);
+            if (!principal.endsWith(realm.toString()))
+            {
+                principalWithRealm.append(principal).append(realm);
+                principal = principalWithRealm.toString();
+            }
+        }
+        
+        return principal.equals(ugi.getUserName());
+    }
+    
+    /**
+     * copy from hbase zkutil 0.94&0.98 A JAAS configuration that defines the login modules that we want to use for
+     * login.
+     */
+    private static class JaasConfiguration extends javax.security.auth.login.Configuration
+    {        
+        private static final Map<String, String> BASIC_JAAS_OPTIONS = new HashMap<String, String>();
+        static
+        {
+            String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
+            if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar))
+            {
+                BASIC_JAAS_OPTIONS.put("debug", "true");
+            }
+        }
+        
+        private static final Map<String, String> KEYTAB_KERBEROS_OPTIONS = new HashMap<String, String>();
+        static
+        {
+            if (IS_IBM_JDK)
+            {              
+            	KEYTAB_KERBEROS_OPTIONS.put("credsType", "both");
+            }
+            else {
+            	KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");              
+            	KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", "false");
+            	KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
+            	KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
+            }
+        	
+            KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
+        }
+        
+        
+        
+        private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = new AppConfigurationEntry(
+            KerberosUtil.getKrb5LoginModuleName(), LoginModuleControlFlag.REQUIRED, KEYTAB_KERBEROS_OPTIONS);
+        
+        private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
+            new AppConfigurationEntry[] {KEYTAB_KERBEROS_LOGIN};
+        
+        private javax.security.auth.login.Configuration baseConfig;
+        
+        private final String loginContextName;
+        
+        private final boolean useTicketCache;
+        
+        private final String keytabFile;
+        
+        private final String principal;
+        
+        
+        public JaasConfiguration(String loginContextName, String principal, String keytabFile) throws IOException
+        {
+            this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
+        }
+        
+        private JaasConfiguration(String loginContextName, String principal, String keytabFile, boolean useTicketCache) throws IOException
+        {
+            try
+            {
+                this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
+            }
+            catch (SecurityException e)
+            {
+                this.baseConfig = null;
+            }
+            this.loginContextName = loginContextName;
+            this.useTicketCache = useTicketCache;
+            this.keytabFile = keytabFile;
+            this.principal = principal;
+            
+            initKerberosOption();
+            LOGGER.info("JaasConfiguration loginContextName=" + loginContextName + " principal=" + principal
+                + " useTicketCache=" + useTicketCache + " keytabFile=" + keytabFile);
+        }
+        
+        private void initKerberosOption() throws IOException
+        {
+            if (!useTicketCache)
+            {
+            	if(IS_IBM_JDK)
+            	{                		
+            		KEYTAB_KERBEROS_OPTIONS.put("useKeytab", keytabFile);
+            	}
+            	else
+            	{
+                    KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
+                    KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
+//                    KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
+                    KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", "false");
+            	}
+            }
+            KEYTAB_KERBEROS_OPTIONS.put("principal", principal);            
+        }
+        
+        public AppConfigurationEntry[] getAppConfigurationEntry(String appName)
+        {
+            if (loginContextName.equals(appName))
+            {
+                return KEYTAB_KERBEROS_CONF;
+            }
+            if (baseConfig != null)
+                return baseConfig.getAppConfigurationEntry(appName);
+            return (null);
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/model/SqlHandlerResultVO.java

@@ -0,0 +1,103 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * SQL拼接结果
+ * @author zhouliang
+ *
+ */
+public class SqlHandlerResultVO implements Serializable
+{
+
+    private static final long serialVersionUID = 1L;
+    /**
+     * 查询sql语句(远程)
+     */
+    private String selectSql;
+    /**
+     * 插入sql语句(本地)
+     */
+    private String insertSql;
+    
+    /**
+     * 前置sql语句(本地)
+     */
+    private String preSql;
+    
+    /**
+     * 映射关系(包含类型转换) 读取字段-存储字段/类型
+     */
+    private Map<String,SqlHandlerTypeVO> fieldMapping;
+
+    /**
+     * @return the selectSql
+     */
+    public String getSelectSql() {
+        return selectSql;
+    }
+
+    /**
+     * @param selectSql the selectSql to set
+     */
+    public void setSelectSql(String selectSql) {
+        this.selectSql = selectSql;
+    }
+
+    /**
+     * @return the insertSql
+     */
+    public String getInsertSql() {
+        return insertSql;
+    }
+
+    /**
+     * @param insertSql the insertSql to set
+     */
+    public void setInsertSql(String insertSql) {
+        this.insertSql = insertSql;
+    }
+
+    /**
+     * @return the preSql
+     */
+    public String getPreSql() {
+        return preSql;
+    }
+
+    /**
+     * @param preSql the preSql to set
+     */
+    public void setPreSql(String preSql) {
+        this.preSql = preSql;
+    }
+
+    /**
+     * @return the fieldMapping
+     */
+    public Map<String, SqlHandlerTypeVO> getFieldMapping() {
+        return fieldMapping;
+    }
+
+    /**
+     * @param fieldMapping the fieldMapping to set
+     */
+    public void setFieldMapping(Map<String, SqlHandlerTypeVO> fieldMapping) {
+        this.fieldMapping = fieldMapping;
+    }
+    
+    //获取转换过后的标题
+    public List<String> getFieldTitle() {
+        List<String> list=new ArrayList<>();
+        fieldMapping.forEach((k,v)->{
+            String code = v.getColumnCode();
+            list.add(StringUtils.isBlank(code)?k:code);
+        });
+        return list;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/model/SqlHandlerTypeVO.java

@@ -0,0 +1,46 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model;
+
+import java.io.Serializable;
+
+/**
+ * SQL字段映射和转换关系
+ * @author zhouliang
+ *
+ */
+public class SqlHandlerTypeVO implements Serializable
+{
+
+    private static final long serialVersionUID = 1L;
+    /**
+     * 存储字段编码
+     */
+    private String columnCode;
+    /**
+     * 转换类型（为空时默认按相同处理）
+     */
+    private String convertType;
+    /**
+     * @return the columnCode
+     */
+    public String getColumnCode() {
+        return columnCode;
+    }
+    /**
+     * @param columnCode the columnCode to set
+     */
+    public void setColumnCode(String columnCode) {
+        this.columnCode = columnCode;
+    }
+    /**
+     * @return the convertType
+     */
+    public String getConvertType() {
+        return convertType;
+    }
+    /**
+     * @param convertType the convertType to set
+     */
+    public void setConvertType(String convertType) {
+        this.convertType = convertType;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/reader/FileBaseReader.java

@@ -0,0 +1,198 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
+
+/**
+ * JDBC操作数据库
+ * @author zhouliang
+ *
+ */
+public class FileBaseReader {
+    private static final Logger LOGGER = LoggerFactory.getLogger(FileBaseReader.class);
+    
+    private BufferedReader reader;
+    private BufferedWriter writer;
+    
+    private Long readerNumber=0L;//已读取行数，方便查看
+    private Long writerNumber=0L;//已写入行数，方便查看
+    
+    private String readerPath="";
+    private String writerPath="";
+    private String writerPathTemp="";//临时文件
+    
+    
+    public BufferedReader getReader() {
+        return reader;
+    }
+    public BufferedWriter getWriter() {
+        return writer;
+    }
+    public Long getReaderNumber() {
+        return readerNumber;
+    }
+    public Long getWriterNumber() {
+        return writerNumber;
+    }
+
+    public String getReaderPath() {
+        return readerPath;
+    }
+    public String getWriterPath() {
+        return writerPath;
+    }
+
+    /**
+     * 创建读取器
+     * @param filePath
+     * @return
+     * @throws IOException
+     */
+    public BufferedReader createReader(String filePath) throws IOException {
+        readerPath=filePath;
+        File file = new File(filePath);
+        reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));
+        return reader;
+    }
+    
+    /**
+     * 读取文件，取固定行数
+     * @param number 读取行数
+     * @return
+     * @throws IOException
+     */
+    public List<String> readLines(int number) throws IOException {
+        List<String> lines = new ArrayList<>();
+        String line;
+        int count = 0;
+        while ((line = reader.readLine()) != null &&  count < number) {
+            if(StringUtils.isNotBlank(line)) {
+                lines.add(line);
+            }
+            count++;
+            readerNumber++;
+        }
+        return lines;
+    }
+    /**
+     * 创建写入器
+     * @param filePath
+     * @return
+     * @throws IOException
+     */
+    public void createWriter(String filePath) throws IOException {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmssSSS");
+        String formattedTime = formatter.format(Instant.now().atZone(ZoneId.systemDefault()));
+        writerPath=filePath;
+        writerPathTemp=filePath+"."+formattedTime+"."+SyncDataConstants.TEMP_FILE_EXTENSION;
+        File file = new File(writerPathTemp);
+        if(!file.exists()) {
+            file.getParentFile().mkdirs();
+        }
+        writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true), StandardCharsets.UTF_8));
+    }
+    /**
+     * 追加内容到文件
+     * @param filePath
+     * @param lines 会合并一次写入的
+     * @param lineSeparator 行分隔符默认 “\n”
+     * @throws IOException
+     */
+    public void writeLines(List<String> lines,String lineSeparator) throws IOException {
+        if(lines==null||lines.isEmpty()) {
+            return;
+        }
+        if(StringUtils.isNotBlank(lineSeparator)) {
+            lineSeparator="\n";
+        }
+        writerNumber=writerNumber+lines.size();
+        writer.append(lineSeparator+StringUtils.join(lines,lineSeparator));
+        lines.clear();
+    }
+    /** 
+     * 重置文件，写入文件头
+     */
+    public void clearWriteTitle(String title,boolean deleteFlag) throws IOException {
+        File file = new File(writerPath);
+        if(!file.exists()) {
+            file.getParentFile().mkdirs();
+        }else if(deleteFlag) {//需要删除
+            if(file.delete()) {
+                //OK
+            }
+        }
+        File tmpfile = new File(writerPathTemp);
+        try(BufferedWriter writer2 = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmpfile, false), StandardCharsets.UTF_8));) {
+            writer2.write(title);
+        }catch (Exception e) {
+            //
+        }
+    }
+    /**
+     * 修改文件名为正式
+     */
+    public void renameTempFile() {
+        File tmpfile = new File(writerPathTemp);
+        if(tmpfile.exists()) {
+            File newFile = new File(writerPath);
+            Matcher fm = SyncDataConstants.FILE_NAME_RULE.matcher(newFile.getName());
+            String ext="";
+            if(fm.find()) {
+                ext=fm.group(2);
+            }
+            int i=2;
+            while(newFile.exists()) {
+                newFile = new File(writerPath.replace(ext, "-"+i+ext));
+                i++;
+            }
+            // 使用renameTo()方法将文件重命名为新的名称
+            boolean success = tmpfile.renameTo(newFile);
+            if (success) {
+               //文件名修改成功
+                LOGGER.info("Successfully modified the file name>>{}", newFile.getName());
+            }
+        }
+    }
+    
+    public void closeReader() {
+        try {
+            if(null!=reader) {
+                reader.close();
+            }
+        }catch (Exception e) {
+            
+        }
+    }
+    public void closeWriter() {
+        try {
+            if(null!=writer) {
+                writer.close();
+            }
+        }catch (Exception e) {
+            
+        }
+    }
+    public void close() {
+        closeReader();
+        closeWriter();
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/reader/JdbcBaseReader.java

@@ -0,0 +1,127 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * JDBC操作数据库
+ * @author zhouliang
+ *
+ */
+public class JdbcBaseReader {
+    private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBaseReader.class);
+
+    
+    Connection connection = null;
+    PreparedStatement preparedStatement = null;
+    
+  /**
+   * JDBC连接数据库
+   * @param url
+   * @param username
+   * @param password
+   * @param driverClassName
+   * @return
+   * @throws Exception
+   */
+  public Connection getConnection(String url, String username,String password,String driverClassName) throws Exception{
+    try {
+        LOGGER.info("Create connection url={}, username={}",url,username);
+        // 加载JDBC驱动
+        Class.forName(driverClassName);
+        // 获取JDBC连接
+        connection = DriverManager.getConnection(url, username!=null?username:"", password!=null?password:"");
+        
+        LOGGER.info("Create connection success!");
+    }catch (Exception e) {
+        LOGGER.info("Create connection failed : "  + e.getMessage());
+    }
+    return connection;
+  }
+  
+  public void closeConnection() {
+      try {
+          closeCurrStatement();
+          if(null!=connection) {
+              connection.close();
+          }
+          LOGGER.info("Close connection success!");
+      }catch (Exception e) {
+          LOGGER.info("Close connection failed : "  + e.getMessage());
+      }
+    }
+  
+   /**
+    * 关闭当前的PreparedStatement
+    * @return
+    * @throws Exception
+    */
+   public void closeCurrStatement(){
+       try {
+           if(null!=preparedStatement) {
+               preparedStatement.close();
+           }
+       }catch (Exception e) {
+           
+       }
+   }
+  
+  /**
+   * 执行批量操作，具体数量由业务控制(主要insert语句)
+   * 使用完记得关闭
+   * @param sql
+   * @param lists
+   * @throws SQLException
+   */
+  public void execBatchSql(String sql,List<List<Object>> lists) throws SQLException {
+      if(preparedStatement==null || preparedStatement.isClosed()) {//默认沿用前面的，不再创建
+          preparedStatement = connection.prepareStatement(sql);
+      }
+      // 添加要插入的数据
+      for(List<Object> list:lists) {
+          for(int i=0;i<list.size();i++) {
+              //数据库索引是1开始
+              preparedStatement.setObject(i+1, list.get(i));
+          }
+          preparedStatement.addBatch();
+      }
+      // 执行批量
+      preparedStatement.executeBatch();
+  }
+  public void execBatchSql2(String sql,List<List<Object>> lists) throws SQLException {
+      LOGGER.warn("Hive does not support jdbc writing, please use file writing!!!");
+  }
+  
+  /**
+   * 执行单条sql
+   * 使用完记得关闭
+   * @param sql
+   * @param list
+   * @throws SQLException
+   */
+  public ResultSet execSql(String sql,List<Object> list) throws SQLException {
+      ResultSet resultSet=null;
+      if(null!=preparedStatement) {//默认不沿用，有则关闭
+          closeCurrStatement();
+      }
+      preparedStatement = connection.prepareStatement(sql);
+      if(list!=null&&!list.isEmpty()) {
+          for(int i=0;i<list.size();i++) {
+              //数据库索引是1开始
+              preparedStatement.setObject(i+1, list.get(i));
+          }
+      }
+      boolean flag=preparedStatement.execute();
+      if(flag) {
+          resultSet=preparedStatement.getResultSet();
+      }
+      return resultSet;
+  }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/common/reader/ReaderWriterHelper.java

@@ -0,0 +1,324 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader;
+
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.DDsProperties;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SqlHandlerUtilx;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.jsydb.JsyHiveJDBCBuilder;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model.SqlHandlerResultVO;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.model.SqlHandlerTypeVO;
+
+/**
+ * 数据读写操作
+ * 使用完记得关闭
+ * @author zhouliang
+ *
+ */
+public class ReaderWriterHelper {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ReaderWriterHelper.class);
+    
+    private JdbcBaseReader jdbcReader=null;//读取数据库
+    private JdbcBaseReader jdbcWriter=null;//写入数据库
+    private FileBaseReader fileReader=null;//文件读写
+    
+    private int singleWriteNumber=20000;//一次写入行
+    private boolean readFileFlag=false;//false是数据库，true是文件
+    private boolean writeFileFlag=false;//false是数据库，true是文件
+    private String fieldSeparator=",";//文件字段分隔
+    private String lineSeparator="\n";//文件行分隔
+    
+    private boolean jdbcWriterHiveFlag=false;//是否hive，特殊处理
+    
+    public JdbcBaseReader getJdbcReader() {
+        return jdbcReader;
+    }
+
+    public JdbcBaseReader getJdbcWriter() {
+        return jdbcWriter;
+    }
+
+    public FileBaseReader getFileReader() {
+        return fileReader;
+    }
+
+    public void setSingleWriteNumber(int singleWriteNumber) {
+        if(singleWriteNumber>0) {
+            this.singleWriteNumber = singleWriteNumber;
+        }
+    }
+
+    public void setReadFileFlag(boolean readFileFlag) {
+        this.readFileFlag = readFileFlag;
+    }
+
+    public void setWriteFileFlag(boolean writeFileFlag) {
+        this.writeFileFlag = writeFileFlag;
+    }
+
+    public void setFieldSeparator(String fieldSeparator) {
+        this.fieldSeparator = fieldSeparator;
+    }
+
+    public void setLineSeparator(String lineSeparator) {
+        this.lineSeparator = lineSeparator;
+    }
+
+
+    /**
+     * 是否文件处理
+     * @param dbName
+     * @return
+     * @throws Exception
+     */
+    public boolean isFile(String dbName) {
+        return SyncDataConstants.DB_NAME_FILE_TXT.equals(dbName)
+                  || SyncDataConstants.DB_NAME_FILE_CSV.equals(dbName)
+                  || dbName.toLowerCase().endsWith("."+SyncDataConstants.DB_NAME_FILE_TXT)
+                  || dbName.toLowerCase().endsWith("."+SyncDataConstants.DB_NAME_FILE_CSV);
+    }
+    
+    
+    /**
+     * 创建读取数据库处理器
+     * @param dbName
+     * @param ds
+     * @return
+     * @throws Exception
+     */
+    public JdbcBaseReader createJdbcReader(String dbName,DDsProperties ds) throws Exception {
+        LOGGER.info("createJdbcReader>>dbName={}",dbName);
+        jdbcReader = createJdbcHandler(dbName,ds);
+        return jdbcReader;
+    }
+    /**
+     * 创建写入数据库处理器
+     * @param dbName
+     * @param ds
+     * @return
+     * @throws Exception
+     */
+    public JdbcBaseReader createJdbcWriter(String dbName,DDsProperties ds) throws Exception {
+        LOGGER.info("createJdbcWriter>>dbName={}",dbName);
+        jdbcWriter = createJdbcHandler(dbName,ds);
+        return jdbcWriter;
+    }
+    
+    /**
+     * 根据数据库类型加载数据库配置
+     * @param dbName
+     * @param ds
+     * @return
+     * @throws Exception
+     */
+    private JdbcBaseReader createJdbcHandler(String dbName,DDsProperties ds) throws Exception {
+        JdbcBaseReader jdbcBaseReader=new JdbcBaseReader();
+        if(ds!=null) {
+            jdbcWriterHiveFlag=ds.getDriverClassName().contains(".hive.");//是否hive
+            //特殊数据库处理
+            if(SyncDataConstants.DB_NAME_JSY_HIVE_DS.equals(dbName)
+                    || dbName.startsWith(SyncDataConstants.DB_NAME_JSY_HIVE_DS)) {
+                ds.setUrl(JsyHiveJDBCBuilder.urlPreBuilder( ds.getUsername(), ds.getPassword(), ds.getConfPath()));
+            }
+            jdbcBaseReader.getConnection(ds.getUrl(), ds.getUsername()
+                    , ds.getPassword(), ds.getDriverClassName());
+        }
+        else {//数据库不存在
+            LOGGER.info("createJdbcHandler Database type does not exist>>dbName={}",dbName);
+            return null;
+        }
+        return jdbcBaseReader;
+    }
+    
+    /**
+     * 创建读文件处理器
+     * @param dbName
+     * @param path
+     * @return
+     * @throws Exception
+     */
+    public FileBaseReader createFileReader(String dbName,String path) throws Exception {
+        LOGGER.info("createFileReader>>dbName={}",dbName);
+        path=createFileHandler(dbName, path);
+        fileReader.createReader(path);
+        return fileReader;
+    }
+    /**
+     * 创建写文件处理器
+     * @param dbName
+     * @param path
+     * @return
+     * @throws Exception
+     */
+    public FileBaseReader createFileWriter(String dbName,String path) throws Exception {
+        LOGGER.info("createFileWriter>>dbName={}",dbName);
+        path=createFileHandler(dbName, path);
+        fileReader.createWriter(path);
+        return fileReader;
+    }
+    /**
+     * 创建文件处理器
+     * @param dbName
+     * @param path
+     * @return
+     * @throws Exception
+     */
+    private String createFileHandler(String dbName,String path) throws Exception {
+        if(null==fileReader) {
+            fileReader =new FileBaseReader();
+        }
+        if(isFile(dbName) && !path.toLowerCase().endsWith(dbName.toLowerCase())) {
+            path=path+"."+dbName;
+        }
+        return path;
+    }
+    
+    /**
+     * 开始写入数据
+     * @param dbName
+     * @return
+     * @throws Exception
+     */
+    public boolean writeData(TblBatchTableMapping mappingInfo,Map<String,String> sqlParams) throws Exception {
+        try {
+            //转换字段映射
+            SqlHandlerResultVO sqlVO = SqlHandlerUtilx.convertJsonToSql(mappingInfo.getRemoteTableSql()
+                    , mappingInfo.getLocalTable(), mappingInfo.getMappingJson());
+            sqlVO.setPreSql(mappingInfo.getLocalPreSql());
+            
+            String selectSql=null;
+            String insertSql=null;
+            if(!readFileFlag) {
+                selectSql=SqlHandlerUtilx.replaceSqlCustomParams(sqlVO.getSelectSql(), sqlParams);
+            }
+            if(!writeFileFlag) {
+                insertSql=SqlHandlerUtilx.replaceSqlCustomParams(sqlVO.getInsertSql(), sqlParams);
+            }
+            String preSql=SqlHandlerUtilx.replaceSqlCustomParams(sqlVO.getPreSql(), sqlParams);
+            if(!readFileFlag) {//从数据库读
+                LOGGER.info("writeData exec selectSql>>{}",selectSql);
+                ResultSet resultSet=jdbcReader.execSql(selectSql, null);
+                long count=0;
+                if(resultSet!=null) {
+                     LOGGER.info("writeData Task Progress>>start");
+                     //解决使用*的返回字段全路径问题
+                     int n=resultSet.getMetaData().getColumnCount();
+                     Map<String,String> colMap=new LinkedHashMap<>();
+                     for(int i=1;i<=n;i++) {
+                         String colname =resultSet.getMetaData().getColumnName(i);
+                         if(colname.contains(".")) {
+                             String colname2=colname.replaceAll("^.*\\.", "");
+                             if(colMap.get(colname2)==null) {//如果确实有同名直接抛弃即可
+                                 colMap.put(colname2, colname);
+                             }
+                         }else {
+                             colMap.put(colname,colname);
+                         }
+                     }
+                     Map<String, SqlHandlerTypeVO> map = sqlVO.getFieldMapping();
+                     Set<String> t1 = map.keySet();
+                     Set<String> t2 = colMap.keySet();
+                     Set<String>  currcols = !map.isEmpty()?map.keySet():colMap.keySet();
+                     //清除本地已有的数据等
+                     if(!writeFileFlag) {//写入数据库
+                         if(StringUtils.isNotBlank(preSql)) {
+                             LOGGER.info("writeData exec preSql>>{}",preSql);
+                             jdbcWriter.execSql(preSql, null);
+                             jdbcWriter.closeCurrStatement();//关闭 
+                         }
+                     }else{
+                        fileReader.clearWriteTitle(StringUtils.join(currcols,fieldSeparator)
+                            , StringUtils.isNotBlank(preSql) && "true".equalsIgnoreCase(preSql.trim()));
+                      }
+                     
+                     //读取数据，N条写一次
+                     List<List<Object>> lists=new ArrayList<>();
+                     while(resultSet.next()) {
+                         count++;
+                         List<Object> row= new ArrayList<>();
+                         if(!map.isEmpty()) {
+                             for(Entry<String, SqlHandlerTypeVO> ex: map.entrySet()) {
+                                 //转换类型
+                                 Object colValue = SqlHandlerUtilx.convertType(resultSet, colMap.get(ex.getKey())
+                                         , ex.getValue().getConvertType(), jdbcWriterHiveFlag||writeFileFlag);//文件需全转string
+                                 row.add(colValue);
+                             }
+                         }else {//为空使用了*号
+                             for(Entry<String, String> ex: colMap.entrySet()) {
+                                 //转换类型
+                                 Object colValue = SqlHandlerUtilx.convertType(resultSet, ex.getValue()
+                                         , null, jdbcWriterHiveFlag||writeFileFlag);//文件需全转string
+                                 row.add(colValue);
+                             }
+                         }
+                         lists.add(row);
+                         if(lists.size()==singleWriteNumber) {//够数
+                             writeDataJdbcOrFile(insertSql, lists,count,mappingInfo.getLocalTable(),currcols);
+                             lists=new ArrayList<>();
+                         }
+                     }
+                     //不够数
+                     writeDataJdbcOrFile(insertSql, lists,count,mappingInfo.getLocalTable(),currcols);
+                     LOGGER.info("writeData Task Progress>>end");
+                }
+            }else {
+                //读文件
+                
+            }
+        } catch (Exception e) {
+            LOGGER.info("writeData error>>{},{}",e.getMessage(),e);
+            return false;
+        } finally {
+            close();
+        }
+        return true;
+    }
+    private void writeDataJdbcOrFile(String insertSql,List<List<Object>> lists,long count, String localTable, Set<String> set) throws Exception {
+        if(!lists.isEmpty()) {
+            LOGGER.info("writeData Task Progress>>total={}",count);
+            if(!writeFileFlag) {//写入数据库
+                if(StringUtils.isBlank(insertSql)) {//为空使用了*号
+                    insertSql=SqlHandlerUtilx.convertInsertSql(set, localTable,1);
+                }
+                if(jdbcWriterHiveFlag) {
+                    jdbcWriter.execBatchSql2(insertSql, lists);
+                }else {
+                    jdbcWriter.execBatchSql(insertSql, lists);
+                }
+            }else {
+                List<String> lists2=new ArrayList<>();
+                for(List<Object> lx:lists) {
+                    lists2.add(StringUtils.join(lx, fieldSeparator));
+                }
+                fileReader.writeLines(lists2, lineSeparator);
+            }
+        }
+    }
+    
+    
+    public void close() {
+        if(null!=jdbcReader) {
+            jdbcReader.closeConnection();
+        }
+        if(null!=jdbcWriter) {
+            jdbcWriter.closeConnection();
+        }
+        if(null!=fileReader) {
+            fileReader.close();
+            //完成后文件名称改名
+            fileReader.renameTempFile();
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/controller/SyncDataReadTaskController.java

@@ -0,0 +1,264 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.controller;
+
+import java.io.File;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.common.TblBatchTaskEnum;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
+import com.jiuyv.sptcc.agile.batch.batchTask.service.IBatchTaskService;
+import com.jiuyv.sptcc.agile.batch.common.R;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.DDsProperties;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.service.SyncDataReadTaskManager;
+
+
+/**
+ * hive数据库任务
+ * 读取远程的两个库
+ * @author zhouliang
+ *
+ */
+@RestController
+@RequestMapping("/batch/")
+public class SyncDataReadTaskController {
+    private static final Logger LOGGER = LoggerFactory.getLogger(SyncDataReadTaskController.class);
+    
+    @Autowired
+    private SyncDataReadTaskManager syncDataReadTaskServiceManager;
+    @Autowired
+    private IBatchTaskService batchTaskService;
+    @Value("${syncdata.clearTempFileDs:}")
+    private String defaultClearTempFileDs;
+    @Value("${syncdata.clearTempFileDays:15}")
+    private int defaultClearTempFileDays;
+    
+    //外部任务调度器触发的任务，线程不是同一个，必须保证任务只能有一个执行
+    //下面实现了通用的按天处理的单表同步模式
+    
+    /**
+     * @title 拉取表数据
+     * 默认拉取T-N这一天。.如果传了日期，则按日期拉取
+     * @param taskNo 任务编码|Y 
+     * @param currDate 日期(2022-01-02)
+     */
+    @GetMapping("syncDay/{taskNo}")
+    public R<String> syncDataByTaskNo(@PathVariable  String taskNo, String currDate,HttpServletRequest request) throws Exception{
+        Map<String, String> params = getUrlParams(request);
+        LOGGER.info("syncDataByTaskNo>>taskNo={}，currDate={}，params={}",taskNo,currDate,params);
+        TblBatchTaskVO task = checkTaskHandler(taskNo);
+        if(task==null) {
+           return  R.fail("runing");
+        }
+        boolean flag=syncDataReadTaskServiceManager.doHiveReaderByTaskNo(task,currDate,params);
+        finishTaskHandler(task, flag);
+        return flag?R.ok("finish"):R.fail("unfinish");
+    }
+    
+    /**
+     * @title 拉取表历史数据
+     * 不传开始日期则不会拉取,从日期开始之后直到T-N
+     * @param taskNo 任务编码|Y 
+     * @param startDate 开始日期(含自身, 2022-01-02)|Y
+     * @param endDate   结束日期(不含自身，2022-01-05) 
+     */ 
+    @GetMapping("syncDayHis/{taskNo}")
+    public R<String> syncDataHisByTaskNo(@PathVariable  String taskNo, String startDate, String endDate,HttpServletRequest request) throws Exception{
+        Map<String, String> params = getUrlParams(request);
+        LOGGER.info("syncDataHisByTaskNo>>taskNo={}，startDate={}，endDate={}，params={}",taskNo,startDate,endDate,params);
+        if(StringUtils.isBlank(startDate)) {
+            return  R.ok();//没有时间直接不执行
+        }
+        TblBatchTaskVO task = checkTaskHandler(taskNo);
+        if(task==null) {
+           return  R.fail("runing");
+        }
+        //跑历史就必须根据失败条件增量继续跑，不可能每次从头开始
+        String nowDate=syncDataReadTaskServiceManager.getCurrDateReduceDay(task, null);
+        if(StringUtils.isBlank(endDate)) {
+            endDate=nowDate;
+        }
+        String currDate=checkDateForFailureConditions(task.getFailureConditions(),startDate);
+        boolean flag=true;
+        while(flag && currDate.compareTo(endDate)<0) {
+            //失败就结束，调度器会再触发
+            flag=syncDataReadTaskServiceManager.doHiveReaderByTaskNo(task,currDate,params);
+            currDate=addDays(currDate, 1);//加1
+        }
+        finishTaskHandler(task, flag);
+        return flag?R.ok("finish"):R.fail("unfinish");
+    }
+    
+    
+    /**
+     * 检查任务状态，是否要开始
+     * @param taskNo
+     * @return
+     * @throws Exception
+     */
+    private TblBatchTaskVO checkTaskHandler(String taskNo) throws Exception{
+        TblBatchTaskVO task = batchTaskService.getDetailBatchTask(taskNo);
+        if(task==null || task.getMappingInfo()==null) {
+            throw new RuntimeException(taskNo+": Task does not exist");
+        }
+        if(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode().equals(task.getBusStatus())) {
+            return null;
+        }
+        //开始任务
+        boolean startFlag = batchTaskService.doBatchTaskStart(task);
+        if(!startFlag) {
+            return null;
+        }
+        return task;
+    }
+    /**
+     * 结束任务
+     * @param taskNo
+     * @return
+     * @throws Exception
+     */
+    private void finishTaskHandler(TblBatchTaskVO task,boolean flag) throws Exception{
+        if(flag) {
+            batchTaskService.doBatchTaskFinish(task);
+        }else {
+            batchTaskService.doBatchTaskUnFinish(task);
+        }
+    }
+    /**
+     * 检查字符串是否日期
+     * 是则返回， 不是则返回传入的startDate
+     * @param str
+     * @return
+     * @throws Exception 
+     */
+    private String checkDateForFailureConditions(String str,String startDate) throws Exception {
+        if(StringUtils.isBlank(str)) {
+            return startDate;
+        }
+        try {
+            new SimpleDateFormat("yyyy-MM-dd").format(str);
+            return str;
+        }catch(Exception e) {
+            return startDate;
+        }
+    }
+    private String addDays(String currDate,int day) throws Exception {
+        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+        Date date = DateUtils.addDays(dateFormat.parse(currDate), day);
+        return dateFormat.format(date);
+    }
+    
+    private  Map<String,String> getUrlParams(HttpServletRequest request) throws Exception{
+        Map<String,String> params=new HashMap<>();
+        Enumeration<String> names = request.getParameterNames();
+        while(names.hasMoreElements()) {
+            String namex = names.nextElement();
+            params.put(namex,(request.getParameter(namex)));
+        }
+        return params;
+    }
+    
+    /**
+     * jsy的，刷新Kerberos凭证(票据的失效时间配置成>=24小时)
+     * 因为单纯的刷新很不灵活，所以还是重登录一次，避免首次运行和长久未运行
+     */
+    @Scheduled(cron = "0 0 */23 * * ?")
+    public void refreshTicket() {
+        try {
+            DDsProperties ds = syncDataReadTaskServiceManager.getDataSourceConf(SyncDataConstants.DB_NAME_JSY_HIVE_DS);
+            String command = String.format(SyncDataConstants.SHELL_KERBEROS_KINIT, ds.getUsername());
+            Process process = Runtime.getRuntime().exec(command);
+            OutputStream outputStream = process.getOutputStream();
+            PrintWriter printWriter = new PrintWriter(outputStream);
+            printWriter.println(ds.getPassword());
+            printWriter.flush();
+            int exitCode= process.waitFor();
+            if (exitCode != 0) {
+                LOGGER.error("refresh Kerberos TGT：Check if the Kerberos command exists and permissions！！");
+            }else {
+                LOGGER.warn("refresh Kerberos TGT Finish.");
+            }
+            printWriter.close();
+        } catch (Exception e) {
+            LOGGER.error("refresh Kerberos TGT Error", e);
+            if(e instanceof InterruptedException) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
+    @PostConstruct
+    public void firstRefreshTicket() {
+        refreshTicket();//启动就初始化一次，这样可以不用先手动创建
+    }
+    
+    
+    /**
+     * 清理临时文件
+     * @return
+     * @throws Exception
+     */
+    @GetMapping("clearTempFile")
+    public R<String> clearTempFile() throws Exception{
+        //如果写入的是文件，则会产生临时文件，定时清理1个月之前文件
+        if(StringUtils.isNotBlank(defaultClearTempFileDs)) {
+            String[] dsarr=defaultClearTempFileDs.split(",");
+            for(String dsx:dsarr) {
+                if(StringUtils.isBlank(dsx)) {
+                    continue;
+                }
+                dsx= dsx.trim();
+                DDsProperties ds = syncDataReadTaskServiceManager.getDataSourceConf(dsx);
+                if(ds==null || StringUtils.isBlank(ds.getReadWritePath())) {
+                    continue;
+                }
+                clearFile(ds.getReadWritePath(),defaultClearTempFileDays);
+            }
+        }
+        
+        return R.ok("finish");
+    }
+    private static void clearFile(String path,int days) {
+        File directory = new File(path);
+        // 检查目录是否存在
+        if (!directory.exists() || !directory.isDirectory()) {
+            return;
+        }
+        // 获取目录下的文件列表
+        File[] files = directory.listFiles();
+        // 遍历文件列表
+        for (File file : files) {
+            if(!file.isFile()) {
+                continue;
+            }
+            // 获取文件的修改时间
+            Date modifiedDate = new Date(file.lastModified());
+            Date date = DateUtils.addDays(modifiedDate, days);
+            Date currDate = new Date();
+            int d= DateUtils.truncatedCompareTo(date, currDate, Calendar.DATE);
+            if(d<0 && file.delete()) {
+                //ok
+            }
+        }
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/controller/SyncDataTestController.java

@@ -0,0 +1,239 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.controller;
+
+import java.io.File;
+import java.sql.ResultSet;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Matcher;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.env.Environment;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.jiuyv.sptcc.agile.batch.common.R;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.DDsProperties;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader.FileBaseReader;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader.JdbcBaseReader;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader.ReaderWriterHelper;
+
+
+/**
+ * 这个用于测试
+ * @author zhouliang
+ *
+ */
+@RestController
+@RequestMapping("/batch/test/")
+public class SyncDataTestController {
+    private static final Logger LOGGER = LoggerFactory.getLogger(SyncDataTestController.class);
+    
+    @Autowired
+    private Environment environment;
+    
+    @Value("${console.readWritePath}")
+    private String defaultReadWritePath;
+    /**
+     * 执行单条sql。主要方便获取所有表
+     * @param sql
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("getTables")
+    public R<Object> getTables(String dbName,String sql,String fieldSeparator,String filename) throws Exception{
+        LOGGER.info("getTables>>dbName={}，sql={}，fieldSeparator={}，filename={}",dbName,sql,fieldSeparator,filename);
+        if(StringUtils.isBlank(dbName) || StringUtils.isBlank(sql)) {
+            return R.fail("dbName不能为空"+"，sql不能为空");
+        }
+        if(StringUtils.isBlank(fieldSeparator)) {
+            fieldSeparator="\t";
+        }
+        String tablecode=SyncDataConstants.getTablecode(sql);
+        JdbcBaseReader jdbcReader = null;
+        FileBaseReader fileReader = null;
+        try {
+            ReaderWriterHelper readerWriterHelper=new ReaderWriterHelper();
+            DDsProperties ds=getDataSourceConf(dbName);
+            readerWriterHelper.createJdbcReader(dbName,ds);
+            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HHmmss");
+            String date= dateFormat.format(Date.from(Instant.now()));
+            String targetPath=defaultReadWritePath+File.separator+dbName+"-table-"+tablecode+date+".txt";
+            if(StringUtils.isNotBlank(filename)) {
+               targetPath=defaultReadWritePath+File.separator+filename+".txt";
+            }
+            readerWriterHelper.createFileWriter("txt",targetPath);
+            readerWriterHelper.setReadFileFlag(false);
+            readerWriterHelper.setWriteFileFlag(true);
+            
+            jdbcReader = readerWriterHelper.getJdbcReader();
+            fileReader = readerWriterHelper.getFileReader();
+            
+            ResultSet resultSet=jdbcReader.execSql(sql, null);
+            long count=0;
+            if(resultSet!=null) {
+                int n=resultSet.getMetaData().getColumnCount();
+                if(StringUtils.isBlank(filename)) {
+                    List<String> titles=new ArrayList<>();
+                    for(int i=1;i<=n;i++) {
+                        String colname =resultSet.getMetaData().getColumnName(i);
+                        if(colname.contains("\\.")) {
+                            titles.add(colname.replace("^.*\\.", ""));
+                        }else {
+                            titles.add(colname);
+                        }
+                    }
+                    fileReader.clearWriteTitle(StringUtils.join(titles, fieldSeparator),true);
+                }
+                //读取数据，N条写一次
+                List<List<Object>> lists=new ArrayList<>();
+                while(resultSet.next()) {
+                    count++;
+                    List<Object> row= new ArrayList<>();
+                    for(int i=1;i<=n;i++) {
+                        row.add(resultSet.getString(i));
+                    }
+                    lists.add(row);
+                    if(lists.size()==30000) {//够数
+                        writeDataJdbcOrFile(fileReader,lists,count,fieldSeparator);
+                        lists=new ArrayList<>();
+                    }
+                }
+                writeDataJdbcOrFile(fileReader,lists,count,fieldSeparator);
+                jdbcReader.closeCurrStatement();
+            }
+        } catch (Exception e) {
+            LOGGER.info("writeData error>>{},{}",e.getMessage(),e);
+            return R.fail(e.getMessage());
+        } finally {
+            if(null!=jdbcReader) {
+                jdbcReader.closeConnection();
+            }
+            if(null!=fileReader) {
+                fileReader.close();
+            }
+        }
+        return R.ok();
+    }
+    
+    private void writeDataJdbcOrFile(FileBaseReader fileReader,List<List<Object>> lists,long count,String fieldSeparator) throws Exception {
+        if(!lists.isEmpty()) {
+            LOGGER.info("writeData Task Progress>>total={}",count);
+            List<String> lists2=new ArrayList<>();
+            for(List<Object> lx:lists) {
+                lists2.add(StringUtils.join(lx, fieldSeparator));
+            }
+            fileReader.writeLines(lists2, "\n");
+        }
+    }
+    
+    /**
+     * 循环执行一条sql。主要为了快速获取多张表的字段信息
+     * @param sql
+     * @param tables
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("getTableColumns")
+    public R<Object> getTableColumns(String dbName,String sql,String tables,String fieldSeparator,String filename) throws Exception{
+        LOGGER.info("getTables>>dbName={}，sql={}，fieldSeparator={}，filename={}，tables={}",dbName,sql,fieldSeparator,filename,tables);
+        if(StringUtils.isBlank(dbName) || StringUtils.isBlank(sql) || StringUtils.isBlank(tables)) {
+            return R.fail("dbName不能为空"+"，sql不能为空"+"，tables不能为空");
+        }
+        if(StringUtils.isBlank(fieldSeparator)) {
+            fieldSeparator="\t";
+        }
+        
+        String tablecode=SyncDataConstants.getTablecode(sql);
+        JdbcBaseReader jdbcReader = null;
+        FileBaseReader fileReader = null;
+        try {
+            ReaderWriterHelper readerWriterHelper=new ReaderWriterHelper();
+            DDsProperties ds=getDataSourceConf(dbName);
+            readerWriterHelper.createJdbcReader(dbName,ds);
+            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HHmmss");
+            String date= dateFormat.format(Date.from(Instant.now()));
+            String targetPath=defaultReadWritePath+File.separator+dbName+"-table2-"+tablecode+date+".txt";
+            if(StringUtils.isNotBlank(filename)) {
+                targetPath=defaultReadWritePath+File.separator+filename+".txt";
+            }
+            readerWriterHelper.createFileWriter("txt",targetPath);
+            readerWriterHelper.setReadFileFlag(false);
+            readerWriterHelper.setWriteFileFlag(true);
+            
+            jdbcReader = readerWriterHelper.getJdbcReader();
+            fileReader = readerWriterHelper.getFileReader();
+            
+            String[] codes = tables.split(" *, *");
+            for(String x:codes) {
+                if(StringUtils.isBlank(x)) {
+                    continue;
+                }
+                String sql2=sql.replace("PP_table", x);
+                ResultSet resultSet=jdbcReader.execSql(sql2, null);
+                long count=0;
+                if(resultSet!=null) {
+                    int n=resultSet.getMetaData().getColumnCount();
+                    if(StringUtils.isBlank(filename)) {
+                        List<String> titles=new ArrayList<>();
+                        for(int i=1;i<=n;i++) {
+                            titles.add(resultSet.getMetaData().getColumnName(i));
+                        }
+                        fileReader.clearWriteTitle(StringUtils.join(titles, fieldSeparator),true);
+                    }
+                    //读取数据，N条写一次
+                    List<List<Object>> lists=new ArrayList<>();
+                    while(resultSet.next()) {
+                        count++;
+                        List<Object> row= new ArrayList<>();
+                        for(int i=1;i<=n && i<3;i++) {
+                            row.add(resultSet.getString(i));
+                        }
+                        lists.add(row);
+                        if(lists.size()==30000) {//够数
+                            writeDataJdbcOrFile(fileReader,lists,count,fieldSeparator);
+                            lists=new ArrayList<>();
+                        }
+                    }
+                    writeDataJdbcOrFile(fileReader,lists,count,fieldSeparator);
+                    jdbcReader.closeCurrStatement();
+                }
+            }
+        } catch (Exception e) {
+            LOGGER.info("writeData error>>{},{}",e.getMessage(),e);
+            return R.fail(e.getMessage());
+        } finally {
+            if(null!=jdbcReader) {
+                jdbcReader.closeConnection();
+            }
+            if(null!=fileReader) {
+                fileReader.close();
+            }
+        }
+        return R.ok();
+    }
+    
+   
+    private DDsProperties getDataSourceConf(String dbName) {
+        DDsProperties dsProperties = new DDsProperties();
+        String url=environment.getProperty(dbName+".url");
+        String username=environment.getProperty(dbName+".username");
+        String password=environment.getProperty(dbName+".password");
+        String driverClassName= environment.getProperty(dbName+".driverClassName");
+        String confPath=environment.getProperty(dbName+".confPath");
+        dsProperties.setUrl(url);
+        dsProperties.setUsername(username);
+        dsProperties.setPassword(password);
+        dsProperties.setDriverClassName(driverClassName);
+        dsProperties.setConfPath(confPath);
+        return dsProperties;
+    }
+}

agile-bacth/agile-batch-service/src/main/java/com/jiuyv/sptcc/agile/batch/syncJiushiData/service/SyncDataReadTaskManager.java

@@ -0,0 +1,187 @@
+package com.jiuyv.sptcc.agile.batch.syncJiushiData.service;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping;
+import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
+import com.jiuyv.sptcc.agile.batch.common.BaseTime;
+import com.jiuyv.sptcc.agile.batch.dao.ISysTimeBaseMapper;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.DDsProperties;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SqlHandlerUtilx;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
+import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.reader.ReaderWriterHelper;
+
+/**
+ * 
+ * @author zhouliang
+ *
+ */
+@Component
+public class SyncDataReadTaskManager {
+    private static final Logger LOGGER = LoggerFactory.getLogger(SyncDataReadTaskManager.class);
+    
+    @Autowired
+    private ISysTimeBaseMapper sysTimeBaseMapper;
+    
+    @Autowired
+    private Environment environment;
+    @Value("${syncdata.singleWriteNumber}")
+    private int defaultSingleWriteNumber;
+    @Value("${console.readWritePath}")
+    private String defaultReadWritePath;
+    
+    
+    //获取系统时间
+    public BaseTime getSysDate() throws Exception {
+        BaseTime timeVO = sysTimeBaseMapper.selectSysCurrentTime();
+        timeVO.setDateTime(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+                .withLocale(Locale.SIMPLIFIED_CHINESE).withZone(ZoneId.of(timeVO.getTimeZone())).format(timeVO.getUtcTime()));
+        timeVO.setDateDay(timeVO.getDateTime().substring(0, 10));
+        return timeVO;
+    }
+    
+    public boolean doHiveReaderByTaskNo(TblBatchTaskVO task, String currDate, Map<String,String> params) throws Exception {
+        boolean finishFlag=true;
+        //内部报错不影响任务状态处理
+        try {
+            currDate=getCurrDateReduceDay(task,currDate);
+            LOGGER.info("doHiveReaderByTaskNo>>currDate={}",currDate);
+            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HHmmss");
+            String datetime = dateFormat.format(Date.from(Instant.now()));
+            //转换字段映射
+            TblBatchTableMapping mappingInfo = task.getMappingInfo();
+            
+            //处理自定义条件参数
+            Map<String,String> sqlParams=new HashMap<>();
+            sqlParams.putAll(params);
+            sqlParams.put("currYear", currDate.substring(0,4));//年
+            sqlParams.put("currMonth", currDate.substring(0,7));//年月
+            sqlParams.put("currMonthSimple", sqlParams.get("currMonth").replace("-", ""));//纯数字形式
+            sqlParams.put("currDate", currDate);//目前都是按天处理,如果有他条件再加
+            sqlParams.put("currDateSimple", currDate.replace("-", ""));//纯数字形式
+            sqlParams.put("datetime", datetime);//主要是用于文件名称
+            
+            ReaderWriterHelper readerWriterHelper=new ReaderWriterHelper();
+            readerWriterHelper.setSingleWriteNumber(defaultSingleWriteNumber);
+            boolean readFileFlag=false;//默认是读数据库
+            boolean writeFileFlag=false;//默认写是数据库
+            DDsProperties ds1=getDataSourceConf(mappingInfo.getRemoteDbName());
+            if(readerWriterHelper.isFile(mappingInfo.getRemoteDbName())) {
+                readFileFlag=true;
+                String sourcePath=defaultReadWritePath+File.separator+mappingInfo.getRemoteTableSql();
+                sourcePath=SqlHandlerUtilx.replaceSqlCustomParams(sourcePath, sqlParams);
+                readerWriterHelper.createFileReader(mappingInfo.getRemoteDbName(),sourcePath);
+            }else {
+                if(ds1==null) {
+                    throw new RuntimeException("Database not configured>>dbName="+mappingInfo.getRemoteDbName());
+                }
+                readerWriterHelper.createJdbcReader(mappingInfo.getRemoteDbName(),ds1);
+            }
+            DDsProperties ds2=getDataSourceConf(mappingInfo.getLocalDbName());
+            if(readerWriterHelper.isFile(mappingInfo.getLocalDbName()) || readerWriterHelper.isFile(mappingInfo.getLocalTable())) {
+                String readerWritePath2=defaultReadWritePath;
+                if(ds2!=null) {
+                    if(ds2.getSingleWriteNumber()!=null&&ds2.getSingleWriteNumber()>0) {
+                        readerWriterHelper.setSingleWriteNumber(ds2.getSingleWriteNumber());
+                    }
+                    if(StringUtils.isNotBlank(ds2.getFieldSeparator())) {
+                        readerWriterHelper.setFieldSeparator(ds2.getFieldSeparator());
+                    }
+                    if(StringUtils.isNotBlank(ds2.getReadWritePath())) {
+                        readerWritePath2=ds2.getReadWritePath();
+                    }
+                }
+                writeFileFlag=true;
+                String targetPath=readerWritePath2+File.separator+mappingInfo.getLocalTable();
+                targetPath=SqlHandlerUtilx.replaceSqlCustomParams(targetPath, sqlParams);
+                readerWriterHelper.createFileWriter(mappingInfo.getLocalDbName(),targetPath);
+            }else {
+                if(ds2==null) {
+                    throw new RuntimeException("Database not configured>>dbName="+mappingInfo.getLocalDbName());
+                }
+                readerWriterHelper.createJdbcWriter(mappingInfo.getLocalDbName(),ds2);
+            }
+            readerWriterHelper.setReadFileFlag(readFileFlag);
+            readerWriterHelper.setWriteFileFlag(writeFileFlag);
+            
+            boolean successFlag = readerWriterHelper.writeData(mappingInfo, sqlParams);
+            if(!successFlag) {
+                finishFlag=false;
+                task.setFailureConditions(currDate);//主要是批量跑历史
+            }
+        }catch(Exception e) {
+            LOGGER.info("doHiveReaderByTaskNo error>>{}",e.getMessage(),e);
+            finishFlag=false;
+            task.setFailureConditions(currDate);//主要是批量跑历史
+        }
+        LOGGER.info("doHiveReaderByTaskNo>>finishFlag={}",finishFlag);
+        return finishFlag;
+    }
+    
+    public DDsProperties getDataSourceConf(String dbName) {
+        DDsProperties dsProperties = new DDsProperties();
+        String url=environment.getProperty(dbName+".url");
+        String username=environment.getProperty(dbName+".username");
+        String password=environment.getProperty(dbName+".password");
+        String driverClassName= environment.getProperty(dbName+".driverClassName");
+        String confPath=environment.getProperty(dbName+".confPath");
+        String singleWriteNumber=environment.getProperty(dbName+".singleWriteNumber");
+        String readWritePath=environment.getProperty(dbName+".readWritePath");
+        String fieldSeparator=environment.getProperty(dbName+".fieldSeparator");
+        dsProperties.setUrl(url);
+        dsProperties.setUsername(username);
+        dsProperties.setPassword(password);
+        dsProperties.setDriverClassName(driverClassName);
+        dsProperties.setConfPath(confPath);
+        if(StringUtils.isNotBlank(singleWriteNumber)) {
+            dsProperties.setSingleWriteNumber(Integer.valueOf(singleWriteNumber));
+        }
+        if(StringUtils.isNotBlank(readWritePath)) {
+            dsProperties.setReadWritePath(readWritePath);
+        }
+        if(StringUtils.isNotBlank(fieldSeparator)) {
+            dsProperties.setFieldSeparator(fieldSeparator);
+        }
+        //如果这些属性全为空则认为是文件或没有配置，返回null
+        if(StringUtils.isBlank(url) && StringUtils.isBlank(username) && StringUtils.isBlank(singleWriteNumber)
+                && StringUtils.isBlank(readWritePath) && StringUtils.isBlank(fieldSeparator)) {
+            return null;
+        }
+        
+        return dsProperties;
+    }
+    
+    /**
+     * 获取当天前推N天的日期
+     * 如果currDate不为空，那么原样返回
+     * @param currDate
+     * @return
+     * @throws Exception
+     */
+    public String getCurrDateReduceDay(TblBatchTaskVO task,String currDate) throws Exception {
+        BaseTime timeVO = getSysDate();
+        if(StringUtils.isBlank(currDate)&&task!=null) {
+            Instant ntime = timeVO.getUtcTime().plus(-task.getMappingInfo().getRemoteDays(), ChronoUnit.DAYS);
+            Date date =new Date(ntime.toEpochMilli());
+            currDate = new SimpleDateFormat("yyyy-MM-dd").format(date);
+        }
+        return currDate;
+    }
+}

agile-bacth/agile-batch-service/src/main/resources/application-dev.yml

@@ -0,0 +1,62 @@
+server:
+  port: 18081
+
+spring:
+  application: 
+    name : batch-service
+  # 服务模块
+  devtools:
+    restart:
+      # 热部署开关
+      enabled: true
+  datasource: #数据源配置
+    driver-class-name: org.postgresql.Driver
+    url: jdbc:postgresql://172.16.12.105:5432/keliubao
+    username: postgres
+    password: postgres
+
+console: 
+  #readWritePath: /home/flink/read_write_data
+  readWritePath: F:\ZLworkspace\agilesystem\agile.batch\src\trunk\agile-bacth\agile-batch-service\read_write_data
+syncdata:
+  singleWriteNumber: 20000 #读取一定数量写入一次
+  clearTempFileDs: klbHiveDs #需清理的数据库，同步任务临时文件、过期文件
+  clearTempFileDays: 15 #过期天数，更新时间超过N天即过期
+
+klbHiveDs: #客流宝hive
+  url: jdbc:hive2://172.16.12.101:10000/hive;socketTimeout=12000;
+  username: flink
+  password: flink
+  driverClassName: org.apache.hive.jdbc.HiveDriver
+  singleWriteNumber: 2000 #读取一定数量写入一次,覆盖默认
+  readWritePath: ${console.readWritePath}2 #文件存放路径,覆盖默认
+  fieldSeparator: ',' #文件字段分隔符号，默认就是逗号
+klbPgDs: #客流宝pg
+  url: ${spring.datasource.url}
+  username: ${spring.datasource.username}
+  password: ${spring.datasource.password}
+  driverClassName: ${spring.datasource.driver-class-name}
+jsyHiveDs: #久事云hive(这个名称不能改)
+  url: 
+  username: gjpf
+  password: Huawei@123
+  driverClassName: org.apache.hive.jdbc.HiveDriver
+  #confPath: /home/sptcc/ #暂未整合,直接写url好像很长
+  confPath: F:\ZLworkspace\agilesystem\agile.batch\src\trunk\agile-bacth\agile-batch-service\src\main\resources\config\hiveJsy
+sjztHiveDs: #数据中台hive
+  url: jdbc:hive2://10.99.104.121:10000/
+  username: minjie
+  password: minjie_123
+  driverClassName: org.apache.hive.jdbc.HiveDriver
+  
+management:
+  endpoints:
+    web:
+      exposure:
+        include: '*'
+  endpoint:
+    health:
+      show-details: ALWAYS
+    shutdown:
+      enabled: true
+  

agile-bacth/agile-batch-service/src/main/resources/application.yml

@@ -0,0 +1,16 @@
+mybatis:
+  type-aliases-super-type:
+  mapper-locations: classpath:mappers/*xml
+  # 加载全局的配置文件
+  configLocation: classpath:mybatis/mybatis-config.xml
+  
+spring:
+  jackson:
+    date-format: yyyy-MM-dd HH:mm:ss # 常用的时间格式
+    default-property-inclusion: non_null # 忽略空对象
+    serialization:
+      fail-on-empty-beans: false # 序列化空对象时不抛出异常
+      indent-output: true # 输出格式化为缩进的JSON
+      write-dates-as-timestamps: false # 日期序列化为时间戳而不是ISO-8601格式
+    deserialization:
+      fail-on-unknown-properties: false # 反序列化时忽略未知的属性

agile-bacth/agile-batch-service/src/main/resources/bootstrap.yml

@@ -0,0 +1,41 @@
+# Spring配置
+spring:
+  application: 
+    name : batch-service
+  profiles:
+    #部署时带上自身名字batch-service-dev
+    active: dev
+  cloud: 
+    config:
+      #本地可以置为关闭false
+      enabled: false
+      discovery:
+        service-id: config-service #使用服务名
+        enabled: true 
+      #uri: http://172.16.12.109:8888
+      name: ${spring.application.name}
+      profile: ${spring.profiles.active}
+      fail-fast: true
+  security:
+    user:
+      name: sptcc  
+      password: 123456
+    
+# 配置eureka客户端信息
+eureka:
+  instance:
+    appname: ${spring.application.name}
+    lease-expiration-duration-in-seconds: 30
+    lease-renewal-interval-in-seconds: 10
+    prefer-ip-address: true
+    instance-id: ${spring.cloud.client.ip-address}:${spring.application.name}:${server.port}
+    metadata-map:
+      user.name: ${spring.security.user.name}
+      user.password: ${spring.security.user.password}
+  client:
+    enabled: true
+    registry-fetch-interval-seconds: 1
+    register-with-eureka: true
+    fetch-registry: true
+    service-url:
+      defaultZone: http://172.16.12.109:8761/eureka/

agile-bacth/agile-batch-service/src/main/resources/config/hiveJsy/hiveclient.properties

@@ -0,0 +1,9 @@
+auth = KERBEROS
+zk.port = 24002
+zk.quorum = 11.125.1.4:24002,11.125.1.6:24002,11.125.1.5:24002
+beeline.entirelineascommand = true
+principal = hive/hadoop.hadoop.com@HADOOP.COM
+sasl.qop = auth-conf
+zooKeeperNamespace = hiveserver2
+serviceDiscoveryMode = zooKeeper
+instanceNo = 0

agile-bacth/agile-batch-service/src/main/resources/config/hiveJsy/krb5.conf

@@ -0,0 +1,47 @@
+[kdcdefaults]
+kdc_ports = 11.125.1.5:21732
+kdc_tcp_ports = ""
+
+[libdefaults]
+default_realm = HADOOP.COM
+kdc_timeout = 2500
+clockskew = 300
+use_dns_lookup = 0
+udp_preference_limit = 1465
+max_retries = 5
+dns_lookup_kdc = false
+dns_lookup_realm = false
+renewable = false
+forwardable = false
+renew_lifetime = 0m
+max_renewable_life = 30m
+allow_extend_version = false
+default_ccache_name = FILE:/tmp//krb5cc_%{uid}
+
+[realms]
+HADOOP.COM = {
+kdc = 11.125.1.5:21732
+kdc = 11.125.1.4:21732
+admin_server = 11.125.1.5:21730
+admin_server = 11.125.1.4:21730
+kpasswd_server = 11.125.1.5:21731
+kpasswd_server = 11.125.1.4:21731
+kpasswd_port = 21731
+kadmind_port = 21730
+kadmind_listen = 11.125.1.5:21730
+kpasswd_listen = 11.125.1.5:21731
+renewable = false
+forwardable = false
+renew_lifetime = 0m
+max_renewable_life = 30m
+acl_file = /opt/huawei/Bigdata/FusionInsight_BASE_6.5.1.10/install/FusionInsight-kerberos-1.17/kerberos/var/krb5kdc/kadm5.acl
+key_stash_file = /opt/huawei/Bigdata/FusionInsight_BASE_6.5.1.10/install/FusionInsight-kerberos-1.17/kerberos/var/krb5kdc/.k5.HADOOP.COM
+}
+
+[domain_realm]
+.hadoop.com = HADOOP.COM
+
+[logging]
+kdc = SYSLOG:INFO:DAEMON
+admin_server = SYSLOG:INFO:DAEMON
+default = SYSLOG:NOTICE:DAEMON

agile-bacth/agile-batch-service/src/main/resources/logback-spring.xml

@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration debug="false">
+    <!-- LOG目录 -->
+    <property name="LOG_HOME" value="logs"/>
+
+    <!-- JSON_LOG目录 -->
+    <property name="JSON_LOG_HOME" value="logs/jsonlog"/>
+
+    <!-- spring.application.name 作为参数 -->
+    <springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <target>System.out</target>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <!-- 按天压缩日志 -->
+    <appender name="ZIP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_HOME}/${APP_NAME}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!--日志文件输出的文件名-->
+            <FileNamePattern>${LOG_HOME}/${APP_NAME}.log.%d{yyyy-MM-dd}.%i.log.zip</FileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <!-- or whenever the file size reaches 100MB -->
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <!-- 不压缩日志,保留7天 -->
+    <appender name="DEFAULT_INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!--日志文件输出的文件名-->
+            <FileNamePattern>${LOG_HOME}/${APP_NAME}.log.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
+            <maxHistory>7</maxHistory>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <!-- or whenever the file size reaches 100MB -->
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <!-- LOGSTASH,输出seuleth项,保留7天 -->
+    <appender name="LOGSTASH" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!--日志文件输出的文件名-->
+            <FileNamePattern>${JSON_LOG_HOME}/${APP_NAME}.json.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
+            <maxHistory>7</maxHistory>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <!-- or whenever the file size reaches 100MB -->
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+            <providers>
+                <pattern>
+                    <pattern>
+                        {
+                        "timestamp": "%d{yyyy-MM-dd HH:mm:ss.SSS}",
+                        "severity": "%level",
+                        "service": "${APP_NAME:-}",
+                        "trace": "%X{X-B3-TraceId:-}",
+                        "span": "%X{X-B3-SpanId:-}",
+                        "parent": "%X{X-B3-ParentSpanId:-}",
+                        "exportable": "%X{X-Span-Export:-}",
+                        "pid": "${PID:-}",
+                        "thread": "%thread",
+                        "class": "%logger{40}",
+                        "rest": "%message"
+                        }
+                    </pattern>
+                </pattern>
+            </providers>
+        </encoder>
+    </appender>
+    <!-- 按天压缩Json日志 -->
+    <appender name="JSON_ZIP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${JSON_LOG_HOME}/${APP_NAME}.json.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!--日志文件输出的文件名-->
+            <FileNamePattern>${JSON_LOG_HOME}/${APP_NAME}.log.%d{yyyy-MM-dd}.%i.json.log.zip</FileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <!-- or whenever the file size reaches 100MB -->
+                <maxFileSize>100MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+        </rollingPolicy>
+        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+            <providers>
+                <pattern>
+                    <pattern>
+                        {
+                        "timestamp": "%d{yyyy-MM-dd HH:mm:ss.SSS}",
+                        "severity": "%level",
+                        "service": "${APP_NAME:-}",
+                        "trace": "%X{X-B3-TraceId:-}",
+                        "span": "%X{X-B3-SpanId:-}",
+                        "parent": "%X{X-B3-ParentSpanId:-}",
+                        "exportable": "%X{X-Span-Export:-}",
+                        "pid": "${PID:-}",
+                        "thread": "%thread",
+                        "class": "%logger{40}",
+                        "rest": "%message"
+                        }
+                    </pattern>
+                </pattern>
+            </providers>
+        </encoder>
+    </appender>
+    <appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
+        <target>System.err</target>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="com.jiuyv.sptccc.agile.thirdparty.dao" level="debug"/>
+    <logger name="org.apache.http" level="error"/>
+
+    <root>
+        <level value="INFO"/>
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="ZIP_FILE"/>
+        <appender-ref ref="DEFAULT_INFO_FILE"/>
+        <appender-ref ref="LOGSTASH"/>
+        <appender-ref ref="JSON_ZIP_FILE"/>
+    </root>
+</configuration>

agile-bacth/agile-batch-service/src/main/resources/mappers/SysBaseMapper.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="com.jiuyv.sptcc.agile.batch.dao.ISysTimeBaseMapper">
+    <select id="selectSysCurrentTime" resultType="com.jiuyv.sptcc.agile.batch.common.BaseTime">
+        SELECT now() AT TIME ZONE 'UTC' AS utcTime, now() AS date
+    </select>
+</mapper>

agile-bacth/agile-batch-service/src/main/resources/mappers/TblBatchTableMappingMapper.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<!-- 批处理同步表映射表 -->
+<mapper namespace="com.jiuyv.sptcc.agile.batch.batchTask.mapper.TblBatchTableMappingMapper">
+    <sql id="_select">
+        a.task_no,
+        a.version_num,
+        a.rec_token,
+        a.remote_table_sql,
+        a.remote_db_name,
+        a.remote_days,
+        a.local_table,
+        a.local_db_name,
+        a.local_pre_sql,
+        a.mapping_json,
+        a.remarks,
+        a.data_status,
+        a.update_time,
+        a.rsv1,
+        a.rsv2,
+        a.rsv3
+    </sql>
+    <sql id="_where">
+      <!-- 默认必有条件 -->
+      <choose>
+        <when test="taskNo != null">and a.task_no = #{taskNo}</when>
+        <otherwise>and a.task_no in(<foreach collection="taskNos" item="idx" separator=",">#{idx}</foreach>)</otherwise>
+      </choose>
+      <if test="versionNum != null" >and version_num = #{versionNum}</if>
+      <if test="recToken != null" >and rec_token = #{recToken}</if>
+      <if test="dataStatus != null" >and data_status = #{dataStatus}</if>
+    </sql>
+    
+    <!-- 查询单条  -->
+    <select id="selectOneByMap" resultType="com.jiuyv.sptcc.agile.batch.batchTask.entity.TblBatchTableMapping" parameterType="com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTableMappingVO">
+      select
+        <include refid="_select"/>
+      from tbl_batch_table_mapping a
+      <where>
+        <include refid="_where"/>
+      </where>
+    </select>
+
+
+</mapper>

agile-bacth/agile-batch-service/src/main/resources/mappers/TblBatchTaskMapper.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<!-- 批处理任务表 -->
+<mapper namespace="com.jiuyv.sptcc.agile.batch.batchTask.mapper.TblBatchTaskMapper">
+    <sql id="_select">
+        a.task_no,
+        a.version_num,
+        a.rec_token,
+        a.task_title,
+        a.pre_start_date,
+        a.pre_end_date,
+        a.pre_total_time,
+        a.curr_start_date,
+        a.failure_conditions,
+        a.bus_status,
+        a.data_status,
+        a.update_time,
+        a.rsv1,
+        a.rsv2,
+        a.rsv3
+    </sql>
+    <sql id="_where">
+      <!-- 默认必有条件 -->
+      and a.task_no = #{taskNo}
+      <if test="versionNum != null" >and version_num = #{versionNum}</if>
+      <if test="recToken != null" >and rec_token = #{recToken}</if>
+      <if test="dataStatus != null" >and data_status = #{dataStatus}</if>
+    </sql>
+    
+    <!-- 查询单条  -->
+    <select id="selectOneByMap" resultType="com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO" parameterType="com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO">
+      select
+        <include refid="_select"/>
+      from tbl_batch_task a
+      <where>
+        <include refid="_where"/>
+      </where>
+    </select>
+    
+    <!-- 更新记录  -->
+    <update id="updateByMap">
+      update tbl_batch_task 
+      <set>
+        <if test="vo.versionNum != null" >version_num = #{vo.versionNum},</if>
+        <if test="vo.versionNum == null" >version_num = version_num+1,</if>
+        <if test="vo.recToken != null and vo.recToken != ''" >rec_token = #{vo.recToken},</if>
+        <if test="vo.taskTitle != null and vo.taskTitle != ''" >task_title = #{vo.taskTitle},</if>
+        <if test="vo.preStartDate != null" >pre_start_date = #{vo.preStartDate},</if>
+        <if test="vo.preEndDate != null" >pre_end_date = #{vo.preEndDate},</if>
+        <if test="vo.preTotalTime != null and vo.preTotalTime != ''" >pre_total_time = #{vo.preTotalTime},</if>
+        <if test="vo.currStartDate != null" >curr_start_date = #{vo.currStartDate},</if>
+        <if test="vo.failureConditions != null and vo.failureConditions != ''" >failure_conditions = #{vo.failureConditions},</if>
+        <if test="vo.busStatus != null and vo.busStatus != ''" >bus_status = #{vo.busStatus},</if>
+        <if test="vo.dataStatus != null and vo.dataStatus != ''" >data_status = #{vo.dataStatus},</if>
+        <if test="vo.updateTime != null" >update_time = #{vo.updateTime},</if>
+        <if test="vo.rsv1 != null and vo.rsv1 != ''" >rsv1 = #{vo.rsv1},</if>
+        <if test="vo.rsv2 != null and vo.rsv2 != ''" >rsv2 = #{vo.rsv2},</if>
+        <if test="vo.rsv3 != null and vo.rsv3 != ''" >rsv3 = #{vo.rsv3},</if>
+      </set>
+      <where>
+        <!-- 默认必有条件 -->
+        and task_no = #{map.taskNo}
+        <if test="map.versionNum != null" >and version_num = #{map.versionNum}</if>
+        <if test="map.recToken != null" >and rec_token = #{map.recToken}</if>
+        <if test="map.dataStatus != null" >and data_status = #{map.dataStatus}</if>
+        <if test="map.busStatus != null" >and bus_status = #{map.busStatus}</if>
+        <if test="map.busStatuss != null" >and bus_status in(<foreach collection="map.busStatuss" item="idx" separator=",">#{idx}</foreach>)</if>
+      </where> 
+    </update>
+    
+    <update id="updateResetAllBusStatus">
+      update tbl_batch_task 
+      <set>
+        bus_status = #{vo.busStatus},
+        update_time = #{vo.updateTime},
+      </set>
+      <where>
+        and bus_status = #{map.busStatus}
+      </where> 
+    </update>
+
+</mapper>

agile-bacth/agile-batch-service/src/main/resources/mybatis/mybatis-config.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE configuration
+PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
+"http://mybatis.org/dtd/mybatis-3-config.dtd">
+<configuration>
+    <!-- 全局参数 -->
+    <settings>
+        <!-- 使全局的映射器启用或禁用缓存 -->
+        <setting name="cacheEnabled"             value="true"   />
+        <!-- 允许JDBC 支持自动生成主键 -->
+        <setting name="useGeneratedKeys"         value="true"   />
+        <!-- 配置默认的执行器.SIMPLE就是普通执行器;REUSE执行器会重用预处理语句(prepared statements);BATCH执行器将重用语句并执行批量更新 -->
+        <setting name="defaultExecutorType"      value="SIMPLE" />
+        <!-- 指定 MyBatis 所用日志的具体实现 -->
+        <setting name="logImpl"                  value="SLF4J"  />
+        <!-- 使用驼峰命名法转换字段 -->
+        <setting name="mapUnderscoreToCamelCase" value="true"/>
+    </settings>
+
+</configuration>

agile-bacth/agile-batch-service/src/test/resources/application.yml

@@ -0,0 +1,14 @@
+spring:
+  h2:
+    console:
+      enabled: false
+  datasource:
+    driver-class-name:  org.h2.Driver
+    url: jdbc:h2:file:~/test
+    username: san
+    password:
+
+  sql:
+    init:
+      data-locations: classpath:db/data.sql
+      schema-locations: classpath:/dbschema.sql

agile-bacth/pom.xml

@@ -0,0 +1,149 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.jiuyv.sptcc.agile.batch</groupId>
+    <artifactId>agile-bacth</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <modules>
+        <module>agile-batch-api</module>
+        <module>agile-batch-service</module>
+        <module>agile-batch-dws</module>
+    </modules>
+    <packaging>pom</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <java.version>1.8</java.version>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven-jar-plugin.version>3.1.1</maven-jar-plugin.version>
+        <maven.clean.version>3.1.0</maven.clean.version>
+        <maven.resource.version>3.1.0</maven.resource.version>
+        <spring.boot.version>2.6.7</spring.boot.version>
+        <hive.version>3.1.0</hive.version>
+        <spring-cloud.version>2021.0.5</spring-cloud.version>
+    </properties>
+
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.cloud</groupId>
+                <artifactId>spring-cloud-dependencies</artifactId>
+                <version>${spring-cloud.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+            <!-- SpringBoot的依赖配置 -->
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring.boot.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+
+            <dependency>
+                <groupId>org.mybatis.spring.boot</groupId>
+                <artifactId>mybatis-spring-boot-starter</artifactId>
+                <version>2.2.2</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <distributionManagement>
+        <repository>
+            <id>nexus-releases</id>
+            <name>Internal Releases</name>
+            <url>http://172.16.12.11:8082/repository/maven-releases/</url>
+        </repository>
+
+        <snapshotRepository>
+            <id>nexus-snapshots</id>
+            <name>Internal Snapshots</name>
+            <url>http://172.16.12.11:8082/repository/maven-snapshots/</url>
+        </snapshotRepository>
+    </distributionManagement>
+
+    <scm>
+        <connection>scm:svn:http://172.16.12.10/sptcc_agile_etl/src/agile-batch/src/trunk/agile-batch</connection>
+        <developerConnection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile-batch/src/trunk/agile-batch</developerConnection>
+    </scm>
+
+    <repositories>
+        <repository>
+            <id>jiuyv</id>
+            <name>jiuyv</name>
+            <url>http://172.16.12.11:8082/repository/maven-public/</url>
+            <snapshots>
+                <enabled>true</enabled>
+                <updatePolicy>always</updatePolicy>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>jboss</id>
+            <name>jboss</name>
+            <url>http://repository.jboss.org/maven2/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>geotools</id>
+            <name>geotools</name>
+            <url>http://maven.geotools.fr/repository/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>jahia</id>
+            <name>jahia</name>
+            <url>http://maven.jahia.org/maven2/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>vars</id>
+            <name>vars</name>
+            <url>http://vars.sourceforge.net/maven2/</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>jiuyv</id>
+            <name>jiuyv Plugin Repository</name>
+            <url>http://172.16.12.11:8082/repository/maven-public/</url>
+        </pluginRepository>
+        <pluginRepository>
+            <id>central</id>
+            <name>Maven Plugin Repository</name>
+            <url>http://repo1.maven.org/maven2/</url>
+        </pluginRepository>
+    </pluginRepositories>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>${java.version}</source>
+                    <target>${java.version}</target>
+                    <encoding>${project.build.sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

agile-portal/agile-portal-api/pom.xml

@@ -10,7 +10,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>com.jiuyv.sptcc.portal</groupId>
-    <artifactId>agile-portsl-api</artifactId>
+    <artifactId>agile-portal-api</artifactId>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>
@@ -19,8 +19,8 @@
 
     <dependencies>
         <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-web</artifactId>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-web</artifactId>
         </dependency>
 
         <!--eureka-client-->
@@ -29,6 +29,12 @@
             <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-openfeign</artifactId>
+            <version>${openfeign.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.jiuyv.agile</groupId>
             <artifactId>agile-common</artifactId>

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/ContentFeignApi.java

@@ -15,11 +15,8 @@ import java.util.List;
 public interface ContentFeignApi {
     String API_PATH_PREFIX = "/content";
 
-    @GetMapping("/banner")
-    R<List<PortalContentDTO>> getBanners();
-
-    @GetMapping("/scenesList")
-    R<List<PortalContentDTO>> getScenesList();
+    @GetMapping("/contentList")
+    R<List<PortalContentDTO>> getContentList(@RequestParam("showType") String showType);
 
     @PostMapping("/information")
     TableDataInfo<PortalContentDTO> getInformationList(@RequestBody ReqPageDTO pageDTO);
@@ -27,4 +24,7 @@ public interface ContentFeignApi {
     @GetMapping("/contentInfo/{contentId}")
     R<PortalContentDTO> contentInfo(@PathVariable("contentId") Long contentId);
 
+    @GetMapping("/images/{imageName}")
+    R<byte[]> getImage(@PathVariable("imageName") String imageName);
+
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/DataApiFeignApi.java

@@ -3,22 +3,21 @@ package com.jiuyv.sptccc.agile.api;
 import com.jiuyv.sptccc.agile.common.core.page.TableDataInfo;
 import com.jiuyv.sptccc.agile.dto.DataApiDTO;
 import com.jiuyv.sptccc.agile.dto.DataApiStatisticsDTO;
-import com.jiuyv.sptccc.agile.dto.ReqDataApiPageDTO;
 import com.jiuyv.sptccc.agile.dto.ReqPageDTO;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 
 public interface DataApiFeignApi {
-    String API_PATH_PREFIX = "api";
+    String API_PATH_PREFIX = "/api";
 
     @PostMapping("/list")
     TableDataInfo<DataApiDTO> getList(@RequestBody ReqPageDTO pageDTO);
 
     @PostMapping("/userApiList")
-    TableDataInfo<DataApiDTO> getUserApiList(@RequestBody ReqDataApiPageDTO pageDTO);
+    TableDataInfo<DataApiDTO> getUserApiList(@RequestBody ReqPageDTO pageDTO);
 
     @PostMapping("/userApiStatistics")
-    TableDataInfo<DataApiStatisticsDTO> getUserApiStatistics(@RequestBody ReqDataApiPageDTO pageDTO);
+    TableDataInfo<DataApiStatisticsDTO> getUserApiStatistics(@RequestBody ReqPageDTO pageDTO);
 
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/DockerApplyFeignApi.java

@@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 public interface DockerApplyFeignApi {
+    String API_PATH_PREFIX = "/dockerApply";
 
     @PostMapping("/list")
     TableDataInfo<DockerApplyInfoDTO> getList(@RequestBody ReqDockerApplyPageDTO reqDTO);

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/DockerDownloadApplyFeignApi.java

@@ -1,14 +1,21 @@
 package com.jiuyv.sptccc.agile.api;
 
+import com.jiuyv.sptccc.agile.common.core.domain.R;
 import com.jiuyv.sptccc.agile.common.core.page.TableDataInfo;
 import com.jiuyv.sptccc.agile.dto.DockerDownloadApplyDTO;
+import com.jiuyv.sptccc.agile.dto.FileTO;
 import com.jiuyv.sptccc.agile.dto.ReqDockerDownApplyPageDTO;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 public interface DockerDownloadApplyFeignApi {
-    String API_PATH_PREFIX = "downloadApply";
+    String API_PATH_PREFIX = "/downloadApply";
 
     @PostMapping("/list")
     TableDataInfo<DockerDownloadApplyDTO> getList(@RequestBody ReqDockerDownApplyPageDTO reqDTO);
+
+    @GetMapping("/download/{downloadApplyId}")
+    R<FileTO> download(@PathVariable("downloadApplyId") Long downloadApplyId);
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/DockerWithUserFeignApi.java

@@ -2,7 +2,9 @@ package com.jiuyv.sptccc.agile.api;
 
 import com.jiuyv.sptccc.agile.common.core.domain.R;
 import com.jiuyv.sptccc.agile.common.core.page.TableDataInfo;
+import com.jiuyv.sptccc.agile.dto.DockerFileDTO;
 import com.jiuyv.sptccc.agile.dto.DockerWithUserDTO;
+import com.jiuyv.sptccc.agile.dto.ReqDockerDownApplyDTO;
 import com.jiuyv.sptccc.agile.dto.ReqDockerWithUserDTO;
 import com.jiuyv.sptccc.agile.dto.ReqDockerWithUserPageDTO;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -10,11 +12,12 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 
 import java.util.List;
 
 public interface DockerWithUserFeignApi {
-    String API_PATH_PREFIX = "dockerWithUser";
+    String API_PATH_PREFIX = "/dockerWithUser";
 
     @PostMapping("/list")
     TableDataInfo<DockerWithUserDTO> getList(@RequestBody ReqDockerWithUserPageDTO reqDTO);
@@ -23,13 +26,15 @@ public interface DockerWithUserFeignApi {
     R<DockerWithUserDTO> getInfo(@PathVariable("applyId") Long applyId);
 
     @PutMapping("/fileBind")
-    R<Void> fileBind(ReqDockerWithUserDTO reqDTO);
+    R<Void> fileBind(@RequestBody ReqDockerWithUserDTO reqDTO);
 
     @PutMapping("/restart")
-    R<Void> restart(ReqDockerWithUserDTO reqDTO);
+    R<Void> restart(@RequestBody ReqDockerWithUserDTO reqDTO);
 
     @GetMapping("/fileList/{applyId}")
-    R<List<String>> fileList(@PathVariable("applyId") Long applyId);
+    R<List<DockerFileDTO>> fileList(@PathVariable("applyId") Long applyId);
 
+    @PutMapping("/applyDown")
+    R<Void> applyDown(@RequestBody ReqDockerDownApplyDTO reqDockerDownApplyDTO);
 
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/FileFeignApi.java

@@ -16,10 +16,12 @@ import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.multipart.MultipartFile;
 
 public interface FileFeignApi {
-    String API_PATH_PREFIX = "file";
+    String API_PATH_PREFIX = "/file";
 
     @PostMapping(value = "/uploadFiles", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
-    R<UploadFileDTO> uploadFiles(@RequestPart("file") MultipartFile file, @RequestParam("remarks") String remarks);
+    R<UploadFileDTO> uploadFiles(@RequestPart("file") MultipartFile file,
+                                 @RequestParam("fileType") String fileType,
+                                 @RequestParam("remarks") String remarks);
 
     @PostMapping("/list")
     TableDataInfo<UploadFileDTO> getList(@RequestBody ReqFileDTO pageDTO);

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/api/PortalUserFeignApi.java

@@ -2,21 +2,25 @@ package com.jiuyv.sptccc.agile.api;
 
 import com.jiuyv.sptccc.agile.common.core.domain.R;
 import com.jiuyv.sptccc.agile.dto.PortalUserDTO;
+import com.jiuyv.sptccc.agile.dto.ResUserPasswordDTO;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
 
 public interface PortalUserFeignApi {
-    String API_PATH_PREFIX = "portalUser";
+    String API_PATH_PREFIX = "/portalUser";
 
     @GetMapping("/selectUserByUserName")
     R<PortalUserDTO> selectUserByUserName(@RequestParam("username") String username);
 
     /**
      * 重置用户登陆状态等信息
-     *
      */
     @PutMapping("/resetError")
     R<Void> resetUserError(@RequestBody PortalUserDTO req);
+
+    @PostMapping("/resetUserPwd")
+    R<Void> resetUserPwd(@RequestBody ResUserPasswordDTO passwordDTO);
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/DockerApplyInfoDTO.java

@@ -2,6 +2,7 @@ package com.jiuyv.sptccc.agile.dto;
 
 import java.io.Serializable;
 import java.util.Date;
+import java.util.List;
 
 /**
  * 实验室数据上传申请
@@ -13,11 +14,6 @@ public class DockerApplyInfoDTO implements Serializable {
      */
     private Long applyId;
 
-    /**
-     * 版本号
-     */
-    private Long versionNum;
-
     /**
      * 随机码
      */
@@ -108,11 +104,6 @@ public class DockerApplyInfoDTO implements Serializable {
      */
     private String remarks;
 
-    /**
-     * 排序
-     */
-    private Long orderNum;
-
     /**
      * 审核状态
      */
@@ -124,49 +115,9 @@ public class DockerApplyInfoDTO implements Serializable {
     private String reviewDesc;
 
     /**
-     * 是否发布
-     */
-    private String releaseFlag;
-
-    /**
-     * 业务状态
-     */
-    private String busStatus;
-
-    /**
-     * 数据状态
-     */
-    private String dataStatus;
-
-    /**
-     * 创建用户id
-     */
-    private String createBy;
-
-    /**
-     * 创建用户
+     * 申请组件
      */
-    private String createByName;
-
-    /**
-     * 创建时间
-     */
-    private Date createTime;
-
-    /**
-     * 更新用户id
-     */
-    private String updateBy;
-
-    /**
-     * 更新用户
-     */
-    private String updateByName;
-
-    /**
-     * 更新时间
-     */
-    private Date updateTime;
+    private List<DockerLibDTO> applyLibList;
 
     public Long getApplyId() {
         return applyId;
@@ -176,14 +127,6 @@ public class DockerApplyInfoDTO implements Serializable {
         this.applyId = applyId;
     }
 
-    public Long getVersionNum() {
-        return versionNum;
-    }
-
-    public void setVersionNum(Long versionNum) {
-        this.versionNum = versionNum;
-    }
-
     public String getRecToken() {
         return recToken;
     }
@@ -328,14 +271,6 @@ public class DockerApplyInfoDTO implements Serializable {
         this.remarks = remarks;
     }
 
-    public Long getOrderNum() {
-        return orderNum;
-    }
-
-    public void setOrderNum(Long orderNum) {
-        this.orderNum = orderNum;
-    }
-
     public String getReviewStatus() {
         return reviewStatus;
     }
@@ -352,75 +287,11 @@ public class DockerApplyInfoDTO implements Serializable {
         this.reviewDesc = reviewDesc;
     }
 
-    public String getReleaseFlag() {
-        return releaseFlag;
-    }
-
-    public void setReleaseFlag(String releaseFlag) {
-        this.releaseFlag = releaseFlag;
-    }
-
-    public String getBusStatus() {
-        return busStatus;
-    }
-
-    public void setBusStatus(String busStatus) {
-        this.busStatus = busStatus;
-    }
-
-    public String getDataStatus() {
-        return dataStatus;
-    }
-
-    public void setDataStatus(String dataStatus) {
-        this.dataStatus = dataStatus;
-    }
-
-    public String getCreateBy() {
-        return createBy;
-    }
-
-    public void setCreateBy(String createBy) {
-        this.createBy = createBy;
-    }
-
-    public String getCreateByName() {
-        return createByName;
-    }
-
-    public void setCreateByName(String createByName) {
-        this.createByName = createByName;
-    }
-
-    public Date getCreateTime() {
-        return createTime;
-    }
-
-    public void setCreateTime(Date createTime) {
-        this.createTime = createTime;
-    }
-
-    public String getUpdateBy() {
-        return updateBy;
-    }
-
-    public void setUpdateBy(String updateBy) {
-        this.updateBy = updateBy;
-    }
-
-    public String getUpdateByName() {
-        return updateByName;
-    }
-
-    public void setUpdateByName(String updateByName) {
-        this.updateByName = updateByName;
-    }
-
-    public Date getUpdateTime() {
-        return updateTime;
+    public List<DockerLibDTO> getApplyLibList() {
+        return applyLibList;
     }
 
-    public void setUpdateTime(Date updateTime) {
-        this.updateTime = updateTime;
+    public void setApplyLibList(List<DockerLibDTO> applyLibList) {
+        this.applyLibList = applyLibList;
     }
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/DockerDownloadApplyDTO.java

@@ -0,0 +1,125 @@
+package com.jiuyv.sptccc.agile.dto;
+
+import java.io.Serializable;
+
+public class DockerDownloadApplyDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * 【 申请主键ID】
+     */
+    private Long downloadApplyId;
+
+    /**
+     * 【 随机码】
+     */
+    private String recToken;
+
+    /**
+     * 【 关联主键ID】
+     */
+    private Long applyId;
+
+    /**
+     * 【实验室名称】允许用户自定义名称，反之默认生成
+     */
+    private String labTitle;
+
+    /**
+     * 【 申请原因】
+     */
+    private String applyDesc;
+
+    /**
+     * 【文件名】
+     */
+    private String fileName;
+
+    /**
+     * 【 备注】
+     */
+    private String remarks;
+
+    /**
+     * 【 审核状态】
+     */
+    private String reviewStatus;
+
+    /**
+     * 【 驳回原因】
+     */
+    private String reviewDesc;
+
+    public Long getDownloadApplyId() {
+        return downloadApplyId;
+    }
+
+    public void setDownloadApplyId(Long downloadApplyId) {
+        this.downloadApplyId = downloadApplyId;
+    }
+
+    public String getRecToken() {
+        return recToken;
+    }
+
+    public void setRecToken(String recToken) {
+        this.recToken = recToken;
+    }
+
+    public Long getApplyId() {
+        return applyId;
+    }
+
+    public void setApplyId(Long applyId) {
+        this.applyId = applyId;
+    }
+
+    public String getLabTitle() {
+        return labTitle;
+    }
+
+    public void setLabTitle(String labTitle) {
+        this.labTitle = labTitle;
+    }
+
+    public String getApplyDesc() {
+        return applyDesc;
+    }
+
+    public void setApplyDesc(String applyDesc) {
+        this.applyDesc = applyDesc;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public String getRemarks() {
+        return remarks;
+    }
+
+    public void setRemarks(String remarks) {
+        this.remarks = remarks;
+    }
+
+    public String getReviewStatus() {
+        return reviewStatus;
+    }
+
+    public void setReviewStatus(String reviewStatus) {
+        this.reviewStatus = reviewStatus;
+    }
+
+    public String getReviewDesc() {
+        return reviewDesc;
+    }
+
+    public void setReviewDesc(String reviewDesc) {
+        this.reviewDesc = reviewDesc;
+    }
+
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/DockerFileDTO.java

@@ -0,0 +1,17 @@
+package com.jiuyv.sptccc.agile.dto;
+
+import java.io.Serializable;
+
+public class DockerFileDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private String fileName;
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/DockerLibDTO.java

@@ -9,11 +9,6 @@ public class DockerLibDTO implements Serializable {
      */
     private Long applyLibId;
 
-    /**
-     * 版本号
-     */
-    private Long versionNum;
-
     /**
      * 随机码
      */
@@ -39,6 +34,10 @@ public class DockerLibDTO implements Serializable {
      */
     private String fileName;
 
+    /**
+     * 【 数据状态】
+     */
+    private String dataStatus;
 
     /**
      * 内容说明
@@ -53,14 +52,6 @@ public class DockerLibDTO implements Serializable {
         this.applyLibId = applyLibId;
     }
 
-    public Long getVersionNum() {
-        return versionNum;
-    }
-
-    public void setVersionNum(Long versionNum) {
-        this.versionNum = versionNum;
-    }
-
     public String getRecToken() {
         return recToken;
     }
@@ -101,6 +92,14 @@ public class DockerLibDTO implements Serializable {
         this.fileName = fileName;
     }
 
+    public String getDataStatus() {
+        return dataStatus;
+    }
+
+    public void setDataStatus(String dataStatus) {
+        this.dataStatus = dataStatus;
+    }
+
     public String getLibDesc() {
         return libDesc;
     }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/DockerWithUserDTO.java

@@ -1,7 +1,6 @@
 package com.jiuyv.sptccc.agile.dto;
 
 import java.io.Serializable;
-import java.util.Date;
 import java.util.List;
 
 public class DockerWithUserDTO implements Serializable {
@@ -68,9 +67,19 @@ public class DockerWithUserDTO implements Serializable {
 
 
     /**
-     * 容器使用自传组件
+     * 管控台组件
      */
-    private List<DockerLibDTO> dockerApplyLib;
+    private List<DockerLibDTO> libList;
+
+    /**
+     * 申请组件
+     */
+    private List<DockerLibDTO> applyLibList;
+
+    /**
+     * 容器文件
+     */
+    private List<DockerFileDTO> dockerFileList;
 
 
     public Long getApplyId() {
@@ -169,11 +178,27 @@ public class DockerWithUserDTO implements Serializable {
         this.loginUsername = loginUsername;
     }
 
-    public List<DockerLibDTO> getDockerApplyLib() {
-        return dockerApplyLib;
+    public List<DockerLibDTO> getLibList() {
+        return libList;
+    }
+
+    public void setLibList(List<DockerLibDTO> libList) {
+        this.libList = libList;
+    }
+
+    public List<DockerLibDTO> getApplyLibList() {
+        return applyLibList;
+    }
+
+    public void setApplyLibList(List<DockerLibDTO> applyLibList) {
+        this.applyLibList = applyLibList;
+    }
+
+    public List<DockerFileDTO> getDockerFileList() {
+        return dockerFileList;
     }
 
-    public void setDockerApplyLib(List<DockerLibDTO> dockerApplyLib) {
-        this.dockerApplyLib = dockerApplyLib;
+    public void setDockerFileList(List<DockerFileDTO> dockerFileList) {
+        this.dockerFileList = dockerFileList;
     }
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/FileTO.java

@@ -0,0 +1,22 @@
+package com.jiuyv.sptccc.agile.dto;
+
+public class FileTO {
+    private String fileName;
+    private byte[] file;
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public byte[] getFile() {
+        return file;
+    }
+
+    public void setFile(byte[] file) {
+        this.file = file;
+    }
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/PortalContentDTO.java

@@ -53,6 +53,17 @@ public class PortalContentDTO implements Serializable {
 
     private Integer sort;
 
+
+    /**
+     * 首页播报 0展示
+     */
+    private String showIndex;
+
+    /**
+     * 副标题
+     */
+    private String subtitle;
+
     public void setContentId(Long contentId)
     {
         this.contentId = contentId;
@@ -177,4 +188,20 @@ public class PortalContentDTO implements Serializable {
     public void setUpdateTime(Date updateTime) {
         this.updateTime = updateTime;
     }
+
+    public String getShowIndex() {
+        return showIndex;
+    }
+
+    public void setShowIndex(String showIndex) {
+        this.showIndex = showIndex;
+    }
+
+    public String getSubtitle() {
+        return subtitle;
+    }
+
+    public void setSubtitle(String subtitle) {
+        this.subtitle = subtitle;
+    }
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqDockerApplyPageDTO.java

@@ -14,11 +14,6 @@ public class ReqDockerApplyPageDTO extends ReqPageDTO {
      */
     private String applyUserId;
 
-    /**
-     * 申请用户
-     */
-    private String applyUserName;
-
     /**
      * 实验室名称
      */
@@ -34,10 +29,6 @@ public class ReqDockerApplyPageDTO extends ReqPageDTO {
      */
     private String reviewStatus;
 
-    /**
-     * 业务状态
-     */
-    private String busStatus;
 
     public Long getApplyId() {
         return applyId;
@@ -55,14 +46,6 @@ public class ReqDockerApplyPageDTO extends ReqPageDTO {
         this.applyUserId = applyUserId;
     }
 
-    public String getApplyUserName() {
-        return applyUserName;
-    }
-
-    public void setApplyUserName(String applyUserName) {
-        this.applyUserName = applyUserName;
-    }
-
     public String getLabTitle() {
         return labTitle;
     }
@@ -86,12 +69,4 @@ public class ReqDockerApplyPageDTO extends ReqPageDTO {
     public void setReviewStatus(String reviewStatus) {
         this.reviewStatus = reviewStatus;
     }
-
-    public String getBusStatus() {
-        return busStatus;
-    }
-
-    public void setBusStatus(String busStatus) {
-        this.busStatus = busStatus;
-    }
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqDockerDownApplyDTO.java

@@ -0,0 +1,48 @@
+package com.jiuyv.sptccc.agile.dto;
+
+import java.io.Serializable;
+
+/**
+ * 文件下载申请 请求体
+ */
+public class ReqDockerDownApplyDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+    /**
+     * 实验室ID
+     */
+    private Long applyId;
+
+    /**
+     * 文件名
+     */
+    private String fileName;
+
+    /**
+     * 申请说明
+     */
+    private String applyDesc;
+
+    public Long getApplyId() {
+        return applyId;
+    }
+
+    public void setApplyId(Long applyId) {
+        this.applyId = applyId;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public String getApplyDesc() {
+        return applyDesc;
+    }
+
+    public void setApplyDesc(String applyDesc) {
+        this.applyDesc = applyDesc;
+    }
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqDockerDownApplyPageDTO.java

@@ -0,0 +1,75 @@
+package com.jiuyv.sptccc.agile.dto;
+
+/**
+ * 文件下载申请 请求
+ */
+public class ReqDockerDownApplyPageDTO extends ReqPageDTO{
+
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * 申请用户id
+     */
+    private String applyUserId;
+
+    /**
+     * 实验室申请id
+     */
+    private Long applyId;
+
+    /**
+     * 实验室名称
+     */
+    private String labTitle;
+
+    /**
+     * 【文件名】
+     */
+    private String fileName;
+
+    /**
+     * 审核状态
+     */
+    private String reviewStatus;
+
+    public String getApplyUserId() {
+        return applyUserId;
+    }
+
+    public void setApplyUserId(String applyUserId) {
+        this.applyUserId = applyUserId;
+    }
+
+    public Long getApplyId() {
+        return applyId;
+    }
+
+    public void setApplyId(Long applyId) {
+        this.applyId = applyId;
+    }
+
+    public String getLabTitle() {
+        return labTitle;
+    }
+
+    public void setLabTitle(String labTitle) {
+        this.labTitle = labTitle;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public String getReviewStatus() {
+        return reviewStatus;
+    }
+
+    public void setReviewStatus(String reviewStatus) {
+        this.reviewStatus = reviewStatus;
+    }
+
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqDockerWithUserDTO.java

@@ -1,8 +1,10 @@
 package com.jiuyv.sptccc.agile.dto;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ReqDockerWithUserDTO {
+public class ReqDockerWithUserDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
     /**
      * 【 申请主键ID】
      */
@@ -13,6 +15,9 @@ public class ReqDockerWithUserDTO {
      */
     private String recToken;
 
+    /**
+     * 文件列表
+     */
     private List<Long> fileIds;
 
     public Long getApplyId() {

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqDockerWithUserPageDTO.java

@@ -9,20 +9,15 @@ public class ReqDockerWithUserPageDTO extends ReqPageDTO {
      */
     private Long applyId;
 
-    /**
-     * 实验室名称
-     */
-    private String labTitle;
-
     /**
      * 用户
      */
     private String applyUserId;
 
     /**
-     * 用户
+     * 实验室名称
      */
-    private String applyUserName;
+    private String labTitle;
 
     /**
      * 业务状态
@@ -37,14 +32,6 @@ public class ReqDockerWithUserPageDTO extends ReqPageDTO {
         this.applyId = applyId;
     }
 
-    public String getLabTitle() {
-        return labTitle;
-    }
-
-    public void setLabTitle(String labTitle) {
-        this.labTitle = labTitle;
-    }
-
     public String getApplyUserId() {
         return applyUserId;
     }
@@ -53,12 +40,12 @@ public class ReqDockerWithUserPageDTO extends ReqPageDTO {
         this.applyUserId = applyUserId;
     }
 
-    public String getApplyUserName() {
-        return applyUserName;
+    public String getLabTitle() {
+        return labTitle;
     }
 
-    public void setApplyUserName(String applyUserName) {
-        this.applyUserName = applyUserName;
+    public void setLabTitle(String labTitle) {
+        this.labTitle = labTitle;
     }
 
     public String getBusStatus() {

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ReqPageDTO.java

@@ -20,7 +20,6 @@ public class ReqPageDTO implements Serializable {
      * 是否升序
      */
     private String isAsc;
-    private Boolean reasonable = true;
 
 
     public Integer getPageNum() {
@@ -55,11 +54,4 @@ public class ReqPageDTO implements Serializable {
         this.isAsc = isAsc;
     }
 
-    public Boolean getReasonable() {
-        return reasonable;
-    }
-
-    public void setReasonable(Boolean reasonable) {
-        this.reasonable = reasonable;
-    }
 }

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/ResUserPasswordDTO.java

@@ -0,0 +1,37 @@
+package com.jiuyv.sptccc.agile.dto;
+
+import java.io.Serializable;
+
+public class ResUserPasswordDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private Long userId;
+
+    private String password;
+
+    private String oldPassword;
+
+    public Long getUserId() {
+        return userId;
+    }
+
+    public void setUserId(Long userId) {
+        this.userId = userId;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getOldPassword() {
+        return oldPassword;
+    }
+
+    public void setOldPassword(String oldPassword) {
+        this.oldPassword = oldPassword;
+    }
+}

agile-portal/agile-portal-api/src/main/java/com/jiuyv/sptccc/agile/dto/UploadFileDTO.java

@@ -1,6 +1,9 @@
 package com.jiuyv.sptccc.agile.dto;
 
+import com.fasterxml.jackson.annotation.JsonFormat;
+
 import java.io.Serializable;
+import java.util.Date;
 
 public class UploadFileDTO implements Serializable {
 
@@ -35,6 +38,11 @@ public class UploadFileDTO implements Serializable {
 
     /** 文件备注 */
     private String remarks;
+    /**
+     * 创建时间
+     */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private Date createTime;
 
     public Long getFileId() {
         return fileId;
@@ -115,4 +123,12 @@ public class UploadFileDTO implements Serializable {
     public void setRemarks(String remarks) {
         this.remarks = remarks;
     }
+
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
 }

agile-portal/agile-portal-gateway/pom.xml

@@ -9,11 +9,10 @@
     <artifactId>agile-portal-gateway</artifactId>
 
     <dependencies>
-
         <dependency>
             <groupId>com.jiuyv.sptcc.portal</groupId>
-            <artifactId>agile-portsl-api</artifactId>
-            <version>${agile-portsl-api.version}</version>
+            <artifactId>agile-portal-api</artifactId>
+            <version>${agile-portal-api.version}</version>
         </dependency>
 
         <dependency>
@@ -22,48 +21,13 @@
             <version>${agile-mobile-message-api.version}</version>
         </dependency>
 
-
-        <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-openfeign</artifactId>
-            <version>${openfeign.version}</version>
-        </dependency>
-
-        <!-- Bucket4j -->
+        <!-- Bucket4j 限流 -->
         <dependency>
             <groupId>com.github.vladimir-bukhtoyarov</groupId>
             <artifactId>bucket4j-core</artifactId>
             <version>7.6.0</version>
         </dependency>
 
-        <!-- 解析客户端操作系统、浏览器等 -->
-        <dependency>
-            <groupId>eu.bitwalker</groupId>
-            <artifactId>UserAgentUtils</artifactId>
-            <version>${bitwalker.version}</version>
-        </dependency>
-
-        <!-- SpringBoot集成mybatis框架 -->
-        <dependency>
-            <groupId>org.mybatis.spring.boot</groupId>
-            <artifactId>mybatis-spring-boot-starter</artifactId>
-            <version>${mybatis-spring-boot.version}</version>
-        </dependency>
-
-        <!-- pagehelper 分页插件 -->
-        <dependency>
-            <groupId>com.github.pagehelper</groupId>
-            <artifactId>pagehelper-spring-boot-starter</artifactId>
-            <version>${pagehelper.boot.version}</version>
-        </dependency>
-
-        <!-- 获取系统信息 -->
-        <dependency>
-            <groupId>com.github.oshi</groupId>
-            <artifactId>oshi-core</artifactId>
-            <version>${oshi.version}</version>
-        </dependency>
-
         <!-- io常用工具类 -->
         <dependency>
             <groupId>commons-io</groupId>
@@ -71,34 +35,6 @@
             <version>${commons.io.version}</version>
         </dependency>
 
-        <!-- 文件上传工具类 -->
-        <dependency>
-            <groupId>commons-fileupload</groupId>
-            <artifactId>commons-fileupload</artifactId>
-            <version>${commons.fileupload.version}</version>
-        </dependency>
-
-        <!-- excel工具 -->
-        <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi-ooxml</artifactId>
-            <version>${poi.version}</version>
-        </dependency>
-
-        <!-- velocity代码生成使用模板 -->
-        <dependency>
-            <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity-engine-core</artifactId>
-            <version>${velocity.version}</version>
-        </dependency>
-
-        <!-- collections工具类 -->
-        <dependency>
-            <groupId>commons-collections</groupId>
-            <artifactId>commons-collections</artifactId>
-            <version>${commons.collections.version}</version>
-        </dependency>
-
         <!-- json logstash encoder -->
         <dependency>
             <groupId>net.logstash.logback</groupId>
@@ -106,111 +42,30 @@
             <version>6.4</version>
         </dependency>
 
-        <!-- Spring框架基本的核心工具 -->
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-context-support</artifactId>
-        </dependency>
-
         <!-- SpringWeb模块 -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
 
-
         <!-- spring security 安全认证 -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-security</artifactId>
         </dependency>
 
-
-        <!-- 自定义验证注解 -->
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-validation</artifactId>
-        </dependency>
-
         <!--常用工具类 -->
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
 
-
-        <!-- yml解析器 -->
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-        </dependency>
-
-
-        <!-- Jaxb -->
-        <dependency>
-            <groupId>javax.xml.bind</groupId>
-            <artifactId>jaxb-api</artifactId>
-        </dependency>
-
-        <!-- pool 对象池 -->
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-pool2</artifactId>
-        </dependency>
-
-
-        <!-- servlet包 -->
-        <dependency>
-            <groupId>javax.servlet</groupId>
-            <artifactId>javax.servlet-api</artifactId>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/com.alibaba/easyexcel -->
-        <dependency>
-            <groupId>com.alibaba</groupId>
-            <artifactId>easyexcel</artifactId>
-            <version>3.1.1</version>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/org.aspectj/aspectjweaver -->
-        <dependency>
-            <groupId>org.aspectj</groupId>
-            <artifactId>aspectjweaver</artifactId>
-        </dependency>
-
-        <!-- SpringBoot 拦截器 -->
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-aop</artifactId>
-        </dependency>
-
-
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-configuration-processor</artifactId>
             <optional>true</optional>
         </dependency>
 
-
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-quartz</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-thymeleaf</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpmime</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-cache</artifactId>
@@ -221,18 +76,6 @@
             <artifactId>caffeine</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.axis</groupId>
-            <artifactId>axis</artifactId>
-            <version>1.4</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.bouncycastle</groupId>
-            <artifactId>bcpkix-jdk15on</artifactId>
-            <version>1.70</version>
-        </dependency>
-
         <dependency>
             <groupId>com.anji-plus</groupId>
             <artifactId>captcha</artifactId>
@@ -269,12 +112,15 @@
                 <configuration>
                     <!--指定生成文档的使用的配置文件,配置文件放在自己的项目中,如果不指定,默认文件名default.json-->
                     <configFile>./src/main/resources/smart-doc.json</configFile>
+                    <includes>
+                        <include>com.jiuyv.sptcc.portal:agile-portal-gateway</include>
+                    </includes>
                 </configuration>
             </plugin>
 
             <!--前后端合并-->
             <!--clean插件 -->
-            <!--<plugin>
+            <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-clean-plugin</artifactId>
                 <version>${maven.clean.version}</version>
@@ -302,14 +148,14 @@
                             <overwrite>true</overwrite>
                             <resources>
                                 <resource>
-                                    &lt;!&ndash;因为vue-cli打包的目录在项目的根目录，所以从这里复制 &ndash;&gt;
+                                    <!--因为vue-cli打包的目录在项目的根目录，所以从这里复制 -->
                                     <directory>${project.parent.basedir}/agile-portal-ui/dist</directory>
                                 </resource>
                             </resources>
                         </configuration>
                     </execution>
                 </executions>
-            </plugin>-->
+            </plugin>
         </plugins>
     </build>
 

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/config/ConsoleConfig.java

@@ -26,6 +26,11 @@ public class ConsoleConfig {
      */
     private String copyrightYear;
 
+    /**
+     * RSA加密算法填充方案，默认RSA/None/NoPadding
+     */
+    private String pwdEncAlg = "RSA";
+
     /**
      * 手机验证码测试开关
      */
@@ -60,6 +65,14 @@ public class ConsoleConfig {
         this.copyrightYear = copyrightYear;
     }
 
+    public String getPwdEncAlg() {
+        return pwdEncAlg;
+    }
+
+    public void setPwdEncAlg(String pwdEncAlg) {
+        this.pwdEncAlg = pwdEncAlg;
+    }
+
     public boolean isCaptchaTest() {
         return captchaTest;
     }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/constant/Constants.java

@@ -7,8 +7,24 @@ package com.jiuyv.sptccc.agile.common.constant;
 public class Constants {
 
     /**
-     * 变量名:用户信息
+     * 变量名:登录用户信息
      */
     public static final String LOGIN_USER_INFO = "loginUserInfo";
 
+    /**
+     * 变量名:找回密码用户信息
+     */
+    public static final String RE_PASSWORD_USER_INFO = "rePasswordUserInfo";
+
+
+    /**
+     * 变量名:登录验证码模板
+     */
+    public static final String LOGIN_VERIFY_CODE_TEMPLATE = "portal_login";
+
+    /**
+     * 变量名:登录验证码模板
+     */
+    public static final String RE_PASSWORD_VERIFY_CODE_TEMPLATE = "portal_re_password";
+
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/core/controller/BaseController.java

@@ -1,31 +1,33 @@
 package com.jiuyv.sptccc.agile.common.core.controller;
 
-import java.util.List;
-
+import com.jiuyv.sptccc.agile.common.core.domain.AjaxResult;
+import com.jiuyv.sptccc.agile.common.core.domain.R;
+import com.jiuyv.sptccc.agile.common.core.page.TableDataInfo;
+import com.jiuyv.sptccc.agile.common.exception.ServiceException;
+import com.jiuyv.sptccc.agile.dto.DataApiDTO;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
 
-import com.jiuyv.sptccc.agile.common.core.page.TableDataInfo;
-
 /**
  * web层通用数据处理
  *
- * @author admin
  */
-public class BaseController {
+public abstract class BaseController {
     protected final Logger logger = LoggerFactory.getLogger(this.getClass());
-    /**
-     * 响应请求分页数据
-     */
-    @SuppressWarnings({"rawtypes", "unchecked"})
-    protected <T> TableDataInfo<T> getDataTable(List<T> list, Integer total) {
-        TableDataInfo rspData = new TableDataInfo();
-        rspData.setCode(HttpStatus.OK.value());
-        rspData.setMsg("查询成功");
-        rspData.setRows(list);
-        rspData.setTotal(total);
-        return rspData;
+
+    protected <T> AjaxResult<T> successResult(R<T> r) {
+        if (r.getCode() != HttpStatus.OK.value()) {
+            throw new ServiceException(r.getMsg());
+        }
+        return AjaxResult.success(r.getData());
+    }
+
+    protected <T> TableDataInfo<T> successResult(TableDataInfo<T> tableDataInfo) {
+        if (tableDataInfo.getCode() != HttpStatus.OK.value()) {
+            throw new ServiceException(tableDataInfo.getMsg());
+        }
+        return tableDataInfo;
     }
 
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/enums/ContentShowType.java

@@ -0,0 +1,24 @@
+package com.jiuyv.sptccc.agile.common.enums;
+
+public enum ContentShowType {
+    BANNER("banner", "1"),
+    INFORMATION("资讯", "2"),
+    SCENES("应用场景", "3"),
+    DATA_PRODUCT("数据产品", "4");
+
+    private final String tag;
+    private final String value;
+
+    ContentShowType(String name, String value) {
+        this.tag = name;
+        this.value = value;
+    }
+
+    public String getTag() {
+        return tag;
+    }
+
+    public String getValue() {
+        return value;
+    }
+}

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/utils/SecurityUtils.java

@@ -47,17 +47,6 @@ public class SecurityUtils {
         }
     }
 
-    /**
-     * 获取用户姓名
-     */
-    public static String getNickname() {
-        try {
-            return getLoginUser().getUser().getNickName();
-        } catch (Exception e) {
-            throw new ServiceException("获取用户账户异常", HttpStatus.UNAUTHORIZED.value());
-        }
-    }
-
     /**
      * 获取用户密钥
      */
@@ -87,36 +76,4 @@ public class SecurityUtils {
         return SecurityContextHolder.getContext().getAuthentication();
     }
 
-    /**
-     * 生成BCryptPasswordEncoder密码
-     *
-     * @param password 密码
-     * @return 加密字符串
-     */
-    public static String encryptPassword(String password) {
-        BCryptPasswordEncoder passwordEncoder = new BCryptPasswordEncoder();
-        return passwordEncoder.encode(password);
-    }
-
-    /**
-     * 判断密码是否相同
-     *
-     * @param rawPassword     真实密码
-     * @param encodedPassword 加密后字符
-     * @return 结果
-     */
-    public static boolean matchesPassword(String rawPassword, String encodedPassword) {
-        BCryptPasswordEncoder passwordEncoder = new BCryptPasswordEncoder();
-        return passwordEncoder.matches(rawPassword, encodedPassword);
-    }
-
-    /**
-     * 是否为管理员
-     *
-     * @param userId 用户ID
-     * @return 结果
-     */
-    public static boolean isAdmin(Long userId) {
-        return userId != null && 1L == userId;
-    }
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/utils/ServletUtils.java

@@ -1,6 +1,7 @@
 package com.jiuyv.sptccc.agile.common.utils;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpSession;
 
 import org.springframework.web.context.request.RequestAttributes;
 import org.springframework.web.context.request.RequestContextHolder;
@@ -18,19 +19,23 @@ public class ServletUtils {
      */
     public static HttpServletRequest getRequest() {
         ServletRequestAttributes requestAttributes = getRequestAttributes();
-        if (null != requestAttributes) {
+        assert requestAttributes != null;
         return requestAttributes.getRequest();
     }
-        return null;
-    }
 
 
     public static ServletRequestAttributes getRequestAttributes() {
         RequestAttributes attributes = RequestContextHolder.getRequestAttributes();
-        if (null != attributes) {
         return (ServletRequestAttributes) attributes;
     }
-        return null;
+
+
+    /**
+     * 获取session
+     */
+    public static HttpSession getSession() {
+        HttpServletRequest request = getRequest();
+        return request.getSession();
     }
 
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/common/utils/StringUtil.java

@@ -1,8 +1,12 @@
 package com.jiuyv.sptccc.agile.common.utils;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Random;
 
+import com.jiuyv.sptccc.agile.common.exception.ServiceException;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.AntPathMatcher;
 
@@ -106,4 +110,17 @@ public class StringUtil {
         }
         return bld.toString();
     }
+
+    /**
+     * 对文件名进行URL编码
+     *
+     */
+    public static String encoderURL(String str) {
+        try {
+            return URLEncoder.encode(str, StandardCharsets.UTF_8.toString()).replace("+", "%20");
+        } catch (UnsupportedEncodingException e) {
+            e.printStackTrace();
+            throw new ServiceException("编码异常");
+        }
+    }
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/framework/config/ApplicationConfig.java

@@ -2,7 +2,6 @@ package com.jiuyv.sptccc.agile.framework.config;
 
 import java.util.TimeZone;
 
-import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -19,7 +18,6 @@ import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 // 表示通过aop框架暴露该代理对象,AopContext能够访问
 @EnableAspectJAutoProxy(exposeProxy = true)
 // 指定要扫描的Mapper类的包的路径
-@MapperScan("com.jiuyv.sptccc.agile.**.mapper")
 public class ApplicationConfig {
     /**
      * 时区配置,long类型解决

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/framework/config/SecurityConfig.java

@@ -4,7 +4,6 @@ import com.jiuyv.sptccc.agile.common.core.domain.AjaxResult;
 import com.jiuyv.sptccc.agile.framework.security.filter.LoginFilter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
-import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
 import org.springframework.security.authentication.AuthenticationManager;
 import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
@@ -12,9 +11,6 @@ import org.springframework.security.config.annotation.method.configuration.Enabl
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.builders.WebSecurity;
 import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
-import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.AuthenticationException;
 import org.springframework.security.core.userdetails.UserDetailsService;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 
@@ -22,11 +18,6 @@ import com.jiuyv.sptccc.agile.framework.config.properties.PermitAllUrlProperties
 import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
 import org.springframework.security.web.csrf.CookieCsrfTokenRepository;
 
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * spring security配置
  *
@@ -66,6 +57,8 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
 
     @Override
     protected void configure(HttpSecurity httpSecurity) throws Exception {
+        CookieCsrfTokenRepository csrfTokenRepository = CookieCsrfTokenRepository.withHttpOnlyFalse();
+        csrfTokenRepository.setCookiePath("/");
         httpSecurity
                 .addFilterBefore(loginFilter, UsernamePasswordAuthenticationFilter.class)
                 .exceptionHandling()
@@ -76,8 +69,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
                 .logoutSuccessHandler((req, resp, auth) -> AjaxResult.success(resp, "退出成功"))
                 .and()
                 .csrf()
-                .disable()
-//                .csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse())
+                .csrfTokenRepository(csrfTokenRepository)
         ;
 
         httpSecurity.authorizeRequests()
@@ -93,7 +85,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
     @Override
     public void configure(WebSecurity web) throws Exception {
         //强制跳过所有静态资源，这样权限不管静态资源，不然没法正确提示
-        web.ignoring().antMatchers("/static/**", "/favicon.**");
+        web.ignoring().antMatchers("/", "/*.html", "/static/**", "/favicon.**");
     }
 
     /**

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/framework/web/service/SysLoginService.java

@@ -12,11 +12,14 @@ import com.jiuyv.sptccc.agile.common.core.domain.R;
 import com.jiuyv.sptccc.agile.common.core.domain.model.LoginBody;
 import com.jiuyv.sptccc.agile.common.core.domain.model.LoginUser;
 import com.jiuyv.sptccc.agile.common.exception.ServiceException;
+import com.jiuyv.sptccc.agile.common.utils.SecurityUtils;
 import com.jiuyv.sptccc.agile.common.utils.StringUtil;
 import com.jiuyv.sptccc.agile.dto.PortalUserDTO;
+import com.jiuyv.sptccc.agile.dto.ResUserPasswordDTO;
 import com.jiuyv.sptccc.agile.feign.portal.PortalUserFeign;
 import com.jiuyv.sptccc.agile.feign.portal.PublicPhoneMsgLogFeign;
 import com.jiuyv.sptccc.agile.portal.domain.TblPortalUser;
+import com.jiuyv.sptccc.agile.portal.dto.RePasswordDTO;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +52,8 @@ public class SysLoginService {
 
     private final UserDetailsService userDetailsService;
 
+    private final SysSecretService secretService;
+
     private final PasswordEncoder passwordEncoder;
 
     private final CaptchaService captchaService;
@@ -57,12 +62,14 @@ public class SysLoginService {
 
     private final ConsoleConfig consoleConfig;
 
-    public SysLoginService(LocalCache localCache, PortalUserFeign userService, UserDetailsService userDetailsService,
+    public SysLoginService(LocalCache localCache, PortalUserFeign userService,
+                           UserDetailsService userDetailsService, SysSecretService secretService,
                            @Lazy PasswordEncoder passwordEncoder, CaptchaService captchaService,
                            PublicPhoneMsgLogFeign phoneMsgLogService, ConsoleConfig consoleConfig) {
         this.localCache = localCache;
         this.userService = userService;
         this.userDetailsService = userDetailsService;
+        this.secretService = secretService;
         this.passwordEncoder = passwordEncoder;
         this.captchaService = captchaService;
         this.phoneMsgLogService = phoneMsgLogService;
@@ -72,7 +79,7 @@ public class SysLoginService {
     /**
      * 登录 校验用户
      */
-    public String verifyUser(LoginBody loginBody, HttpSession session) {
+    public String verifyUser(LoginBody loginBody) {
         CaptchaVO captchaVO = new CaptchaVO();
         captchaVO.setCaptchaVerification(loginBody.getCaptchaVerification());
         ResponseModel response = captchaService.verification(captchaVO);
@@ -86,30 +93,46 @@ public class SysLoginService {
         if (StringUtils.isBlank(phone)) {
             throw new ServiceException("该用户没有绑定手机号");
         }
-        if (!passwordEncoder.matches(loginBody.getPassword(), user.getPassword())) {
+        if (!passwordEncoder.matches(secretService.decodePassword(loginBody.getPassword()), user.getPassword())) {
             // 更新用户锁定状态
             userLocked(user.getUser());
             throw new ServiceException("用户名或密码错误");
         }
-        // 清除密码
-        user.getUser().setPassword(null);
         // 记录登录信息
         recordLoginInfo(user.getUserId());
-        session.setAttribute(Constants.LOGIN_USER_INFO, user);
+        ServletUtils.getSession().setAttribute(Constants.LOGIN_USER_INFO, user);
         return StringUtil.strHide(phone);
     }
 
-
     /**
-     * 发送手机验证码
+     * 发送登录验证码
      *
      */
-    public String sendPhoneCode(HttpSession session) {
-        LoginUser user = (LoginUser) session.getAttribute(Constants.LOGIN_USER_INFO);
+    public String loginVerifyCode() {
+        LoginUser user = (LoginUser) ServletUtils.getSession().getAttribute(Constants.LOGIN_USER_INFO);
         if (user == null) {
             throw new ServiceException("非法操作，用户未验证");
         }
-        String phone = user.getUser().getPhonenumber();
+        return sendPhoneCode(user.getUser().getPhonenumber(), Constants.LOGIN_VERIFY_CODE_TEMPLATE);
+    }
+
+    /**
+     * 找回密码验证码
+     *
+     */
+    public String rePasswordVerifyCode() {
+        RePasswordDTO rePasswordDTO = (RePasswordDTO) ServletUtils.getSession().getAttribute(Constants.RE_PASSWORD_USER_INFO);
+        if (rePasswordDTO == null) {
+            throw new ServiceException("非法操作");
+        }
+        return sendPhoneCode(rePasswordDTO.getPhoneNumber(), Constants.RE_PASSWORD_VERIFY_CODE_TEMPLATE);
+    }
+
+
+    /**
+     * 发送手机验证码
+     */
+    private String sendPhoneCode(String phone, String msgTemplate) {
         String captcha = localCache.getValueOfCacheName(CacheNames.CACHE_1MIN, phone, String.class);
         if (StringUtils.isNotBlank(captcha)) {
             throw new ServiceException("请勿重复提交，请稍后再试");
@@ -122,7 +145,7 @@ public class SysLoginService {
         }
         ReqPublicPhoneMsgSendDTO msgLog = new ReqPublicPhoneMsgSendDTO();
         msgLog.setPhoneNumber(phone);
-        msgLog.setMsgTemplateCode("portal_login");//用模板
+        msgLog.setMsgTemplateCode(msgTemplate);//用模板
         Map<String, String> msgParams = msgLog.getMsgMapParams();
         msgParams.put("code", captcha);
         R<Long> r = phoneMsgLogService.sendPhoneMsg(msgLog);
@@ -193,4 +216,83 @@ public class SysLoginService {
         throw new ServiceException(msg);
     }
 
+    /**
+     * 找回密码，根据用户名查询手机号
+     *
+     */
+    public String getPhoneByUser(String username) {
+        LoginUser user = (LoginUser) userDetailsService.loadUserByUsername(username);
+        String phone = user.getUser().getPhonenumber();
+        if (StringUtils.isBlank(phone)) {
+            throw new ServiceException("该用户没有绑定手机号");
+        }
+        RePasswordDTO rePasswordDTO = new RePasswordDTO();
+        rePasswordDTO.setPhoneNumber(phone);
+        rePasswordDTO.setUserId(user.getUserId());
+        ServletUtils.getSession().setAttribute(Constants.RE_PASSWORD_USER_INFO, rePasswordDTO);
+        return StringUtil.strHide(phone);
+    }
+
+    /**
+     * 找回密码 - 验证手机验证码
+     *
+     */
+    public void verifyPhoneCode(String phoneCode) {
+        RePasswordDTO rePasswordDTO = (RePasswordDTO) ServletUtils.getSession()
+                .getAttribute(Constants.RE_PASSWORD_USER_INFO);
+
+        if (rePasswordDTO == null) {
+            throw new ServiceException("非法操作");
+        }
+        String phone = rePasswordDTO.getPhoneNumber();
+        String captcha = localCache.getValueOfCacheName(CacheNames.CACHE_1MIN, phone, String.class);
+        if (!phoneCode.equals(captcha)) {
+            throw new ServiceException("验证码错误");
+        }
+        localCache.removeValueOfCacheName(CacheNames.CACHE_1MIN, phone);
+        rePasswordDTO.setCheckPassed(true);
+    }
+
+    /**
+     * 找回密码 - 密码重置
+     *
+     */
+    public void resetPassword(ResUserPasswordDTO userPasswordDTO) {
+        HttpSession session = ServletUtils.getSession();
+        RePasswordDTO rePasswordDTO = (RePasswordDTO) session.getAttribute(Constants.RE_PASSWORD_USER_INFO);
+        if (rePasswordDTO == null || !rePasswordDTO.isCheckPassed()) {
+            throw new ServiceException("非法操作，手机未验证");
+        }
+        userPasswordDTO.setUserId(rePasswordDTO.getUserId());
+        String password = secretService.decodePassword(userPasswordDTO.getPassword());
+        userPasswordDTO.setPassword(passwordEncoder.encode(password));
+        R<Void> r = userService.resetUserPwd(userPasswordDTO);
+        if (r.getCode() != HttpStatus.OK.value()) {
+            logger.error("密码重置失败:{}", r.getMsg());
+            throw new ServiceException("密码重置失败。" + r.getMsg());
+        }
+        session.removeAttribute(Constants.RE_PASSWORD_USER_INFO);
+        session.invalidate();
+    }
+
+    /**
+     * 用户中心修改密码
+     *
+     */
+    public void changePassword(ResUserPasswordDTO userPasswordDTO) {
+        String oldPassword = secretService.decodePassword(userPasswordDTO.getOldPassword());
+        TblPortalUser user = SecurityUtils.getLoginUser().getUser();
+        if (!passwordEncoder.matches(oldPassword, user.getPassword())) {
+            throw new ServiceException("原密码错误");
+        }
+        String encodePassword = passwordEncoder.encode(secretService.decodePassword(userPasswordDTO.getPassword()));
+        userPasswordDTO.setPassword(encodePassword);
+        userPasswordDTO.setUserId(user.getUserId());
+        R<Void> r = userService.resetUserPwd(userPasswordDTO);
+        if (r.getCode() != HttpStatus.OK.value()) {
+            logger.error("修改密码失败:{}", r.getMsg());
+            throw new ServiceException("修改密码失败。" + r.getMsg());
+        }
+        ServletUtils.getSession().invalidate();
+    }
 }

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/framework/web/service/SysSecretService.java

@@ -0,0 +1,62 @@
+package com.jiuyv.sptccc.agile.framework.web.service;
+
+import com.jiuyv.sptccc.agile.common.config.ConsoleConfig;
+import com.jiuyv.sptccc.agile.common.exception.ServiceException;
+import org.apache.commons.codec.binary.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import javax.crypto.Cipher;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+
+/**
+ * 处理密钥服务
+ */
+@Service
+public class SysSecretService {
+    private static final Logger LOG = LoggerFactory.getLogger(SysSecretService.class);
+    private static final int KEY_SIZE = 1024;
+    // 密码超时时间，512秒
+    private static final long TIME_OUT = 1000 << 9;
+    private final KeyPair keyPair;
+    private final Cipher cipher;
+
+    public SysSecretService(ConsoleConfig consoleConfig) throws GeneralSecurityException {
+        // 初始化生成密钥对
+        KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
+        keyPairGenerator.initialize(KEY_SIZE);
+        this.keyPair = keyPairGenerator.generateKeyPair();
+        this.cipher = Cipher.getInstance(consoleConfig.getPwdEncAlg());
+        this.cipher.init(Cipher.DECRYPT_MODE, keyPair.getPrivate());
+    }
+
+    /**
+     * 获取公钥
+     */
+    public String getPublicKey() {
+        return Base64.encodeBase64String(keyPair.getPublic().getEncoded());
+    }
+
+    /**
+     * 对前端密码解密
+     */
+    public String decodePassword(String password) {
+        try {
+            byte[] bytes = cipher.doFinal(Base64.decodeBase64(password));
+            String[] ps = new String(bytes, StandardCharsets.UTF_8).split(",");
+            if (System.currentTimeMillis() - Long.parseLong(ps[1]) > TIME_OUT) {
+                LOG.info(">>>>>密码过期>>>>>");
+                throw new ServiceException("密码超时");
+            }
+            return ps[0];
+        } catch (GeneralSecurityException e) {
+            LOG.info(">>>>>密码无效>>>>>", e);
+            throw new ServiceException("密码无效");
+        }
+    }
+
+}

agile-portal/agile-portal-gateway/src/main/java/com/jiuyv/sptccc/agile/framework/web/service/UserDetailsServiceImpl.java

@@ -41,15 +41,11 @@ public class UserDetailsServiceImpl implements UserDetailsService {
         PortalUserDTO userDTO = userRes.getData();
         if (userDTO == null) {
             log.info("登录用户：{} 不存在.", username);
-            throw new ServiceException("账户或密码错误");
+            throw new ServiceException("用户不存在");
         }
         if (UserStatus.DELETED.getCode().equals(userDTO.getDelFlag())) {
             log.info("登录用户：{} 已被删除.", username);
-            throw new ServiceException("账户或密码错误");
-        }
-        if (UserStatus.DISABLE.getCode().equals(userDTO.getStatus())) {
-            log.info("登录用户：{} 已被停用.", username);
-            throw new ServiceException("账户或密码错误");
+            throw new ServiceException("用户不存在");
         }
         TblPortalUser user = new TblPortalUser();
         BeanUtils.copyProperties(userDTO, user);