master
RENCHAO 3 months ago
parent 4932096ba3
commit 046cfe6f7c

@ -3,7 +3,7 @@
<parent>
<groupId>com.jiuyv.sptcc</groupId>
<artifactId>carbon-dataprocess</artifactId>
<version>0.4.4-SNAPSHOT</version>
<version>0.5.1-SNAPSHOT</version>
</parent>
<artifactId>carbon-dataprocess-api</artifactId>

@ -4,7 +4,7 @@
<parent>
<groupId>com.jiuyv.sptcc</groupId>
<artifactId>carbon-dataprocess</artifactId>
<version>0.4.4-SNAPSHOT</version>
<version>0.5.1-SNAPSHOT</version>
</parent>
<artifactId>carbon-dataprocess-service</artifactId>

@ -76,7 +76,7 @@ public class DataManageController implements DataManageApi {
}
Integer offset = reqPage.getOffset();
Integer limit = reqPage.getLimit();
PageMethod.startPage(offset / limit + 1, limit, orderBy).setReasonable(true);
PageMethod.startPage(offset / limit + 1, limit, orderBy).setReasonable(false);
}

@ -13,7 +13,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.support.DefaultSingletonBeanRegistry;
import org.springframework.boot.SpringApplicationRunListener;
import org.springframework.context.ApplicationContext;
import org.springframework.core.metrics.StartupStep;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -22,8 +24,12 @@ import java.lang.reflect.Field;
import java.text.DecimalFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
/**
* @author ren_chao
@ -109,30 +115,10 @@ public class TestController {
public static void main(String[] args) {
BcCalcFactor calcFactor = new BcCalcFactor();
calcFactor.setEnd(LocalDateTime.parse("20240801000000", REQ_SDF));
calcFactor.setStart(LocalDateTime.parse("20240820000000", REQ_SDF));
long l = System.currentTimeMillis();
for (int i = 0; i < 20000; i++) {
getCalcFactor("20240809000000", calcFactor);
}
System.out.println(":::" + (System.currentTimeMillis() - l));
}
private static final DateTimeFormatter REQ_SDF = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
private static BcCalcFactor getCalcFactor(String intoTransTime, BcCalcFactor calcFactor) {
LocalDateTime time = LocalDateTime.parse(intoTransTime, REQ_SDF);
if (calcFactor == null || time.isAfter(calcFactor.getEnd()) || time.isBefore(calcFactor.getStart())) {
if (calcFactor == null) {
LOGGER.error("未查询到排放因子数据:{}", intoTransTime);
throw new ServiceException("未查询到排放因子数据");
}
}
return calcFactor;
Map<String, String> map = new LinkedHashMap<>();
map.put("a", "a");
map.put("a", "b");
System.out.println(map);
}

@ -8,6 +8,7 @@ import org.springframework.cloud.openfeign.FeignClient;
*
* @author ren_chao
*/
@FeignClient(value = "${carbon-data-process.auth-serve}", contextId = "carbonAuthFeign")
// @FeignClient(value = "${carbon-data-process.auth-serve}", contextId = "carbonAuthFeign")
@FeignClient(url = "http://localhost:12080", name = "carbonAuthFeign")
public interface CarbonAuthFeign extends IAccountApi {
}

@ -8,6 +8,7 @@ import org.springframework.cloud.openfeign.FeignClient;
*
* @author ren_chao
*/
@FeignClient(value = "${carbon-data-process.gateway-serve}", contextId = "carbonDataProcessFeign")
// @FeignClient(value = "${carbon-data-process.gateway-serve}", contextId = "carbonDataProcessFeign")
@FeignClient(url = "http://localhost:15978", name = "carbonDataProcessFeign")
public interface CarbonDataProcessFeign extends CarbonReductionApi {
}

@ -81,7 +81,7 @@ public class CarbonDataTask {
throw e;
} catch (Exception e) {
LOGGER.error("未知异常", e);
smsSend("中台数据读取异常:" + e.getMessage());
smsSend("中台数据读取异常:未知异常,请查看日志");
throw e;
}
}

@ -48,6 +48,7 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -128,8 +129,6 @@ public class ReadFileServiceImpl implements IReadFileService {
File file = getFile(dateStr, currentTime);
String csvName = file.getName();
long startTime = System.currentTimeMillis();
// 是否第一次读取
boolean notFirst = bcTravelNoticeMapper.selectCountByFileName(csvName) != 0;
try (FileInputStream fis = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8);
@ -140,11 +139,9 @@ public class ReadFileServiceImpl implements IReadFileService {
if (CollectionUtils.isEmpty(list)) {
break;
}
if (notFirst) {
list = filterDuplicates(list);
if (CollectionUtils.isEmpty(list)) {
continue;
}
list = filterDuplicates(list);
if (CollectionUtils.isEmpty(list)) {
continue;
}
// 授权校验及其他信息配置
authVerify(list, currentTime);
@ -165,14 +162,6 @@ public class ReadFileServiceImpl implements IReadFileService {
}
}
/**
*
*/
private List<BcTravelNotice> filterDuplicates(List<BcTravelNotice> list) {
Set<String> TravelNoList = bcTravelNoticeMapper.selectTravelNoListByTravelNoList(list);
return list.stream().filter(tn -> !TravelNoList.contains(tn.getTravelNo())).collect(Collectors.toList());
}
/**
*
*/
@ -185,17 +174,20 @@ public class ReadFileServiceImpl implements IReadFileService {
GResultDTO<ResDictionaryDTO> resultDTO;
Map<String, Double> data;
do {
while (true) {
resultDTO = carbonDataProcessFeign.getDictionary(reqVo);
if (!ResultCode.SUCCESS.getCode().equals(resultDTO.getRespCode())) {
LOGGER.error("获取里程数据异常");
throw new ServiceException("获取里程数据异常");
}
data = resultDTO.getContent().getData();
if (CollectionUtils.isEmpty(data)) {
break;
}
List<BcSubwayMileage> list = buildSubwayMileageList(data);
subwayMileageMapper.updateMileage(list);
reqVo.setPage(reqVo.getPage() + 1);
} while (!CollectionUtils.isEmpty(data));
}
}
@ -249,6 +241,17 @@ public class ReadFileServiceImpl implements IReadFileService {
}
/**
*
*/
private List<BcTravelNotice> filterDuplicates(List<BcTravelNotice> list) {
Set<String> travelNoList = bcTravelNoticeMapper.selectTravelNoListByTravelNoList(list);
if (CollectionUtils.isEmpty(travelNoList)) {
return list;
}
return list.stream().filter(tn -> !travelNoList.contains(tn.getTravelNo())).collect(Collectors.toList());
}
private BcCalcFactor getCalcFactor(String intoTransTime, String sceneCode) {
BcCalcFactor calcFactor = calcFactorCache.get(sceneCode);
@ -268,11 +271,13 @@ public class ReadFileServiceImpl implements IReadFileService {
private List<BcTravelNotice> getBcTravelNoticeList(BufferedReader br, String fileName) throws IOException {
List<BcTravelNotice> list = new ArrayList<>();
Set<String> set = new HashSet<>();
String line;
while ((line = br.readLine()) != null && list.size() < BATCH_SIZE) {
LOGGER.debug("获取记录:{}", line);
BcTravelNotice travelNotice = getBcTravelNotice(line, fileName);
if (travelNotice != null) {
if (travelNotice != null && !set.contains(travelNotice.getTravelNo())) {
set.add(travelNotice.getTravelNo());
checkAbnormalData(travelNotice);
list.add(travelNotice);
}
@ -417,7 +422,7 @@ public class ReadFileServiceImpl implements IReadFileService {
}
/**
* false
*
*
*/
private void calculateMileage(BcTravelNotice travelNotice) {

@ -1,10 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!-- LOG目录 -->
<property name="LOG_HOME" value="logs/log" />
<!-- JSON_LOG目录 -->
<property name="JSON_LOG_HOME" value="logs/jsonlog" />
<!-- spring.application.name 作为参数 -->
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<target>System.out</target>
@ -13,84 +8,9 @@
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- 按天压缩日志 -->
<appender name="ZIP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_HOME}/${APP_NAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--日志文件输出的文件名-->
<FileNamePattern>${LOG_HOME}/${APP_NAME}.log.%d{yyyy-MM-dd}.%i.log.zip</FileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- 不压缩日志,保留7天 -->
<appender name="DEFAULT_INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--日志文件输出的文件名-->
<FileNamePattern>${LOG_HOME}/${APP_NAME}.log.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<maxHistory>7</maxHistory>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- LOGSTASH,输出seuleth项,保留7天 -->
<!-- <appender name="LOGSTASH" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
日志文件输出的文件名
<FileNamePattern>${JSON_LOG_HOME}/${APP_NAME}.json.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<maxHistory>7</maxHistory>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
or whenever the file size reaches 100MB
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<pattern>
<pattern>
{
"timestamp": "%d{yyyy-MM-dd HH:mm:ss.SSS}",
"severity": "%level",
"service": "${APP_NAME:-}",
"trace": "%X{X-B3-TraceId:-}",
"span": "%X{X-B3-SpanId:-}",
"parent": "%X{X-B3-ParentSpanId:-}",
"exportable": "%X{X-Span-Export:-}",
"pid": "${PID:-}",
"thread": "%thread",
"class": "%logger{40}",
"rest": "%message"
}
</pattern>
</pattern>
</providers>
</encoder>
</appender> -->
<appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
<target>System.err</target>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root>
<level value="INFO" />
<appender-ref ref="STDOUT" />
<appender-ref ref="ZIP_FILE" />
<appender-ref ref="DEFAULT_INFO_FILE" />
<!-- <appender-ref ref="LOGSTASH" /> -->
</root>
</configuration>

@ -12,18 +12,12 @@
<result property="outStationName" column="OUT_STATION_NAME" jdbcType="VARCHAR"/>
</resultMap>
<sql id="Base_Column_List">
IN_STATION_ID,OUT_STATION_ID,MILEAGE,
IN_STATION_NAME,OUT_STATION_NAME
</sql>
<select id="selectSubwayMileagePage" resultMap="BaseResultMap" useCache="false">
select
<include refid="Base_Column_List" />
select IN_STATION_ID,OUT_STATION_ID,MILEAGE,IN_STATION_NAME,OUT_STATION_NAME
from TBL_BC_SUBWAY_MILEAGE
<where>
<if test="inStationId != null and inStationId != ''">
and IN_STATION_ID = #{inStationId,jdbcType=VARCHAR}
IN_STATION_ID = #{inStationId,jdbcType=VARCHAR}
</if>
<if test="outStationId != null and outStationId != ''">
and OUT_STATION_ID = #{outStationId,jdbcType=VARCHAR}
@ -34,14 +28,13 @@
<update id="updateMileage">
MERGE INTO TBL_BC_SUBWAY_MILEAGE target
USING (
SELECT '' AS IN_STATION_ID, '' AS OUT_STATION_ID, '' AS MILEAGE, '' AS IN_STATION_NAME, '' AS OUT_STATION_NAME FROM dual WHERE 1=0
<foreach collection="list" item="m">
UNION ALL
SELECT #{m.inStationId}, #{m.outStationId}, #{m.mileage},
<if test="m.inStationName != null"> #{m.inStationName} </if>
<if test="m.inStationName == null"> NULL</if>,
<if test="m.outStationName != null"> #{m.outStationName}</if>
<if test="m.outStationName == null"> NULL</if>
<foreach collection="list" item="m" separator="UNION ALL">
SELECT
#{m.inStationId,jdbcType=VARCHAR} AS IN_STATION_ID,
#{m.outStationId,jdbcType=VARCHAR} AS OUT_STATION_ID,
#{m.mileage,jdbcType=DECIMAL} AS MILEAGE,
#{m.inStationName,jdbcType=VARCHAR} AS IN_STATION_NAME,
#{m.outStationName,jdbcType=VARCHAR} AS OUT_STATION_NAME
FROM dual
</foreach>
) source

@ -310,9 +310,8 @@
<select id="selectTravelNoListByTravelNoList" resultType="java.lang.String">
WITH travel_list AS (
SELECT '' AS TRAVEL_NO FROM dual WHERE 1=0
<foreach collection="list" item="tn">
UNION ALL SELECT #{tn.travelNo} FROM dual
<foreach collection="list" item="tn" separator="UNION ALL">
SELECT #{tn.travelNo} AS TRAVEL_NO FROM dual
</foreach>
)
SELECT tn.TRAVEL_NO
@ -438,11 +437,13 @@
<update id="updateBatchBySeqNo">
MERGE INTO TBL_BC_TRAVEL_NOTICE target
USING (
SELECT '' AS SEQ_NO, '' AS CARBON_FOOTPRINT, '' AS REDUCTION_CALCULATE_TIME, '' AS STATUS, '' AS CHAIN_STATUS
FROM dual WHERE 1=0
<foreach collection="list" item="tn">
UNION ALL
SELECT #{tn.seqNo}, #{tn.carbonFootprint}, #{tn.reductionCalculateTime}, #{tn.status}, #{tn.chainStatus}
<foreach collection="list" item="tn" separator="UNION ALL">
SELECT
#{tn.seqNo} AS SEQ_NO,
#{tn.carbonFootprint} AS CARBON_FOOTPRINT,
#{tn.reductionCalculateTime} AS REDUCTION_CALCULATE_TIME,
#{tn.status} AS STATUS,
#{tn.chainStatus} AS CHAIN_STATUS
FROM dual
</foreach>
) source

@ -3,7 +3,7 @@
<groupId>com.jiuyv.sptcc</groupId>
<artifactId>carbon-dataprocess</artifactId>
<version>0.4.4-SNAPSHOT</version>
<version>0.5.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>carbon-dataprocess</name>
@ -27,7 +27,7 @@
<spring.cloud.version>2021.0.2</spring.cloud.version>
<logback.version>1.2.11</logback.version>
<spring-boot.version>2.6.7</spring-boot.version>
<carbon-pushgate-api.version>0.10.1-SNAPSHOT</carbon-pushgate-api.version>
<carbon-pushgate-api.version>0.11.0</carbon-pushgate-api.version>
<carbon-auth-api.version>0.8.0</carbon-auth-api.version>
</properties>

@ -3,7 +3,7 @@
<parent>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-datacenter</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.3-SNAPSHOT</version>
</parent>
<artifactId>agile-datacenter-api</artifactId>
<packaging>jar</packaging>

@ -1,6 +1,6 @@
package com.jiuyv.sptccc.agile.api.fegin.dto.linepassenger;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.NotBlank;
/**
@ -15,7 +15,7 @@ public class ReqSjztLinePassengerNonrtimeQueryDTO implements java.io.Serializabl
/**
* |yyyy-MM-dd
*/
@NotNull(message="日期不能为空")
@NotBlank(message="日期不能为空")
private String txnDate;
/**

@ -1,6 +1,6 @@
package com.jiuyv.sptccc.agile.api.fegin.dto.linepassenger;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.NotBlank;
/**
@ -20,8 +20,8 @@ public class ReqSjztLinePassengerRealtimeQueryDTO implements java.io.Serializabl
/**
* |Y
*/
@NotNull(message="时间段不能为空")
private Integer timeCode;
@NotBlank(message="时间段不能为空")
private String timeCode;
@ -41,13 +41,13 @@ public class ReqSjztLinePassengerRealtimeQueryDTO implements java.io.Serializabl
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -1,6 +1,6 @@
package com.jiuyv.sptccc.agile.api.fegin.dto.linepassenger;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.NotBlank;
/**
@ -20,8 +20,8 @@ public class ReqSjztLinePassengerTotalQueryDTO implements java.io.Serializable {
/**
* |Y
*/
@NotNull(message="时间段不能为空")
private Integer timeCode;
@NotBlank(message="时间段不能为空")
private String timeCode;
@ -41,13 +41,13 @@ public class ReqSjztLinePassengerTotalQueryDTO implements java.io.Serializable {
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -22,7 +22,7 @@ public class ResSjztLinePassengerNonrtimeQueryDTO implements java.io.Serializabl
/**
*
*/
private Integer timeCode;
private String timeCode;
/**
* /线
@ -105,13 +105,13 @@ public class ResSjztLinePassengerNonrtimeQueryDTO implements java.io.Serializabl
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}
@ -247,30 +247,30 @@ public class ResSjztLinePassengerNonrtimeQueryDTO implements java.io.Serializabl
//拼接为文件的格式,方便调整
// public String toFileString(String splitStr) {
// return txnDate + splitStr + settleCenterCode
// + splitStr + timeCode + splitStr + inLineCode + splitStr + totalCnt
// + splitStr + outTxnCnt + splitStr + inStationCnt + splitStr + jtkInStationCnt
// + splitStr + jtkBusTransferCnt + splitStr + jtkMetroTransferCnt
// + splitStr + ewmInStationCnt + splitStr + qrBusTransferCnt
// + splitStr + qrMetroTransferCnt;
// }
// public String toFileTitle(String splitStr) {
// return "txnDate" + splitStr + "settleCenterCode"
// + splitStr + "timeCode" + splitStr + "inLineCode" + splitStr + "totalCnt"
// + splitStr + "outTxnCnt" + splitStr + "inStationCnt" + splitStr + "jtkInStationCnt"
// + splitStr + "jtkBusTransferCnt" + splitStr + "jtkMetroTransferCnt"
// + splitStr + "ewmInStationCnt" + splitStr + "qrBusTransferCnt"
// + splitStr + "qrMetroTransferCnt";
// }
public String toFileString(String splitStr) {
return settleCenterCode + splitStr + timeCode + splitStr + inLineCode + splitStr + txnDate
+ splitStr + totalCnt + splitStr + outTxnCnt + splitStr + inStationCnt
+ splitStr + jtkInStationCnt + splitStr + qrBusTransferCnt+"\n";
return txnDate + splitStr + settleCenterCode
+ splitStr + timeCode + splitStr + inLineCode + splitStr + totalCnt
+ splitStr + outTxnCnt + splitStr + inStationCnt + splitStr + jtkInStationCnt
+ splitStr + jtkBusTransferCnt + splitStr + jtkMetroTransferCnt
+ splitStr + ewmInStationCnt + splitStr + qrBusTransferCnt
+ splitStr + qrMetroTransferCnt+"\n";
}
public String toFileTitle(String splitStr) {
return "settleCenterCode" + splitStr + "timeCode" + splitStr + "inLineCode" + splitStr +"txnDate"
+ splitStr + "totalCnt" + splitStr + "outTxnCnt" + splitStr + "inStationCnt"
+ splitStr + "jtkInStationCnt" + splitStr + "qrBusTransferCnt"+"\n";
return "txnDate" + splitStr + "settleCenterCode"
+ splitStr + "timeCode" + splitStr + "inLineCode" + splitStr + "totalCnt"
+ splitStr + "outTxnCnt" + splitStr + "inStationCnt" + splitStr + "jtkInStationCnt"
+ splitStr + "jtkBusTransferCnt" + splitStr + "jtkMetroTransferCnt"
+ splitStr + "ewmInStationCnt" + splitStr + "qrBusTransferCnt"
+ splitStr + "qrMetroTransferCnt"+"\n";
}
// public String toFileString(String splitStr) {
// return settleCenterCode + splitStr + timeCode + splitStr + inLineCode + splitStr + txnDate
// + splitStr + totalCnt + splitStr + outTxnCnt + splitStr + inStationCnt
// + splitStr + jtkInStationCnt + splitStr + qrBusTransferCnt+"\n";
// }
// public String toFileTitle(String splitStr) {
// return "settleCenterCode" + splitStr + "timeCode" + splitStr + "inLineCode" + splitStr +"txnDate"
// + splitStr + "totalCnt" + splitStr + "outTxnCnt" + splitStr + "inStationCnt"
// + splitStr + "jtkInStationCnt" + splitStr + "qrBusTransferCnt"+"\n";
// }
}

@ -22,7 +22,7 @@ public class ResSjztLinePassengerRealtimeQueryDTO implements java.io.Serializabl
/**
*
*/
private Integer timeCode;
private String timeCode;
/**
* /线
@ -85,13 +85,13 @@ public class ResSjztLinePassengerRealtimeQueryDTO implements java.io.Serializabl
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -22,7 +22,7 @@ public class ResSjztLinePassengerTotalQueryDTO implements java.io.Serializable {
/**
*
*/
private Integer timeCode;
private String timeCode;
/**
* /线
@ -105,13 +105,13 @@ public class ResSjztLinePassengerTotalQueryDTO implements java.io.Serializable {
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -3,7 +3,7 @@
<parent>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-datacenter</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.3-SNAPSHOT</version>
</parent>
<artifactId>agile-datacenter-service</artifactId>

@ -9,17 +9,12 @@ public class SyncDatacenterConstants
throw new IllegalStateException("Utility class");
}
/** 日期转换用 */
public static final String FORMAT_DATETIME ="yyyy-MM-dd HH:mm:ss";
public static final String FORMAT_DATEDAY ="yyyy-MM-dd";
public static final String FORMAT_DATETIME_SIMPLE ="yyyyMMddHHmmss";
public static final String FORMAT_DATEDAY_SIMPLE ="yyyyMMdd";
/** 文件名后缀 */
public static final String FILEEXT_CSV =".csv";
public static final String FILEEXT_ZIP =".zip";
/** 文件名 */
public static final String FILENAME_LINE_PASSENGER_NONRTIME ="line_passenger_nonrtime_";
public static final String FILENAME_LINE_PASSENGER_NONRTIME ="line_passenger_nonrtime";
public static final String FILENAME_LINE_PASSENGER_NONRTIME_W ="line_passenger_nonrtime_%s.zip";
}

@ -9,6 +9,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.time.DateFormatUtils;
@ -16,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -30,10 +32,16 @@ import com.jiuyv.sptccc.agile.business.common.constant.SyncDatacenterConstants;
import com.jiuyv.sptccc.agile.business.entity.TblSjztLinePassengerRealtime;
import com.jiuyv.sptccc.agile.business.entity.TblSjztLinePassengerTotal;
import com.jiuyv.sptccc.agile.business.service.ISjztLinePassengerService;
import com.jiuyv.sptccc.agile.business.service.impl.SftpFileServer;
import com.jiuyv.sptccc.agile.common.annotation.LogIgnore;
import com.jiuyv.sptccc.agile.common.constant.Constants;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.common.utils.ServletUtils;
import com.jiuyv.sptccc.agile.common.utils.sftp.SFTPChannel;
import com.jiuyv.sptccc.agile.common.utils.sftp.SshClientHostPool;
import com.jiuyv.sptccc.agile.common.utils.sftp.SshClientPool;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
import com.jiuyv.sptccc.agile.framework.config.SftpConfigProperties;
/**
@ -46,39 +54,54 @@ import com.jiuyv.sptccc.agile.common.utils.ServletUtils;
public class SjztLinePassengerController {
@Autowired
private ISjztLinePassengerService sjztLinePassengerService;
@Autowired
private SftpConfigProperties sftpConfigProperties;
@Autowired
private SftpFileServer sftpFileServer;
@LogIgnore
@PostMapping("/querySjztLinePassengerNonrtime")
public void querySjztLinePassengerNonrtime(HttpServletResponse response,@RequestBody ReqSjztLinePassengerNonrtimeQueryDTO reqInfo) throws Exception {
//先去sftp服务器检索是否已有文件
//@LogIgnore
@PostMapping("/querySjztLinePassengerNonrtime")
public void querySjztLinePassengerNonrtime(HttpServletResponse response,@RequestBody @Validated ReqSjztLinePassengerNonrtimeQueryDTO reqInfo) throws Exception {
String fileName = ServletUtils.urlEncode(String.format(SyncDatacenterConstants.FILENAME_LINE_PASSENGER_NONRTIME_W
,reqInfo.getTxnDate().replace("-", "")));
response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE);
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fileName);
String src = (sftpConfigProperties.getUploadPath()+"/"+SyncDatacenterConstants.FILENAME_LINE_PASSENGER_NONRTIME
+"/"+fileName).replace("//", "");
//先去sftp服务器检索是否已有文件
SshClientPool hostPool = SshClientHostPool.getHostPool(getSftpConf());//ssh连接池
SFTPChannel channel = new SFTPChannel();
if(sftpFileServer.downloadSjztLinePassengerNonrtimeOfSftp(hostPool, channel, response, src)) {
return;
}
//获取实时文件
Path path = sjztLinePassengerService.querySjztLinePassengerNonrtime(reqInfo);
File file = path.toFile();
if(file != null) {
OutputStream outputStream = response.getOutputStream();
try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
response.setContentLength(inputStream.available());
response.setHeader(HttpHeaders.CONTENT_LENGTH, inputStream.available()+"");
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
outputStream.flush();
} finally {
//上传到sftp并删除文件
file.deleteOnExit();
} catch (Exception e) {
response.sendError(HttpStatus.NOT_FOUND.value(), "File not found");
}
finally {
sftpFileServer.uploadSjztLinePassengerNonrtimeOfSftp(hostPool, channel, response, src, file);
}
}else {
response.sendError(HttpStatus.NOT_FOUND.value(), "File not found");
}
}
@PostMapping("/querySjztLinePassengerRealtime")
public R<List<ResSjztLinePassengerRealtimeQueryDTO>> querySjztLinePassengerRealtime(@RequestBody ReqSjztLinePassengerRealtimeQueryDTO reqInfo) throws Exception {
public R<List<ResSjztLinePassengerRealtimeQueryDTO>> querySjztLinePassengerRealtime(@RequestBody @Validated ReqSjztLinePassengerRealtimeQueryDTO reqInfo) throws Exception {
List<TblSjztLinePassengerRealtime> list = sjztLinePassengerService.querySjztLinePassengerNonrtime(reqInfo);
List<ResSjztLinePassengerRealtimeQueryDTO> data = new ArrayList<>();
for(TblSjztLinePassengerRealtime x:list) {
@ -102,7 +125,7 @@ public class SjztLinePassengerController {
}
@PostMapping("/querySjztLinePassengerTotal")
public R<List<ResSjztLinePassengerTotalQueryDTO>> querySjztLinePassengerTotal(@RequestBody ReqSjztLinePassengerTotalQueryDTO reqInfo)throws Exception {
public R<List<ResSjztLinePassengerTotalQueryDTO>> querySjztLinePassengerTotal(@RequestBody @Validated ReqSjztLinePassengerTotalQueryDTO reqInfo)throws Exception {
List<TblSjztLinePassengerTotal> list = sjztLinePassengerService.querySjztLinePassengerNonrtime(reqInfo);
List<ResSjztLinePassengerTotalQueryDTO> data = new ArrayList<>();
for(TblSjztLinePassengerTotal x:list) {
@ -124,4 +147,20 @@ public class SjztLinePassengerController {
}
return R.ok(data);
}
public SFTPConfig getSftpConf() {
SFTPConfig sftpDetails = new SFTPConfig();
sftpDetails.setUsername(sftpConfigProperties.getUsername());
sftpDetails.setPassword(sftpConfigProperties.getPassword());
sftpDetails.setPort(sftpConfigProperties.getPort());
sftpDetails.setHost(sftpConfigProperties.getHost());
sftpDetails.setPoolConfig(sftpConfigProperties.getPoolConfig());
return sftpDetails;
}
@PostConstruct
public void SshClient() {
SshClientHostPool.getHostPool(getSftpConf());//ssh连接池
}
}

@ -21,7 +21,7 @@ public class TblSjztLinePassengerNonrtime implements java.io.Serializable {
private String settleCenterCode;
/** 时间段 */
private Integer timeCode;
private String timeCode;
/** 地铁车站号/公交线路号 */
private String inLineCode;
@ -84,13 +84,13 @@ public class TblSjztLinePassengerNonrtime implements java.io.Serializable {
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -21,7 +21,7 @@ public class TblSjztLinePassengerRealtime implements java.io.Serializable {
private String settleCenterCode;
/** 时间段 */
private Integer timeCode;
private String timeCode;
/** 地铁车站号/公交线路号 */
private String inLineCode;
@ -84,13 +84,13 @@ public class TblSjztLinePassengerRealtime implements java.io.Serializable {
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -21,7 +21,7 @@ public class TblSjztLinePassengerTotal implements java.io.Serializable {
private String settleCenterCode;
/** 时间段 */
private Integer timeCode;
private String timeCode;
/** 地铁车站号/公交线路号 */
private String inLineCode;
@ -84,13 +84,13 @@ public class TblSjztLinePassengerTotal implements java.io.Serializable {
/**
* Get
*/
public Integer getTimeCode(){
public String getTimeCode(){
return timeCode;
}
/**
* Set
*/
public void setTimeCode(Integer timeCode){
public void setTimeCode(String timeCode){
this.timeCode = timeCode;
}

@ -0,0 +1,85 @@
package com.jiuyv.sptccc.agile.business.service.impl;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.SftpException;
import com.jiuyv.sptccc.agile.common.utils.sftp.SFTPChannel;
import com.jiuyv.sptccc.agile.common.utils.sftp.SshClientPool;
/**
* sftp
*/
@Component
public class SftpFileServer {
private static final Logger LOGGER = LoggerFactory.getLogger(SftpFileServer.class);
/** 下载文件 */
public boolean downloadSjztLinePassengerNonrtimeOfSftp(SshClientPool hostPool,SFTPChannel channel
,HttpServletResponse response,String src) {
try {
ChannelSftp chSftp = channel.getChannelNew(hostPool.getSession());
Long size = checkFileExistOfSftp(chSftp, src);
if(size==null) {
return false;
}
response.setHeader(HttpHeaders.CONTENT_LENGTH, size+"");
chSftp.get(src, response.getOutputStream());
chSftp.quit();
return true;
}catch(Exception e) {
LOGGER.info("从sftp下载失败>>src={}",src, e);
return false;
}finally {
channel.closeInnerChannel();
hostPool.returnSession(channel.getSession());//不再关闭,直接返还连接
}
}
private Long checkFileExistOfSftp(ChannelSftp chSftp,String src) {
try {
return chSftp.stat(src).getSize();
} catch (SftpException e) {
return null;//不存在忽略错误
}
}
/** 上传文件 */
@Async
public void uploadSjztLinePassengerNonrtimeOfSftp(SshClientPool hostPool,SFTPChannel channel
,HttpServletResponse response,String src,File file) {
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file))) {
ChannelSftp chSftp = channel.getChannelNew(hostPool.getSession());
autoMkdir(channel, src);//创建目录
chSftp.put(in, src);
chSftp.quit();
}catch(Exception e) {
LOGGER.info("上传到sftp失败>>src={}",src,e);
}finally {
channel.closeInnerChannel();
hostPool.returnSession(channel.getSession());//不再关闭,直接返还连接
file.deleteOnExit();
}
}
/**
*
* @param chSftp
* @param dst
* @throws Exception
*/
public static void autoMkdir(SFTPChannel channel,String dst) throws Exception {
//linux斜杠必须是 “/”
channel.commitCurrShellCmd("mkdir -p '"+new File(dst).getParent().replace("\\", "/")+"/'");
}
}

@ -36,9 +36,9 @@ import com.jiuyv.sptccc.agile.business.mapper.TblSjztLinePassengerNonrtimeMapper
import com.jiuyv.sptccc.agile.business.mapper.TblSjztLinePassengerRealtimeMapper;
import com.jiuyv.sptccc.agile.business.mapper.TblSjztLinePassengerTotalMapper;
import com.jiuyv.sptccc.agile.business.service.ISjztLinePassengerService;
import com.jiuyv.sptccc.agile.common.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.constant.Constants;
import com.jiuyv.sptccc.agile.common.core.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.core.domain.BaseTime;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import net.logstash.logback.encoder.org.apache.commons.lang3.StringUtils;

@ -1,4 +1,4 @@
package com.jiuyv.sptccc.agile.common.core;
package com.jiuyv.sptccc.agile.common;
import java.time.Instant;
import java.time.ZoneId;
@ -9,8 +9,8 @@ import java.util.Locale;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jiuyv.sptccc.agile.common.core.domain.BaseTime;
import com.jiuyv.sptccc.agile.common.utils.spring.SpringUtils;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import com.jiuyv.sptccc.agile.framework.SpringUtils;
import com.jiuyv.sptccc.agile.system.mapper.ISysTimeBaseMapper;
/**
* 使

@ -10,8 +10,10 @@ public class Constants {
/** 一个固定的密钥,用于配置文件内容加密/解密 */
public static final String SM4_SECERT_KEY = "a14751855ccb428d982c33dfa3535a57";
/** 日期格式 */
public static final String FORMAT_DATEDAY = "yyyy-MM-dd";
/** 日期转换用 */
public static final String FORMAT_DATETIME ="yyyy-MM-dd HH:mm:ss";
public static final String FORMAT_DATEDAY ="yyyy-MM-dd";
public static final String FORMAT_DATETIME_SIMPLE ="yyyyMMddHHmmss";
public static final String FORMAT_DATEDAY_SIMPLE ="yyyyMMdd";
/** 默认时区 */

@ -5,7 +5,7 @@ import java.util.Locale;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import com.jiuyv.sptccc.agile.common.utils.spring.SpringUtils;
import com.jiuyv.sptccc.agile.framework.SpringUtils;
/**
* i18n

@ -0,0 +1,170 @@
package com.jiuyv.sptccc.agile.common.utils.sftp;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jiuyv.sptccc.agile.common.exception.UtilException;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
/**
* SFTP
* 使
* @author zhouliang
*
*/
public class SFTPChannel {
private Session session = null;
private Channel channel = null;
private static final Logger LOG = LoggerFactory.getLogger(SFTPChannel.class);
public Session getSession() {
return session;
}
public Channel getChannel() {
return channel;
}
/**
*
* @param sftpDetails
* @param timeout
* @return
* @throws JSchException
*/
public ChannelSftp getChannel(SFTPConfig sftpDetails, int timeout) throws JSchException {
setConnect(sftpDetails, timeout);
channel = session.openChannel("sftp"); // 打开SFTP通道
channel.connect(); // 建立SFTP通道的连接
LOG.debug("Connected successfully to ftpHost = " + sftpDetails.getHost() + ",as ftpUserName = " + sftpDetails.getUsername());
return (ChannelSftp) channel;
}
/**
*
* @return
* @throws JSchException
*/
public ChannelSftp getChannelNew(final Session session) throws JSchException {
this.session = session;
this.channel = session.openChannel("sftp"); // 打开SFTP通道
this.channel.connect(); // 建立SFTP通道的连接
LOG.debug("Connected successfully to ftpHost");
return (ChannelSftp) channel;
}
public void setConnect(SFTPConfig sftpDetails, int timeout) throws JSchException {
String ftpHost = sftpDetails.getHost();
int ftpPort = sftpDetails.getPort();
String ftpUserName = sftpDetails.getUsername();
String ftpPassword = sftpDetails.getPassword();
JSch jsch = new JSch(); // 创建JSch对象
session = jsch.getSession(ftpUserName, ftpHost, ftpPort); // 根据用户名主机ip端口获取一个Session对象
if (ftpPassword != null) {
session.setPassword(ftpPassword); // 设置密码
}
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
config.put("PreferredAuthentications", "password");
config.put("X11Forwarding", "no");
session.setConfig(config); // 为Session对象设置properties
session.setTimeout(timeout); // 设置timeout时间
session.connect(); // 通过Session建立链接
}
/**
*
*/
public void closeChannel() {
if (channel != null) {
channel.disconnect();
}
if (session != null) {
session.disconnect();
}
LOG.debug("Channel connection close!");
}
/**
*
*/
public void closeInnerChannel() {
if (channel != null) {
channel.disconnect();
}
}
/**
* ,
* 使exec(shell)
* @param shell
* @param command
* @return
* @throws IOException
* @throws JSchException
*/
public List<String> commitCurrShellCmd(String command) throws IOException, JSchException {
return commitCurrShellCmd(command, true);
}
public List<String> commitCurrShellCmd(String command,boolean errFlag) throws IOException, JSchException {
LOG.info("commitCurrShellCmd command>> "+command);//为了测试看
List<String> result=new ArrayList<>();
ChannelExec shell=(ChannelExec) session.openChannel("exec");
shell.setPtyType("dumb");//禁用颜色
shell.setCommand(command);
shell.connect();
InputStream in = shell.getInputStream();
InputStream errin = shell.getErrStream();
try(BufferedReader reader = new BufferedReader(new InputStreamReader(in));
BufferedReader reader2 = new BufferedReader(new InputStreamReader(errin));) {
String line=null;
while ((line = reader.readLine())!=null) {
// if(LOG.isDebugEnabled()) {
// LOG.info("commitCurrShellCmd line>>"+line);//为了测试看
// }
if(StringUtils.isNotBlank(line)&& !line.contains("]$")
&& !line.startsWith("<")) {
result.add(line);
}
}
String errormsg=null;
String line2=null;
while (StringUtils.isNotBlank((line2 = reader2.readLine()))) {
// if(LOG.isDebugEnabled()) {
// LOG.info("commitCurrShellCmd line error>>"+line2);//为了测试看
// }
errormsg=line2;
}
if(StringUtils.isNotBlank(errormsg)) {
LOG.info("commitCurrShellCmd line error>>"+errormsg);
if(errFlag) {
throw new UtilException("Command execution exception");
}else {
result.add(errormsg);//收集错误,直接返回
}
}
}catch (Exception e) {
LOG.info("commitCurrShellCmd error{}",e.getMessage(),e);
}finally {
shell.disconnect();
}
LOG.info("commitCurrShellCmd result>>\n"+StringUtils.join(result,"\n"));//为了测试看
return result;
}
}

@ -0,0 +1,74 @@
package com.jiuyv.sptccc.agile.common.utils.sftp;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
/**
* ssh/
*/
public class SshClientFactory implements PooledObjectFactory<Session>{
private static final Logger LOG = LoggerFactory.getLogger(SshClientFactory.class);
private SFTPConfig config;
public SshClientFactory(SFTPConfig sshClientConfig){
this.config = sshClientConfig;
}
@Override
public PooledObject<Session> makeObject() throws Exception {
JSch jSch = new JSch();
// if (!StringUtils.isEmpty(config.getRsaPath())){
// jSch.addIdentity(config.getRsaPath());
// }
Session session = jSch.getSession(config.getUsername(),config.getHost(),config.getPort());
if (!StringUtils.isEmpty(config.getPassword())){
session.setPassword(config.getPassword());
}
session.setConfig("StrictHostKeyChecking", "no");
session.setConfig("PreferredAuthentications", "password");
session.setConfig("X11Forwarding", "no");
session.setTimeout(120000); // 设置timeout时间
LOG.info("SshClientFactory init ssh client{}{}",config.getHost(), config.getUsername());
return new DefaultPooledObject<>(session);
}
@Override
public void destroyObject(PooledObject<Session> pooledObject) throws Exception {
Session session = pooledObject.getObject();
if (session != null){
session.disconnect();
}
}
@Override
public boolean validateObject(PooledObject<Session> pooledObject) {
Session session = pooledObject.getObject();
return session.isConnected();
}
@Override
public void activateObject(PooledObject<Session> pooledObject) throws Exception {
Session session = pooledObject.getObject();
try {
session.connect();
}catch (JSchException e) {
//忽略错误
//LOG.info("activateObject>>",e);
}
}
@Override
public void passivateObject(PooledObject<Session> pooledObject) throws Exception {
// 忽略
}
}

@ -0,0 +1,36 @@
package com.jiuyv.sptccc.agile.common.utils.sftp;
import java.util.HashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
/**
* ssh
*/
public class SshClientHostPool {
private static final Logger LOG = LoggerFactory.getLogger(SshClientFactory.class);
private static Map<String, SshClientPool> poolMap = new HashMap<>();
public static SshClientPool getHostPool(SFTPConfig config){
SshClientPool clientPool = poolMap.get(config.getHost()+"-"+config.getUsername());
if (clientPool==null){
synchronized (poolMap){
//当两个线程都来到synchronized这一行时需要再代码块再次进行判断否则会再次新建
clientPool = poolMap.get(config.getHost()+"-"+config.getUsername());
if (clientPool==null){
LOG.info("SSH client does not existcreate {}", config.getHost()+"-"+config.getUsername());
clientPool = new SshClientPool(config);
poolMap.put(config.getHost()+"-"+config.getUsername(), clientPool);
}
}
}
return clientPool;
}
}

@ -0,0 +1,48 @@
package com.jiuyv.sptccc.agile.common.utils.sftp;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import com.jcraft.jsch.Session;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
/**
* ssh
*/
public class SshClientPool {
private GenericObjectPool<Session> pool;
public SshClientPool(SFTPConfig sshClientConfig){
init(sshClientConfig);
}
public void init(SFTPConfig sshClientConfig){
SshClientFactory factory = new SshClientFactory(sshClientConfig);
GenericObjectPoolConfig<Session> poolConfig = sshClientConfig.getPoolConfig();
pool = new GenericObjectPool<>(factory, poolConfig!=null?poolConfig:new GenericObjectPoolConfig<>());
}
/**
*
* @return
* @throws Exception
*/
public Session getSession() throws Exception{
return pool.borrowObject();
}
/**
*
* @param session
*/
public void returnSession(Session session){
try {
if (session != null){
pool.returnObject(session);
}
} catch (Exception e) {
//无需错误
}
}
}

@ -0,0 +1,126 @@
package com.jiuyv.sptccc.agile.common.utils.sftp.model;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import com.jcraft.jsch.Session;
/**
* sftp
* @author zhouliang
*
*/
public class SFTPConfig implements java.io.Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
/**
* ip
*/
private String host;
/**
*
*/
private int port=22;
/**
*
*/
private String username;
/**
*
*/
private String password;
/**
* location
*/
private int location;
/** 连接池配置*/
private transient GenericObjectPoolConfig<Session> poolConfig;
/**
* @return the host
*/
public String getHost() {
return host;
}
/**
* @param host the host to set
*/
public void setHost(String host) {
this.host = host;
}
/**
* @return the port
*/
public int getPort() {
return port;
}
/**
* @param port the port to set
*/
public void setPort(int port) {
this.port = port;
}
/**
* @return the username
*/
public String getUsername() {
return username;
}
/**
* @param username the username to set
*/
public void setUsername(String username) {
this.username = username;
}
/**
* @return the password
*/
public String getPassword() {
return password;
}
/**
* @param password the password to set
*/
public void setPassword(String password) {
this.password = password;
}
/**
* @return the location
*/
public int getLocation() {
return location;
}
/**
* @param location the location to set
*/
public void setLocation(int location) {
this.location = location;
}
/**
* @return the poolConfig
*/
public GenericObjectPoolConfig<Session> getPoolConfig() {
return poolConfig;
}
/**
* @param poolConfig the poolConfig to set
*/
public void setPoolConfig(GenericObjectPoolConfig<Session> poolConfig) {
this.poolConfig = poolConfig;
}
}

@ -1,4 +1,4 @@
package com.jiuyv.sptccc.agile.common.config;
package com.jiuyv.sptccc.agile.framework.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

@ -2,7 +2,7 @@ package com.jiuyv.sptccc.agile.system.mapper;
import org.apache.ibatis.annotations.Mapper;
import com.jiuyv.sptccc.agile.common.core.domain.BaseTime;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
@Mapper
public interface ISysTimeBaseMapper {

@ -6,7 +6,7 @@ console:
# 开发环境配置
server:
port: 18083 #本地
port: 18086 #本地
tomcat:
# tomcat的URI编码
uri-encoding: UTF-8

@ -2,7 +2,7 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.jiuyv.sptccc.agile.system.mapper.ISysTimeBaseMapper">
<select id="selectSysCurrentTime" resultType="com.jiuyv.sptccc.agile.common.core.domain.BaseTime">
<select id="selectSysCurrentTime" resultType="com.jiuyv.sptccc.agile.common.model.BaseTime">
SELECT now() AS utcTime
</select>

@ -3,7 +3,7 @@
<parent>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-datacenter</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.3-SNAPSHOT</version>
</parent>
<artifactId>agile-datacenter-sync</artifactId>
@ -12,6 +12,16 @@
</properties>
<dependencies>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-mobile-message-api</artifactId>
<exclusions>
<exclusion>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-datacenter-api</artifactId>

@ -19,6 +19,7 @@ import com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO;
import com.jiuyv.sptccc.agile.business.service.IDatacenterTaskStatusService;
import com.jiuyv.sptccc.agile.business.service.impl.SyncDatacenterService;
import com.jiuyv.sptccc.agile.common.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import com.jiuyv.sptccc.agile.common.utils.sftp.SFTPChannel;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
@ -49,6 +50,10 @@ public class SyncDatacenterBatch {
/** 防止重置任务还没完成 */
private static boolean resetFinishFlag = false;
public static void setResetFinishFlag(){
resetFinishFlag = true;
}
/**
* ()
*
@ -59,36 +64,48 @@ public class SyncDatacenterBatch {
LOGGER.info("【数据同步】重置运行中的任务状态");
datacenterTaskStatusService.doResetDatacenterTaskStatus();
//补充缺失的任务,必须任务正常运行一次后才能补充
datacenterTaskStatusService.doCreateMissingDatacenterTaskStatus();
//注册任务
List<TblDatacenterTask> tasks = datacenterTaskStatusService.getDatacenterTaskNormal();
if(tasks!=null && !tasks.isEmpty()) {
LOGGER.info("【数据同步】待注册任务数量={}",tasks.size());
for(TblDatacenterTask x: tasks) {
LOGGER.info("【数据同步】任务注册={}cron={}",x.getTaskNo(),x.getScheduledCron());
String[] crons = x.getScheduledCron().split(",|");
String[] crons = x.getScheduledCron().split("[,]");
boolean flag = false;
for(int ti=0;ti<crons.length;ti++) {
if(StringUtils.isBlank(crons[ti])) {
continue;
}
//任务实现
Runnable task = () -> {
syncDatacenterService.doStartTaskStatusForNormal(x.getTaskNo());
};
cronTaskRegistrar.addCronTask(x.getTaskNo()+"-"+ti, task, crons[ti].trim());
String cronx = crons[ti].trim().replaceAll(" +", " ");
String[] arrs = cronx.split(" ");
if(arrs.length==6) {
//任务实现
Runnable task = () -> syncDatacenterService.doStartTaskStatusForNormal(x.getTaskNo());
cronTaskRegistrar.addCronTask(x.getTaskNo()+"-"+ti, task, crons[ti].trim());
flag = true;
}
}
//只要有一个错误就报错
if(!flag) {
LOGGER.info("【数据同步】{}的表达式无效, 请检查cron={}",x.getTaskNo(),x.getScheduledCron());
throw new ServiceException("Task cron error");
}
}
}else {
LOGGER.info("【数据同步】数据库无待注册任务");
}
resetFinishFlag = true;
setResetFinishFlag();
}
@Scheduled(cron = "0 0 0/1 * * ?")
@Scheduled(cron = "0 0/30 * * * ?")
public void resetRuningTask2() throws Exception{
//检查是否有运行超过一定时间的任务强制结束(正常任务应该不会很长
LOGGER.info("【数据同步】重置超期的运行中任务状态");
datacenterTaskStatusService.doResetOverdateDatacenterTaskStatus();
}
/**
@ -116,7 +133,7 @@ public class SyncDatacenterBatch {
*
*/
@Scheduled(cron = "0 0 2 * * ?")
public void clearFileTask() throws Exception{
public void clearFileTask() {
LOGGER.info("【数据同步】清理数据源的超期文件");
String clearDs = ConsoleConfig.getClearDs();

@ -1,5 +1,7 @@
package com.jiuyv.sptccc.agile.business.common.constant;
import java.util.regex.Pattern;
/**
*
*/
@ -19,7 +21,8 @@ public class SyncDatacenterConstants
public static final String TEMP_FILE_EXTENSION ="tmp";
/** 命令:移除超期的文件 */
public static final String SHELL_RM_OVERDATE ="find %s -type f -mtime +%s -exec rm {} \\;";
// public static final String SHELL_RM_OVERDATE ="find %s -type f -mtime +%s -exec rm {} \\;";
public static final String SHELL_RM_OVERDATE ="find %s -type f -mtime +%s -delete";
//类型转换
public static final String CONVERT_TYPE_TO_DATE="Date";
@ -29,10 +32,10 @@ public class SyncDatacenterConstants
public static final String CONVERT_TYPE_TO_BIGDECIMAL="BigDecimal";
public static final String CONVERT_TYPE_TO_STRING="String";
/** 匹配2020-01-01 13:00:00格式 */
public static final Pattern RULE_DATETIME = Pattern.compile("^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$");
/** 创建日期字段*/
public static final String KEY_TASK_CREATE_TIME="task_create_time";
/** 日期转换用 */
public static final String FORMAT_DATETIME ="yyyy-MM-dd HH:mm:ss";
public static final String FORMAT_DATEDAY ="yyyy-MM-dd";
public static final String FORMAT_DATETIME_SIMPLE ="yyyyMMddHHmmss";
public static final String FORMAT_DATEDAY_SIMPLE ="yyyyMMdd";
}

@ -10,19 +10,23 @@ public class TblDatacenterTaskStatusEnum {
/** 任务类型*/
public enum TASK_TYPE {
TT1("1", "天"),
TT2("2", "时段"),
TT3("3", "小时"),
TT4("4", "分钟"),
TT99("99", "其他"),//默认也按天
TT1("1", "天","1d"),
TT2("2", "时段","1h"),
TT3("3", "小时","1h"),
TT4("4", "分钟","1m"),
TT5("5", "月","1M"),
TT6("6", "年","1y"),
TT99("99", "其他","1"),//默认不会自动补充任务
;
private String code;
private String msg;
private String time;//该时间用于后面计算循环
TASK_TYPE(String code, String msg) {
TASK_TYPE(String code, String msg,String time) {
this.code = code;
this.msg = msg;
this.time = time;
}
public String getCode() {
@ -32,6 +36,10 @@ public class TblDatacenterTaskStatusEnum {
public String getMsg() {
return msg;
}
public String getTime() {
return time;
}
}
/** 业务状态*/
@ -82,4 +90,17 @@ public class TblDatacenterTaskStatusEnum {
return msg;
}
}
/** 检查是否支持任务时间类型 */
public static boolean checkIsSupportTaskType(String taskType) {
if(TASK_TYPE.TT99.getCode().equals(taskType)) {
return false;
}
for(TASK_TYPE t:TASK_TYPE.values()) {
if(t.getCode().equals(taskType)) {
return true;
}
}
return false;
}
}

@ -4,6 +4,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@ -46,6 +47,11 @@ public class SqlHandlerResultVO implements Serializable
*/
private boolean customFieldFlag=false;
/**
*
*/
private Set<String> cols;
/**
* @return the selectSql
*/
@ -127,4 +133,12 @@ public class SqlHandlerResultVO implements Serializable
public void setLocalSelectSql(String localSelectSql) {
this.localSelectSql = localSelectSql;
}
public Set<String> getCols() {
return cols;
}
public void setCols(Set<String> cols) {
this.cols = cols;
}
}

@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jiuyv.sptccc.agile.business.common.constant.SyncDatacenterConstants;
import com.jiuyv.sptccc.agile.business.common.model.SqlHandlerResultVO;
import com.jiuyv.sptccc.agile.business.common.model.SqlHandlerTypeVO;
import com.jiuyv.sptccc.agile.common.constant.Constants;
import com.jiuyv.sptccc.agile.common.utils.JsonUtil;
/**
@ -95,6 +96,7 @@ public class SqlHandlerUtilx {
cols.add(vals[0]);
}
vo.setCols(cols);
vo.setInsertSql(convertInsertSql(cols, toTableName, 1));
}else {//这个是只转换字段类型
String[] arrs = cfgJson.split(";|");
@ -133,7 +135,7 @@ public class SqlHandlerUtilx {
}
//把SQL关系配置转换为实际的sql语句(远程来源是文件)
public static SqlHandlerResultVO convertJsonToSql(Collection<String> cols,String toTableName, String cfgJson) throws Exception {
public static SqlHandlerResultVO convertJsonToSql(Set<String> cols,String toTableName, String cfgJson) throws Exception {
if(StringUtils.isBlank(cfgJson)) {
cfgJson = "*";
}
@ -198,6 +200,7 @@ public class SqlHandlerUtilx {
vo.setLocalSelectSql("SELECT * FROM "+toTableName+" WHERE 1>2");//用于获取字段信息
vo.setCols(cols);
String sqlInsert = convertInsertSql(cols, toTableName, 1);
vo.setInsertSql(sqlInsert);
@ -364,10 +367,10 @@ public class SqlHandlerUtilx {
public static java.sql.Date convertStr2SqlDate(String datestr) {
try {
String format=null;
if(datestr.length()==SyncDatacenterConstants.FORMAT_DATETIME_SIMPLE.length()){
format = SyncDatacenterConstants.FORMAT_DATETIME_SIMPLE;
}else if(datestr.length()==SyncDatacenterConstants.FORMAT_DATEDAY_SIMPLE.length()) {
format = SyncDatacenterConstants.FORMAT_DATEDAY_SIMPLE;
if(datestr.length()==Constants.FORMAT_DATETIME_SIMPLE.length()){
format = Constants.FORMAT_DATETIME_SIMPLE;
}else if(datestr.length()==Constants.FORMAT_DATEDAY_SIMPLE.length()) {
format = Constants.FORMAT_DATEDAY_SIMPLE;
}
if(format != null) {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
@ -382,10 +385,10 @@ public class SqlHandlerUtilx {
public static java.sql.Timestamp convertStr2SqlTimestamp(String datestr) {
try {
String format=null;
if(datestr.length()==SyncDatacenterConstants.FORMAT_DATETIME_SIMPLE.length()){
format = SyncDatacenterConstants.FORMAT_DATETIME_SIMPLE;
}else if(datestr.length()==SyncDatacenterConstants.FORMAT_DATEDAY_SIMPLE.length()) {
format = SyncDatacenterConstants.FORMAT_DATEDAY_SIMPLE;
if(datestr.length()==Constants.FORMAT_DATETIME_SIMPLE.length()){
format = Constants.FORMAT_DATETIME_SIMPLE;
}else if(datestr.length()==Constants.FORMAT_DATEDAY_SIMPLE.length()) {
format = Constants.FORMAT_DATEDAY_SIMPLE;
}
if(format != null) {
SimpleDateFormat dateFormat = new SimpleDateFormat(format);

@ -14,15 +14,18 @@ import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import com.jiuyv.sptccc.agile.business.common.enums.TblDatacenterTaskStatusEnum;
import com.jiuyv.sptccc.agile.business.entity.TblDatacenterTask;
import com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO;
import com.jiuyv.sptccc.agile.common.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.constant.Constants;
/**
@ -47,6 +50,14 @@ public class SyncDatacenterUtils {
taskConditions = datemap.get("dateDay2");
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT2.getCode().equals(task.getTaskType())) {
taskConditions = datemap.get("dateDay2")+"-"+datemap.get("hourSpan");
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT3.getCode().equals(task.getTaskType())) {
taskConditions = datemap.get("dateHour2");
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT4.getCode().equals(task.getTaskType())) {
taskConditions = datemap.get("dateMinute2");
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT5.getCode().equals(task.getTaskType())) {
taskConditions = datemap.get("dateMonth2");
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT6.getCode().equals(task.getTaskType())) {
taskConditions = datemap.get("dateYear2");
}else {
taskConditions = datemap.get("dateTime2");
}
@ -67,12 +78,132 @@ public class SyncDatacenterUtils {
taskStatus.setReadCount("0");
taskStatus.setVersionNum(0);
taskStatus.setRecToken(BaseManagerUtils.getNewRecToken());
taskStatus.setCreateTime(time);
//让其等于任务原始时间,而不是实时运行时间,方便数据库分区统一
taskStatus.setCreateTime(convertStr2Date(datemap.get("nowdate")));
taskStatus.setUpdateTime(time);
taskStatus.setCurrStartDate(time);
taskStatus.setDatacenterTask(task);//配置
return taskStatus;
}
/**
*
* @param date
* @param dateExpression 10s,1m,1h,1d,1M,1y
*/
public static Map<String,String> convertQueryDateParam(Date date,String dateExpression) {
//dateYear=2020,dateMonth=2020-01,dateDay=2020-01-02,dateHour=2020-01-02 13,dateMinute=2020-01-02 13:20,dateTime=2020-01-02 13:20:00
//dateYear2=2020,dateMonth2=202001,dateDay2=20200102,dateHour2=2020010213,dateMinute=202001021320,dateTime2=20200102132000
//year=2020, month=01, day=02, hour=13, minute=20, second=00, hourSpan=130, span=0(前半小时0后半小时1)
//partDay=am或pm
Map<String,String> param = new LinkedHashMap<>();
LocalDateTime currentDate = convertDateMinus(date, dateExpression);
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(Constants.FORMAT_DATETIME);//格式不能改
formatter.withZone(ZoneId.of(Constants.DEFAULT_TIMEZONE));
dateExpression = currentDate.format(formatter);
String datestr = dateExpression.replaceAll("[: -]+", "");
//20200111123050
String year = datestr.substring(0, 4);
String month = datestr.substring(4, 6);
String day = datestr.substring(6, 8);
String hour = datestr.substring(8, 10);
String minute = datestr.substring(10, 12);
String second = datestr.substring(12, 14);
//hourSpan=前半小时为0后半小时为1
String span = Integer.parseInt(minute)<30?"0":"1";
//am上午pm下午
String partDay = Integer.parseInt(hour)<12?"am":"pm";
param.put("dateYear", year);
param.put("dateMonth", dateExpression.substring(0, 7));
param.put("dateDay", dateExpression.substring(0, 10));
param.put("dateHour", dateExpression.substring(0, 13));
param.put("dateMinute", dateExpression.substring(0, 16));
param.put("dateTime", dateExpression);
param.put("dateYear2", year);
param.put("dateMonth2", datestr.substring(0, 6));
param.put("dateDay2", datestr.substring(0, 8));
param.put("dateHour2", datestr.substring(0, 10));
param.put("dateMinute2", datestr.substring(0, 12));
param.put("dateTime2", datestr);
param.put("year", year);
param.put("month", month);
param.put("day", day);
param.put("hour", hour);
param.put("minute", minute);
param.put("second", second);
param.put("span", span);
param.put("hourSpan", hour+span);
param.put("partDay", partDay);
param.put("nowdate", DateFormatUtils.format(date, Constants.FORMAT_DATETIME, TimeZone.getTimeZone(ZoneId.of(Constants.DEFAULT_TIMEZONE))));
return param;
}
/**
*
* @param arrs cron
* @param taskType
* @param endtime
* @return
*/
public static LocalDateTime handleDateByTaskType(String[] arrs,String taskType, Date endtime) {
//如下类型与时间有关的必须是整点(否则时间会漂移)
LocalDateTime currentDate = SyncDatacenterUtils.convertDateMinus(endtime, null);
if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT1.getCode().equals(taskType)) {
String str = arrs[2] + arrs[1] + arrs[0];
String str2 = currentDate.getHour()+""+currentDate.getMinute()+""+currentDate.getSecond();
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT1.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0])).withMinute(Integer.parseInt(arrs[1]))
.withHour(Integer.parseInt(arrs[2]));
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT2.getCode().equals(taskType)) {
String str = arrs[1] + arrs[0];
String str2 = currentDate.getMinute()+""+currentDate.getSecond();
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT2.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0])).withMinute(Integer.parseInt(arrs[1]));
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT3.getCode().equals(taskType)) {
String str = arrs[1] + arrs[0];
String str2 = currentDate.getMinute()+""+currentDate.getSecond();
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT3.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0])).withMinute(Integer.parseInt(arrs[1]));
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT4.getCode().equals(taskType)) {
String str = arrs[0];
String str2 = currentDate.getSecond()+"";
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT4.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0]));
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT5.getCode().equals(taskType)) {
String str = arrs[3]+ arrs[2] + arrs[1] + arrs[0];
String str2 = currentDate.getDayOfMonth()+""+currentDate.getHour()+""+currentDate.getMinute()
+""+currentDate.getSecond();
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT5.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0])).withMinute(Integer.parseInt(arrs[1]))
.withHour(Integer.parseInt(arrs[2])).withDayOfMonth(Integer.parseInt(arrs[3]));
}else if(TblDatacenterTaskStatusEnum.TASK_TYPE.TT6.getCode().equals(taskType)) {
String str = arrs[4]+arrs[3]+ arrs[2] + arrs[1] + arrs[0];
String str2 = currentDate.getMonth() +""+ currentDate.getDayOfMonth()+""+currentDate.getHour()
+""+currentDate.getMinute()+""+currentDate.getSecond();
if(str2.compareTo(str)<0) {//该时刻任务跳过
currentDate = convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.TT6.getTime());
}
currentDate = currentDate.withSecond(Integer.parseInt(arrs[0])).withMinute(Integer.parseInt(arrs[1]))
.withHour(Integer.parseInt(arrs[2])).withDayOfMonth(Integer.parseInt(arrs[3]))
.withMonth(Integer.parseInt(arrs[4]));
}
return currentDate;
}
/**
*
@ -136,19 +267,26 @@ public class SyncDatacenterUtils {
return totaltime.compareTo(new BigDecimal(0))>0;
}
public static Date convertStr2Date(String datestr) {
try {
SimpleDateFormat dateFormat = new SimpleDateFormat(Constants.FORMAT_DATETIME);
dateFormat.setTimeZone(TimeZone.getTimeZone(ZoneId.of(Constants.DEFAULT_TIMEZONE)));
return dateFormat.parse(datestr);
} catch (Exception e) {
}
return null;
}
/**
*
*
* @param date
* @param param
* @param dateExpression 10s,1m,1h,1d,1M,1y
* @return
*/
public static Map<String,String> convertQueryDateParam(Date date,String dateExpression) {
//dateYear=2020,dateMonth=2020-01,dateDay=2020-01-02,dateHour=2020-01-02 13,dateMinute=2020-01-02 13:20,dateTime=2020-01-02 13:20:00
//dateYear2=2020,dateMonth2=202001,dateDay2=20200102,dateHour2=2020010213,dateMinute=202001021320,dateTime2=20200102132000
//year=2020, month=01, day=02, hour=13, minute=20, second=00, hourSpan=0(前半小时0后半小时1)
Map<String,String> param = new LinkedHashMap<>();
LocalDateTime currentDate = LocalDateTime.ofInstant(date.toInstant(), ZoneId.of("+08:00"));
public static LocalDateTime convertDateMinus(Date date,String dateExpression) {
LocalDateTime currentDate = LocalDateTime.ofInstant(date.toInstant(), ZoneId.of(Constants.DEFAULT_TIMEZONE));
return convertDateMinus(currentDate, dateExpression);
}
public static LocalDateTime convertDateMinus(LocalDateTime currentDate,String dateExpression) {
if(StringUtils.isNotBlank(dateExpression)) {
String operation = "s";
int value = 0;
@ -160,7 +298,6 @@ public class SyncDatacenterUtils {
if(m.groupCount() == 2) {
operation = m.group(2);
}
param.put("diffVal", dateExpression);//存一下
}
}catch(Exception e) {
}
@ -187,54 +324,9 @@ public class SyncDatacenterUtils {
break;
}
}
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
dateExpression = currentDate.format(formatter);
String datestr = dateExpression.replaceAll("[: -]+", "");
//20200111123050
String year = datestr.substring(0, 4);
String month = datestr.substring(4, 6);
String day = datestr.substring(6, 8);
String hour = datestr.substring(8, 10);
String minute = datestr.substring(10, 12);
String second = datestr.substring(12, 14);
//hourSpan=前半小时为0后半小时为1
String span = Integer.parseInt(minute)<30?"0":"1";
param.put("dateYear", year);
param.put("dateMonth", dateExpression.substring(0, 7));
param.put("dateDay", dateExpression.substring(0, 10));
param.put("dateHour", dateExpression.substring(0, 13));
param.put("dateMinute", dateExpression.substring(0, 16));
param.put("dateTime", dateExpression);
param.put("dateYear2", year);
param.put("dateMonth2", datestr.substring(0, 6));
param.put("dateDay2", datestr.substring(0, 8));
param.put("dateHour2", datestr.substring(0, 10));
param.put("dateMinute2", datestr.substring(0, 12));
param.put("dateTime2", datestr);
param.put("year", year);
param.put("month", month);
param.put("day", day);
param.put("hour", hour);
param.put("minute", minute);
param.put("second", second);
param.put("span", span);
param.put("hourSpan", hour+span);
return param;
return currentDate;
}
public static Date convertStr2Date(String datestr) {
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return dateFormat.parse(datestr);
} catch (Exception e) {
}
return null;
}
/**
*
*
@ -261,7 +353,9 @@ public class SyncDatacenterUtils {
}
if(formatStr.startsWith("fdate") || formatStr.startsWith("ftime")){
DateTimeFormatter oldFormatter = DateTimeFormatter.ofPattern(v1);
oldFormatter.withZone(ZoneId.of(Constants.DEFAULT_TIMEZONE));
DateTimeFormatter newFormatter = DateTimeFormatter.ofPattern(v2);
newFormatter.withZone(ZoneId.of(Constants.DEFAULT_TIMEZONE));
if(formatStr.startsWith("fdate")){//"2020年01月02日", "fdate(yyyy年MM月dd日,yyyy/MM/dd)"
val = LocalDate.parse(val, oldFormatter).format(newFormatter);
}

@ -15,6 +15,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jiuyv.sptccc.agile.business.common.constant.SyncDatacenterConstants;
import com.jiuyv.sptccc.agile.business.common.model.DDsProperties;
import com.jiuyv.sptccc.agile.business.common.model.SqlHandlerResultVO;
import com.jiuyv.sptccc.agile.business.common.model.SqlHandlerTypeVO;
@ -49,6 +50,10 @@ public class ReaderWriterHelper {
private boolean jdbcWriterHiveFlag=false;//是否hive特殊处理
private String datetime=null;//当前任务时间
private Object nowtime=null;//转换为最终格式
public JdbcBaseReader getJdbcReader() {
return jdbcReader;
}
@ -85,6 +90,9 @@ public class ReaderWriterHelper {
public void setHasTitleFlag(boolean hasTitleFlag) {
this.hasTitleFlag = hasTitleFlag;
}
public void setDatetime(String datetime) {
this.datetime = datetime;
}
/**
*
@ -209,21 +217,37 @@ public class ReaderWriterHelper {
}
//清除本地已有的数据等
Map<String,String> colTypeMap=new LinkedHashMap<>();
if(!writeSftpFlag) {//写入数据库
//实时获取数据库的类型
colTypeMap = getLocalJdbcType(sqlVO.getLocalSelectSql());
if(StringUtils.isNotBlank(preSql)) {
LOGGER.info("writeData exec preSql>>{}",preSql);
jdbcWriter.execSql(preSql, null);
jdbcWriter.closeCurrStatement();//关闭
}
if(StringUtils.isBlank(insertSql)) {//为空使用了*号
insertSql=SqlHandlerUtilx.convertInsertSql(currcols, mappingInfo.getLocalTable(),1);
}
//创建日期
String createTimeType = colTypeMap.get(SyncDatacenterConstants.KEY_TASK_CREATE_TIME);
if(createTimeType != null) {
this.nowtime = SqlHandlerUtilx.convertType(datetime,SyncDatacenterConstants.KEY_TASK_CREATE_TIME, createTimeType
, jdbcWriterHiveFlag||writeSftpFlag);
if(customFieldFlag) {//自定义字段
currcols = sqlVO.getCols();
}
currcols.add(SyncDatacenterConstants.KEY_TASK_CREATE_TIME);
insertSql = SqlHandlerUtilx.convertInsertSql(currcols, mappingInfo.getLocalTable(), 1);
}
LOGGER.info("writeData exec insertSql>>{}",insertSql);
}else{
//未实现
}
LOGGER.info("writeData exec preSql>>time={}",(System.currentTimeMillis()-ttime2));
if(StringUtils.isBlank(insertSql)) {//为空使用了*号
insertSql=SqlHandlerUtilx.convertInsertSql(currcols, mappingInfo.getLocalTable(),1);
}
//读取数据N条写一次
ttime2 = System.currentTimeMillis();
List<List<Object>> lists=null;
@ -276,33 +300,26 @@ public class ReaderWriterHelper {
Map<String, SqlHandlerTypeVO> map = sqlVO.getFieldMapping();
//清除本地已有的数据等
Map<String,String> colMap=new LinkedHashMap<>();
Map<String,String> colTypeMap=new LinkedHashMap<>();
if(!writeSftpFlag) {//写入数据库
//实时获取数据库的类型
ResultSet resultSet=jdbcWriter.execSql(sqlVO.getLocalSelectSql(), null);
if(resultSet!=null) {
ResultSetMetaData metaData = resultSet.getMetaData();
int n=metaData.getColumnCount();
for(int i=1;i<=n;i++) {
String colname =metaData.getColumnName(i);
String type = metaData.getColumnTypeName(i);
if(colname.contains(".")) {
String colname2=colname.replaceAll("^.*\\.", "");
if(colMap.get(colname2)==null) {//如果确实有同名直接抛弃即可
colMap.put(colname2.toLowerCase(), type);
}
}else {
colMap.put(colname.toLowerCase(),type);
}
}
}
jdbcWriter.closeCurrStatement();//关闭
colTypeMap = getLocalJdbcType(sqlVO.getLocalSelectSql());
if(StringUtils.isNotBlank(preSql)) {
LOGGER.info("writeData exec preSql>>{}",preSql);
jdbcWriter.execSql(preSql, null);
jdbcWriter.closeCurrStatement();//关闭
}
//创建日期
String createTimeType = colTypeMap.get(SyncDatacenterConstants.KEY_TASK_CREATE_TIME);
if(createTimeType != null) {
this.nowtime = SqlHandlerUtilx.convertType(datetime,SyncDatacenterConstants.KEY_TASK_CREATE_TIME, createTimeType
, jdbcWriterHiveFlag||writeSftpFlag);
currcols = sqlVO.getCols();
currcols.add(SyncDatacenterConstants.KEY_TASK_CREATE_TIME);
insertSql = SqlHandlerUtilx.convertInsertSql(currcols, mappingInfo.getLocalTable(), 1);
}
LOGGER.info("writeData exec insertSql>>{}",insertSql);
}else{
//未实现
}
@ -311,7 +328,7 @@ public class ReaderWriterHelper {
//读取数据N条写一次
ttime2 = System.currentTimeMillis();
List<List<Object>> lists = null;
while(!(lists = readFileLines(inputReader,customFieldFlag, fields,map, colMap)).isEmpty()) {
while(!(lists = readFileLines(inputReader,customFieldFlag, fields,map, colTypeMap)).isEmpty()) {
LOGGER.debug("writeData Task Progress>>read time={}",(System.currentTimeMillis()-ttime2));
writeDataJdbcOrFile(insertSql, lists);
@ -331,6 +348,31 @@ public class ReaderWriterHelper {
}
return true;
}
private Map<String,String> getLocalJdbcType(String localSelectSql) throws Exception {
Map<String,String> colMap=new LinkedHashMap<>();
//实时获取数据库的类型
ResultSet resultSet=jdbcWriter.execSql(localSelectSql, null);
if(resultSet!=null) {
ResultSetMetaData metaData = resultSet.getMetaData();
int n=metaData.getColumnCount();
for(int i=1;i<=n;i++) {
String colname =metaData.getColumnName(i);
String type = metaData.getColumnTypeName(i);
if(colname.contains(".")) {
String colname2=colname.replaceAll("^.*\\.", "");
if(colMap.get(colname2)==null) {//如果确实有同名直接抛弃即可
colMap.put(colname2.toLowerCase(), type);
}
}else {
colMap.put(colname.toLowerCase(),type);
}
}
}
jdbcWriter.closeCurrStatement();//关闭
return colMap;
}
private List<List<Object>> readJdbcLines(ResultSet resultSet,boolean customFieldFlag
, Map<String,Integer> colMap, Map<String, SqlHandlerTypeVO> map) throws Exception {
List<List<Object>> lists=new ArrayList<>();
@ -354,6 +396,9 @@ public class ReaderWriterHelper {
row.add(colValue);
}
}
if(nowtime != null) {//创建日期
row.add(nowtime);
}
lists.add(row);
if(lists.size()>=singleWriteNumber) {//够数
break;
@ -362,7 +407,7 @@ public class ReaderWriterHelper {
return lists;
}
private List<List<Object>> readFileLines(BufferedReader inputReader,boolean customFieldFlag
, String[] fields, Map<String, SqlHandlerTypeVO> map, Map<String,String> colMap) throws Exception {
, String[] fields, Map<String, SqlHandlerTypeVO> map, Map<String,String> colTypeMap) throws Exception {
List<String> lines = new ArrayList<>();
String line;
while ((line = inputReader.readLine()) != null) {
@ -386,10 +431,10 @@ public class ReaderWriterHelper {
String factFieldType = null;
if(mappingval != null) {
factFieldType = colMap.get(mappingval.getColumnCode().toLowerCase());
factFieldType = colTypeMap.get(mappingval.getColumnCode().toLowerCase());
vals[i] = SyncDatacenterUtils.convertFieldFormat(vals[i], mappingval.getConvertFormat());
}else {
factFieldType = colMap.get(field.toLowerCase());
factFieldType = colTypeMap.get(field.toLowerCase());
}
if(factFieldType != null) {
//文件默认都是string强制按插入数据库转换
@ -404,7 +449,9 @@ public class ReaderWriterHelper {
row.add(colValue);
}
}
if(nowtime != null) {//创建日期
row.add(nowtime);
}
lists.add(row);
}

@ -115,7 +115,7 @@ public class SftpFileBaseReader {
if(null!=reader) {
reader.closeInnerChannel();
}
if(null!=poolReader) {
if(null!=poolReader && null!=reader) {
poolReader.returnSession(reader.getSession());//不再关闭,直接返还连接
}
}

@ -1,7 +1,5 @@
package com.jiuyv.sptccc.agile.business.entity;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.Date;
@ -12,7 +10,7 @@ import java.util.Date;
*/
public class TblDatacenterTask implements java.io.Serializable {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
/** 任务编号 */
private String taskNo;
@ -71,6 +69,9 @@ public class TblDatacenterTask implements java.io.Serializable {
/** 表达式 */
private String scheduledCron;
/** 异常运行时间 */
private String errorIntervalTime;
/** 备注 */
private String remarks;
@ -344,6 +345,19 @@ public class TblDatacenterTask implements java.io.Serializable {
this.scheduledCron = scheduledCron;
}
/**
* Get
*/
public String getErrorIntervalTime(){
return errorIntervalTime;
}
/**
* Set
*/
public void setErrorIntervalTime(String errorIntervalTime){
this.errorIntervalTime = errorIntervalTime;
}
/**
* Get
*/

@ -1,7 +1,5 @@
package com.jiuyv.sptccc.agile.business.entity;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.Date;
@ -53,6 +51,9 @@ public class TblDatacenterTaskStatus implements java.io.Serializable {
/** 执行间隔时间 */
private String runIntervalTime;
/** 异常运行时间 */
private String errorIntervalTime;
/** 已读取数量 */
private String readCount;
@ -248,6 +249,19 @@ public class TblDatacenterTaskStatus implements java.io.Serializable {
this.runIntervalTime = runIntervalTime;
}
/**
* Get
*/
public String getErrorIntervalTime(){
return errorIntervalTime;
}
/**
* Set
*/
public void setErrorIntervalTime(String errorIntervalTime){
this.errorIntervalTime = errorIntervalTime;
}
/**
* Get
*/
@ -351,5 +365,4 @@ public class TblDatacenterTaskStatus implements java.io.Serializable {
public void setRsv3(String rsv3){
this.rsv3 = rsv3;
}
}

@ -21,11 +21,6 @@ public class TblDatacenterTaskStatusVO extends TblDatacenterTaskStatus implement
/** 项目原始配置 */
private TblDatacenterTask datacenterTask;
/** 间隔时间类型m、h */
private String overdateType;
/** 间隔时间 30 */
private String overdate;
public List<String> getBusStatuss() {
return busStatuss;
}
@ -39,17 +34,4 @@ public class TblDatacenterTaskStatusVO extends TblDatacenterTaskStatus implement
public void setDatacenterTask(TblDatacenterTask datacenterTask) {
this.datacenterTask = datacenterTask;
}
public String getOverdateType() {
return overdateType;
}
public void setOverdateType(String overdateType) {
this.overdateType = overdateType;
}
public String getOverdate() {
return overdate;
}
public void setOverdate(String overdate) {
this.overdate = overdate;
}
}

@ -26,6 +26,9 @@ public interface TblDatacenterTaskStatusMapper{
/** 更新记录 */
int updateByMap(@Param("vo") TblDatacenterTaskStatus record,@Param("map") TblDatacenterTaskStatusVO paramMap);
/** 查询运行的任务集合 */
List<TblDatacenterTaskStatusVO> selectRunningTaskList(TblDatacenterTaskStatusVO paramMap);
/** 查询错误待重启任务集合 */
List<TblDatacenterTaskStatusVO> selectErrorTaskList(TblDatacenterTaskStatusVO paramMap);
@ -34,4 +37,7 @@ public interface TblDatacenterTaskStatusMapper{
/** 更新待重置的任务记录 */
int updateResetByMap(@Param("vo") TblDatacenterTaskStatus record,@Param("map") TblDatacenterTaskStatusVO paramMap);
/** 查询任务是否存在 */
List<TblDatacenterTaskStatusVO> selectLastTaskTimeExist(TblDatacenterTaskStatusVO paramMap);
}

@ -18,6 +18,9 @@ public interface IDatacenterTaskStatusService {
/** 重置运行中的任务 */
public void doResetDatacenterTaskStatus() throws Exception;
/** 重置运行中的任务(超期的) */
public void doResetOverdateDatacenterTaskStatus() throws Exception;
/** 获取待注册定时任务 */
public List<TblDatacenterTask> getDatacenterTaskNormal() throws Exception;
@ -40,4 +43,7 @@ public interface IDatacenterTaskStatusService {
/** 准备重启失败的任务 */
public List<TblDatacenterTaskStatusVO> doRestartTaskStatusForError(BaseTime timeVO) throws Exception;
/** 创建停机期间缺失的任务(任务运行过一次才会自动补充,反之需要去填写临时条件) */
public void doCreateMissingDatacenterTaskStatus() throws Exception;
}

@ -1,18 +1,28 @@
package com.jiuyv.sptccc.agile.business.service.impl;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.jiuyv.sptccc.agile.business.common.constant.SyncDatacenterConstants;
import com.jiuyv.sptccc.agile.business.common.enums.TblDatacenterTaskStatusEnum;
import com.jiuyv.sptccc.agile.business.common.utils.SyncDatacenterUtils;
import com.jiuyv.sptccc.agile.business.entity.TblDatacenterTask;
@ -23,7 +33,10 @@ import com.jiuyv.sptccc.agile.business.mapper.TblDatacenterTaskMapper;
import com.jiuyv.sptccc.agile.business.mapper.TblDatacenterTaskStatusMapper;
import com.jiuyv.sptccc.agile.business.service.IDatacenterTaskStatusService;
import com.jiuyv.sptccc.agile.common.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.constant.CacheNameConstants;
import com.jiuyv.sptccc.agile.common.constant.Constants;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import com.jiuyv.sptccc.agile.framework.config.caffeine.RedisCache;
/**
@ -40,6 +53,9 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
private TblDatacenterTaskStatusMapper tblDatacenterTaskStatusMapper;
@Autowired
private TblDatacenterTaskMapper tblDatacenterTaskMapper;
@Autowired
private RedisCache redisCache;
@Override
public void doResetDatacenterTaskStatus() throws Exception {
@ -56,6 +72,38 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
tblDatacenterTaskStatusMapper.updateResetByMap(vo, param);
}
@Override
public void doResetOverdateDatacenterTaskStatus() throws Exception {
Date time = BaseManagerUtils.getSystemTime().getDate();
TblDatacenterTaskStatusVO param = new TblDatacenterTaskStatusVO();
param.setBusStatus(TblDatacenterTaskStatusEnum.BUS_STATUS.RUNING.getCode());
List<TblDatacenterTaskStatusVO> list = tblDatacenterTaskStatusMapper.selectRunningTaskList(param);
if(list.isEmpty()) {
return;
}
//只需要超过间隔的
list = list.stream().filter(x->StringUtils.isNotBlank(x.getErrorIntervalTime())
&& SyncDatacenterUtils.checkTimeExceed(x.getCurrStartDate(),time, x.getErrorIntervalTime())
&& null==redisCache.getValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, x.getTaskNo()+ x.getTaskConditions()))
.collect(Collectors.toList());
if(list.isEmpty()) {
return;
}
for(TblDatacenterTaskStatusVO x:list) {
param = new TblDatacenterTaskStatusVO();
param.setTaskNo(x.getTaskNo());
param.setTaskConditions(x.getTaskConditions());
param.setBusStatus(TblDatacenterTaskStatusEnum.BUS_STATUS.RUNING.getCode());
//让这些任务结束次数增加1
TblDatacenterTaskStatus vo = new TblDatacenterTaskStatus();
vo.setRecToken(BaseManagerUtils.getNewRecToken());
vo.setUpdateTime(time);
vo.setBusStatus(TblDatacenterTaskStatusEnum.BUS_STATUS.END.getCode());
tblDatacenterTaskStatusMapper.updateResetByMap(vo, param);
}
}
@Override
public List<TblDatacenterTask> getDatacenterTaskNormal() throws Exception {
TblDatacenterTaskVO taskparam = new TblDatacenterTaskVO();
@ -84,6 +132,7 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
taskvo.setUpdateTime(taskStatus.getCurrStartDate());
taskvo.setLastRunDate(taskStatus.getCurrStartDate());
taskvo.setRecToken(BaseManagerUtils.getNewRecToken());
taskvo.setVersionNum(task.getVersionNum());//不用更新
TblDatacenterTaskVO taskparam = new TblDatacenterTaskVO();
taskparam.setTaskNo(task.getTaskNo());
taskparam.setVersionNum(task.getVersionNum());
@ -108,7 +157,8 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
if(data != null) {
return 0;//0表示不处理
}
taskStatus.setBusStatus(TblDatacenterTaskStatusEnum.BUS_STATUS.RUNING.getCode());
taskStatus.setBusStatus(StringUtils.isBlank(taskStatus.getBusStatus())
?TblDatacenterTaskStatusEnum.BUS_STATUS.RUNING.getCode():taskStatus.getBusStatus());
taskStatus.setDataStatus(TblDatacenterTaskStatusEnum.DATA_STATUS.NORMAL.getCode());
return tblDatacenterTaskStatusMapper.insert(taskStatus);
}catch(Exception e) {
@ -164,8 +214,24 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
list = list.stream().filter(x-> SyncDatacenterUtils.checkTimeExceed(x.getCurrStartDate(),time, x.getRunIntervalTime()))
.collect(Collectors.toList());
if(!list.isEmpty()) {
//获取任务配置
List<String> ids = list.stream().map(x->x.getTaskNo()).distinct().collect(Collectors.toList());
TblDatacenterTaskVO paramTask = new TblDatacenterTaskVO();
paramTask.setDataStatus(TblDatacenterTaskStatusEnum.DATA_STATUS.NORMAL.getCode());
paramTask.setTaskNos(ids);
List<TblDatacenterTask> tasklist = tblDatacenterTaskMapper.selectListByMap(paramTask);
Map<String, TblDatacenterTask> map = tasklist.stream().distinct().collect(Collectors.toMap(x->x.getTaskNo(), x->x));
String newRecToken = BaseManagerUtils.getNewRecToken();
for(TblDatacenterTaskStatus x:list) {
for(TblDatacenterTaskStatusVO x:list) {
TblDatacenterTask datacenterTask = map.get(x.getTaskNo());
if(datacenterTask==null) {//任务已失效
x.setDataStatus(TblDatacenterTaskStatusEnum.DATA_STATUS.DELETED.getCode());
continue;
}
x.setDatacenterTask(datacenterTask);//配置
//让这些任务开始次数减少1
TblDatacenterTaskStatus vo = new TblDatacenterTaskStatus();
vo.setRecToken(newRecToken);
@ -178,69 +244,231 @@ public class DatacenterTaskStatusServiceImpl implements IDatacenterTaskStatusSer
param.setBusStatuss(Arrays.asList(TblDatacenterTaskStatusEnum.BUS_STATUS.UNFINISH.getCode()
,TblDatacenterTaskStatusEnum.BUS_STATUS.END.getCode()));
tblDatacenterTaskStatusMapper.updateErrorByMap(vo, param);
}
List<String> ids = list.stream().map(x->x.getTaskNo()).distinct().collect(Collectors.toList());
TblDatacenterTaskVO paramTask = new TblDatacenterTaskVO();
paramTask.setDataStatus(TblDatacenterTaskStatusEnum.DATA_STATUS.NORMAL.getCode());
paramTask.setTaskNos(ids);
List<TblDatacenterTask> tasklist = tblDatacenterTaskMapper.selectListByMap(paramTask);
Map<String, TblDatacenterTask> map = tasklist.stream().distinct().collect(Collectors.toMap(x->x.getTaskNo(), x->x));
for(TblDatacenterTaskStatusVO x:list) {
x.setVersionNum(x.getVersionNum()+1);//一定加1
x.setRecToken(newRecToken);
x.setCurrStartDate(time);
x.setUpdateTime(time);
x.setDatacenterTask(map.get(x.getTaskNo()));//配置
x.setAttemptsNum(x.getAttemptsNum()-1);
redisCache.setValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, x.getTaskNo()+ x.getTaskConditions(), "1");
}
list = list.stream().filter(x->TblDatacenterTaskStatusEnum.DATA_STATUS.NORMAL.getCode()
.equals(x.getDataStatus())).collect(Collectors.toList());
}
}
//临时任务
TblDatacenterTaskVO paramtask = new TblDatacenterTaskVO();
paramtask.setDataStatus(TblDatacenterTaskStatusEnum.DATA_STATUS.NORMAL.getCode());
// List<TblDatacenterTask> tasklist = tblDatacenterTaskMapper.selectTaskList(paramtask);
// if(!tasklist.isEmpty()) {
// List<TblDatacenterTaskStatusVO> tempTasks = new ArrayList<>();
// for(TblDatacenterTask task :tasklist) {
// if(StringUtils.isBlank(task.getTempQueryJson())) {
// continue;
// }
// String[] querys = task.getTempQueryJson().split(",|");
// boolean flag = false;
// for(String qx:querys) {
// if(StringUtils.isBlank(qx)) {
// continue;
// }
// //转换时间
// Date date = SyncDatacenterUtils.convertStr2Date(qx.trim());
// Map<String,String> datemap = SyncDatacenterUtils.convertQueryDateParam(date, task.getDefaultQueryJson());
// TblDatacenterTaskStatusVO taskStatus = SyncDatacenterUtils.createTaskStatusVO(time, task, datemap);
// this.doCreateDatacenterTaskStatus(taskStatus);
// tempTasks.add(taskStatus);
// flag = true;
// }
// if(flag) {//需要立即清空
// TblDatacenterTaskVO taskvo = new TblDatacenterTaskVO();
// taskvo.setTaskNo(task.getTaskNo());
// taskvo.setLastRunDate(time);
// taskvo.setUpdateTime(time);
// tblDatacenterTaskMapper.updateClearTempQueryJson(taskvo);
// }
// }
// if(!tempTasks.isEmpty()) {
// list.addAll(tempTasks);
// }
//
// //查询任务是否有缺失,自动补充(日期最多的一条来计算间隔)
//// List<TblDatacenterTaskStatusVO> waitTasks = new ArrayList<>();
//// if(!waitTasks.isEmpty()) {
//// list.addAll(waitTasks);
//// }
// }
List<TblDatacenterTask> tasklist = getDatacenterTaskNormal();
if(!tasklist.isEmpty()) {
List<TblDatacenterTaskStatusVO> tempTasks = new ArrayList<>();
for(TblDatacenterTask task :tasklist) {
if(StringUtils.isBlank(task.getTempQueryJson()) || !TblDatacenterTaskStatusEnum.checkIsSupportTaskType(task.getTaskType())) {
continue;
}
try{
//需要立即清空
TblDatacenterTaskVO taskvo = new TblDatacenterTaskVO();
taskvo.setTaskNo(task.getTaskNo());
taskvo.setLastRunDate(time);
taskvo.setUpdateTime(time);
tblDatacenterTaskMapper.updateClearTempQueryJson(taskvo);
//一般只会写一个日期,也支持本来的源数据是有空白区间的
String[] querys = task.getTempQueryJson().split(",|");
for(String qx:querys) {
if(StringUtils.isBlank(qx)) {
continue;
}
qx = qx.trim();
Date starttime = null;
Date endtime = time;
if(!qx.contains("--")) {
Matcher m = SyncDatacenterConstants.RULE_DATETIME.matcher(qx);
if(m.find()) {
starttime = SyncDatacenterUtils.convertStr2Date(qx);
}
}else {
String[] qxarr = qx.split("--");
String tm1 = qxarr[0].trim();
String tm2 = qxarr[1].trim();
if(StringUtils.isNotBlank(tm1) && StringUtils.isNotBlank(tm2)) {
Matcher m = SyncDatacenterConstants.RULE_DATETIME.matcher(tm1);
if(m.find()) {
starttime = SyncDatacenterUtils.convertStr2Date(tm1);
}
m = SyncDatacenterConstants.RULE_DATETIME.matcher(tm2);
if(m.find()) {
endtime = SyncDatacenterUtils.convertStr2Date(tm2);
if(endtime.getTime()>time.getTime()) {//超过则覆盖
endtime = time;
}
}
}
}
if(starttime!=null && endtime!=null && endtime.getTime()>=starttime.getTime()) {//大于才有可能补充需要
task.setUpdateTime(time);
boolean flag = doAppendMissingDatacenterTaskStatus(tempTasks,task,starttime,endtime,true);
if(!flag) {
LOGGER.info("doRestartTaskStatusForError error time>>taskNo={},time={}",task.getTaskNo(),qx);
}
}else {
LOGGER.info("doRestartTaskStatusForError error time>>taskNo={},time={}",task.getTaskNo(),qx);
}
}
}catch (Exception e) {
// 忽略错误
LOGGER.info("doRestartTaskStatusForError error>>taskNo={}, tempQueryJson={}",task.getTaskNo(),task.getTempQueryJson());
}
}
if(!tempTasks.isEmpty()) {
list.addAll(tempTasks);
}
}
return list;
}
private boolean doAppendMissingDatacenterTaskStatus(List<TblDatacenterTaskStatusVO> tempTasks, TblDatacenterTask task
,Date starttime,Date endtime,boolean runFlag) throws Exception {
Date time = task.getUpdateTime();
List<Date> dtlist = new ArrayList<>();
//根据cron表达式和任务类型进行计算得出所有日期
String taskType= task.getTaskType();
String[] crons = task.getScheduledCron().split(",|");
for(String cronx:crons) {
if(StringUtils.isBlank(cronx)) {
continue;
}
cronx = cronx.trim().replaceAll(" +", " ");
String[] arrs = cronx.split(" ");
if(arrs.length != 6) {
continue;
}
//如下类型与时间有关的必须是整点(否则时间会漂移)
LocalDateTime currentDate = SyncDatacenterUtils.handleDateByTaskType(arrs, taskType, endtime);
Date newdate = Date.from(currentDate.atZone(ZoneId.of(Constants.DEFAULT_TIMEZONE)).toInstant());
dtlist.add(newdate);
while(newdate.getTime() >= starttime.getTime()) {
currentDate = SyncDatacenterUtils.convertDateMinus(currentDate,TblDatacenterTaskStatusEnum.TASK_TYPE.valueOf(taskType).getTime());
newdate = Date.from(currentDate.atZone(ZoneId.of(Constants.DEFAULT_TIMEZONE)).toInstant());
dtlist.add(newdate);
}
}
if(dtlist.isEmpty()) {
return false;
}
dtlist.sort((d1,d2)->{
return d1.compareTo(d2);
});
dtlist = dtlist.stream().distinct().collect(Collectors.toList());
boolean flag = false;
for(Date date:dtlist) {
Map<String,String> datemap = SyncDatacenterUtils.convertQueryDateParam(date, task.getDefaultQueryJson());
// System.out.println(datemap.get("dateTime"));
TblDatacenterTaskStatusVO taskStatus = SyncDatacenterUtils.createTaskStatusVO(time, task, datemap);
if(!runFlag) {//这种情况默认不运行,等后面的任务来检测
taskStatus.setAttemptsNum(taskStatus.getAttemptsNum()>0?taskStatus.getAttemptsNum():1);
taskStatus.setBusStatus(TblDatacenterTaskStatusEnum.BUS_STATUS.UNFINISH.getCode());
}
int num = this.doCreateDatacenterTaskStatus(taskStatus);
if(num>0) {
if(runFlag) {
redisCache.setValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, taskStatus.getTaskNo()+taskStatus.getTaskConditions(), "1");
}
tempTasks.add(taskStatus);
flag =true;
}
}
return flag;
}
@Override
public void doCreateMissingDatacenterTaskStatus() throws Exception {
List<TblDatacenterTaskStatusVO> tempTasks = new ArrayList<>();
List<TblDatacenterTask> tasklist = getDatacenterTaskNormal();
if(tasklist.isEmpty()) {
return;
}
Map<String, TblDatacenterTask> map = tasklist.stream().collect(Collectors.toMap(x->x.getTaskNo(), x->x));
BaseTime timeVO = BaseManagerUtils.getSystemTime();
//查询任务是否有缺失,根据任务条件往前推算一次即可
TblDatacenterTaskStatusVO param = new TblDatacenterTaskStatusVO();
param.setCreateTime(Date.from(timeVO.getUtcTime().minus(370, ChronoUnit.DAYS)));//1年5天,正常应该没有跨两年的任务吧
List<TblDatacenterTaskStatusVO> statuslist = tblDatacenterTaskStatusMapper.selectLastTaskTimeExist(param);
for(TblDatacenterTaskStatusVO stax:statuslist) {
TblDatacenterTask task = map.get(stax.getTaskNo());
String taskType= task.getTaskType();
if(!TblDatacenterTaskStatusEnum.checkIsSupportTaskType(taskType)){
continue;
}
timeVO = BaseManagerUtils.getSystemTime();//获取实时时间
task.setUpdateTime(timeVO.getDate());
String[] crons = task.getScheduledCron().split(",|");
boolean flag = false;
Date starttime2=null;
Date endtime2=null;
for(String cronx:crons) {
if(StringUtils.isBlank(cronx)) {
continue;
}
cronx = cronx.trim().replaceAll(" +", " ");
String[] arrs = cronx.split(" ");
if(arrs.length != 6) {
continue;
}
Date starttime=stax.getCreateTime();
Date endtime=timeVO.getDate();
//如下类型与时间有关的必须是整点(否则时间会漂移)
LocalDateTime currentDate = SyncDatacenterUtils.handleDateByTaskType(arrs, taskType, endtime);
Date newdate = Date.from(currentDate.atZone(ZoneId.of(Constants.DEFAULT_TIMEZONE)).toInstant());
//取得最大的日期和当前日期比较
Map<String,String> maxdatemap = SyncDatacenterUtils.convertQueryDateParam(starttime, task.getDefaultQueryJson());
TblDatacenterTaskStatusVO maxtaskStatus = SyncDatacenterUtils.createTaskStatusVO(endtime, task, maxdatemap);
Map<String,String> datemap = SyncDatacenterUtils.convertQueryDateParam(newdate, task.getDefaultQueryJson());
TblDatacenterTaskStatusVO taskStatus = SyncDatacenterUtils.createTaskStatusVO(newdate, task, datemap);
if(maxtaskStatus.getTaskConditions().equals(taskStatus.getTaskConditions())) {
flag = true;
starttime2 = null;
starttime2 = null;
}else if(starttime2==null && endtime2==null) {
starttime2 = starttime;
endtime2 = endtime;
}
}
//有缺失的则把缺失的补上
if(!flag) {
doAppendMissingDatacenterTaskStatus(tempTasks ,task, starttime2, endtime2, false);
}
}
}
// public static void main(String[] args) throws Exception {
// TblDatacenterTask task=new TblDatacenterTask();
// task.setDefaultQueryJson("1d");
// task.setTaskType("1");
// task.setScheduledCron("0 0 1 * * *");
//// task.setDefaultQueryJson("30m");
//// task.setTaskType("2");
//// task.setScheduledCron("0 10 * * * *,0 40 * * * * ");
// SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// Date time=dateFormat.parse("2024-08-20 00:20:00");
// Date starttime=dateFormat.parse("2024-08-19 00:20:00");
// Date endtime=dateFormat.parse("2024-08-19 00:20:00");
//// doAppendMissingDatacenterTaskStatus(null, task, time, starttime, endtime);
// Instant nowtime= Instant.now();
//// System.out.println(nowtime.minus(370, ChronoUnit.DAYS));
// }
}

@ -0,0 +1,55 @@
package com.jiuyv.sptccc.agile.business.service.impl;
import java.util.HashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import com.jiuyv.sptccc.agile.api.fegin.constants.WaringType;
import com.jiuyv.sptccc.agile.api.fegin.dto.publicPhoneMsgLog.ReqSendWaringPhoneMsgDTO;
import com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.fegin.PublicPhoneMsgLogFeignApix;
/**
*
*/
@Component
public class SendSmsServer {
private static final Logger LOGGER = LoggerFactory.getLogger(SendSmsServer.class);
/** 数据中心同步任务失败消息模板 */
public static final String CODE_SJZT_SYNC_TASK_ERROR_PHONE = "sjzt_sync_task_error_phone";
@Autowired
private PublicPhoneMsgLogFeignApix publicPhoneMsgLogFeignApix;
/** 发送短信,异步忽略错误 */
@Async
public void sendMsgAsync(TblDatacenterTaskStatusVO taskStatus) {
try {
Map<String, String> msgMapParams = new HashMap<>();
msgMapParams.put("taskNo", taskStatus.getTaskNo());
msgMapParams.put("taskConditions", taskStatus.getTaskConditions());
msgMapParams.put("taskTitle", taskStatus.getDatacenterTask().getTaskTitle());
ReqSendWaringPhoneMsgDTO msgvo = new ReqSendWaringPhoneMsgDTO();
msgvo.setWaringType(WaringType.DS.getCode());
msgvo.setMsgTemplateCode(CODE_SJZT_SYNC_TASK_ERROR_PHONE);
msgvo.setMsgMapParams(msgMapParams );
msgvo.setSysType("console");//portal/console
R<Long> res = publicPhoneMsgLogFeignApix.sendWarningPhoneMsgLog(msgvo);
if(!res.isSuccess()) {
LOGGER.info("sms error>>{}",res.getMsg());//直接输出
}
} catch (Exception e) {
//忽略短信错误
}
}
}

@ -4,10 +4,12 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -22,11 +24,14 @@ import com.jiuyv.sptccc.agile.business.entity.TblDatacenterTask;
import com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO;
import com.jiuyv.sptccc.agile.business.service.IDatacenterTaskStatusService;
import com.jiuyv.sptccc.agile.common.BaseManagerUtils;
import com.jiuyv.sptccc.agile.common.constant.CacheNameConstants;
import com.jiuyv.sptccc.agile.common.constant.Constants;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import com.jiuyv.sptccc.agile.common.utils.sftp.SshClientHostPool;
import com.jiuyv.sptccc.agile.common.utils.sftp.SshClientPool;
import com.jiuyv.sptccc.agile.common.utils.sftp.model.SFTPConfig;
import com.jiuyv.sptccc.agile.framework.config.SftpConfigProperties;
import com.jiuyv.sptccc.agile.framework.config.caffeine.RedisCache;
import net.logstash.logback.encoder.org.apache.commons.lang3.StringUtils;
@ -46,6 +51,11 @@ public class SyncDatacenterService {
private Environment environment;
@Autowired
private SftpConfigProperties sftpConfigProperties;
@Autowired
private SendSmsServer sendSmsServer;
@Autowired
private RedisCache redisCache;
/** 获取数据源配置 */
public DDsProperties getDataSourceConf(String dbName) {
@ -106,6 +116,7 @@ public class SyncDatacenterService {
@Async
public void doStartTaskStatusForNormal(String taskNo) {
String key="XXX";
try {
BaseTime timeVO = BaseManagerUtils.getSystemTime();
Date time = timeVO.getDate();
@ -120,11 +131,16 @@ public class SyncDatacenterService {
if(num==0) {
return;
}
key = taskNo+taskStatus.getTaskConditions();
redisCache.setValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, key, "1");
LOGGER.info("【数据同步】doStartTaskStatusForNormal>>TaskNo={}TaskConditions={},Date={}",taskStatus.getTaskNo()
,taskStatus.getTaskConditions(),datemap.get("dateTime"));
executeOneTask(taskStatus, datemap);
}catch (Exception e) {
LOGGER.info("【数据同步】执行任务异常>>{}",taskNo,e);
}finally {
//无论对错移除缓存标志
redisCache.removeValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, key);
}
}
/**
@ -150,6 +166,7 @@ public class SyncDatacenterService {
ReaderWriterHelper readerWriterHelper=new ReaderWriterHelper();
readerWriterHelper.setSingleWriteNumber(remoteDbCfg.getSingleWriteNumber());
readerWriterHelper.setDatetime(DateFormatUtils.format(taskStatus.getUpdateTime(), Constants.FORMAT_DATETIME));
boolean readSftpFlag=StringUtils.isNotBlank(task.getRemoteFileName());//默认是读数据库
boolean writeSftpFlag=false;//默认写是数据库
@ -184,6 +201,9 @@ public class SyncDatacenterService {
datacenterTaskStatusService.doFinishDatacenterTaskStatus(taskStatus);
}else {
datacenterTaskStatusService.doUnfinishDatacenterTaskStatus(taskStatus);
if(taskStatus.getAttemptsNum()<=0) {//没有尝试次数了则认为失败了
sendSmsServer.sendMsgAsync(taskStatus);
}
}
}
}
@ -196,6 +216,10 @@ public class SyncDatacenterService {
for(List<TblDatacenterTaskStatusVO> taskStatus:list) {
BaseTime timeVO = BaseManagerUtils.getSystemTime();
executorBatchAsyncTask(timeVO,taskStatus);
for(TblDatacenterTaskStatusVO taskStatusx:taskStatus) {
//无论对错移除缓存标志
redisCache.removeValueOfCacheName(CacheNameConstants.CACHE_TASK_RUNNING, taskStatusx.getTaskNo()+taskStatusx.getTaskConditions());
}
}
}
@ -227,8 +251,11 @@ public class SyncDatacenterService {
//等待所有异步操作都完成
//allFutures.thenRun(() -> { System.out.println("All tasks have completed.");}).join();
allFutures.get();
}catch (Exception e) {
}catch (ExecutionException e) {
LOGGER.info("【数据同步】executorBatchAsyncTask error>>",e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.info("【数据同步】executorBatchAsyncTask error Interrupted>>",e);
}finally {
if (executor != null){
executor.shutdown();

@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory;
import com.jiuyv.sptccc.agile.business.mapper.ISysTimeBaseMapper;
import com.jiuyv.sptccc.agile.common.model.BaseTime;
import com.jiuyv.sptccc.agile.common.utils.SpringUtils;
import com.jiuyv.sptccc.agile.framework.SpringUtils;
/**
* 使
* @author zhouliang

@ -0,0 +1,18 @@
package com.jiuyv.sptccc.agile.common.constant;
/**
*
*
*/
public class CacheNameConstants
{
private CacheNameConstants() {
throw new IllegalStateException("Utility class");
}
/**
*
*/
public static final String CACHE_TASK_RUNNING = "cache_task_running";
}

@ -9,4 +9,13 @@ public class Constants {
/** 一个固定的密钥,用于配置文件内容加密/解密 */
public static final String SM4_SECERT_KEY = "a14751855ccb428d982c33dfa3535a57";
/** 日期转换用 */
public static final String FORMAT_DATETIME ="yyyy-MM-dd HH:mm:ss";
public static final String FORMAT_DATEDAY ="yyyy-MM-dd";
public static final String FORMAT_DATETIME_SIMPLE ="yyyyMMddHHmmss";
public static final String FORMAT_DATEDAY_SIMPLE ="yyyyMMdd";
/** 默认时区 */
public static final String DEFAULT_TIMEZONE = "+08:00";
}

@ -12,7 +12,7 @@ public final class ServiceException extends RuntimeException
/**
*
*/
private Integer code;
private String code;
/**
*
@ -50,7 +50,7 @@ public final class ServiceException extends RuntimeException
return message;
}
public Integer getCode()
public String getCode()
{
return code;
}

@ -0,0 +1,19 @@
package com.jiuyv.sptccc.agile.fegin;
import org.springframework.cloud.openfeign.FeignClient;
import com.jiuyv.sptccc.agile.api.fegin.PublicPhoneMsgLogFeignApi;
import com.jiuyv.sptccc.agile.api.fegin.fallback.PublicPhoneMsgLogFallback;
/**
* ()
* @author zhouliang
* @date 2023-04-25
*/
@FeignClient(contextId = "PublicPhoneMsgLogFeignApi", name = "${gateway.messageService}"
, fallbackFactory = PublicPhoneMsgLogFallback.class)
public interface PublicPhoneMsgLogFeignApix extends PublicPhoneMsgLogFeignApi {
}

@ -1,4 +1,4 @@
package com.jiuyv.sptccc.agile.common.utils;
package com.jiuyv.sptccc.agile.framework;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanFactoryPostProcessor;

@ -0,0 +1,43 @@
package com.jiuyv.sptccc.agile.framework.config.caffeine;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.cache.CacheManager;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.jiuyv.sptccc.agile.common.constant.CacheNameConstants;
/**
* caffeine
* @author Administrator
*/
@Configuration
public class CaffeineCacheConfig {
/**
* ,
*
* ([CacheTimestampedValue]/)
*
* @return
*/
@Bean
@Qualifier(value = "commonCacheManager")
public CacheManager commonCacheManager() {
SimpleCacheManager cacheManager = new SimpleCacheManager();
List<CaffeineCache> caches = new ArrayList<>();
caches.add(new CaffeineCache(CacheNameConstants.CACHE_TASK_RUNNING,
Caffeine.newBuilder().expireAfterAccess(3, TimeUnit.DAYS).maximumSize(5000).build()));
cacheManager.setCaches(caches);
return cacheManager;
}
}

@ -0,0 +1,74 @@
package com.jiuyv.sptccc.agile.framework.config.caffeine;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.cache.Cache;
import org.springframework.cache.Cache.ValueWrapper;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Component;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
/**
* Caffeine
*
* @author admin
**/
@SuppressWarnings(value = { "unchecked"})
@Component
public class RedisCache
{
@Autowired
@Qualifier(value = "commonCacheManager")
private CacheManager cacheManager;
/**
*
* @param <T>
* @param cacheName
* @param key
* @param value
*/
public <T> void setValueOfCacheName(final String cacheName, final String key, final T value)
{
Cache cache = cacheManager.getCache(cacheName);
if(cache==null) {
throw new ServiceException("Cache does not exist!");
}
cache.put(key, value);
}
/**
*
* @param <T>
* @param cacheName
* @param key
* @param value
*/
public <T> T getValueOfCacheName(final String cacheName, final String key)
{
Cache cache = cacheManager.getCache(cacheName);
if(cache==null) {
throw new ServiceException("Cache does not exist!");
}
ValueWrapper wval = cache.get(key);
if(wval==null) {
return null;
}
return (T) wval.get();
}
/**
*
* @param cacheName
* @param key
*/
public void removeValueOfCacheName(final String cacheName, final String key)
{
Cache cache = cacheManager.getCache(cacheName);
if(cache==null) {
throw new ServiceException("Cache does not exist!");
}
cache.evict(key);
}
}

@ -24,10 +24,9 @@ public class GlobalExceptionHandler {
*
*/
@ExceptionHandler({ServiceException.class})
public R<?> handleServiceException(ServiceException e, HttpServletRequest request, HttpServletResponse response) {
public R<String> handleServiceException(ServiceException e, HttpServletRequest request, HttpServletResponse response) {
log.error(e.getMessage(), e);
Integer code = e.getCode();
String code = e.getCode();
return code!=null ? R.fail(code, e.getMessage()) : R.fail(e.getMessage());
}
@ -35,7 +34,7 @@ public class GlobalExceptionHandler {
*
*/
@ExceptionHandler(Exception.class)
public R<?> handleException(Exception e, HttpServletRequest request, HttpServletResponse response) {
public R<String> handleException(Exception e, HttpServletRequest request, HttpServletResponse response) {
response.setStatus(301);
String requestURI = request.getRequestURI();

@ -6,18 +6,12 @@ console:
clearDs: sjztHiveDs #要清理的数据源,逗号分隔
clearDay: 14 #数据源的文件保留天数,到期清除
#路由服务地址
gateway:
messageService: message-service
# 开发环境配置
server:
port: 18084 #本地
port: 18087 #本地
servlet:
# 应用的访问路径
context-path: /datacenter-sync
context-path: null
tomcat:
# tomcat的URI编码
uri-encoding: UTF-8

@ -1,6 +1,10 @@
console:
version: 1.0
#路由服务地址
gateway:
messageService: message-service
# Spring配置
spring:
application:

@ -6,8 +6,9 @@
a.task_no, a.task_title, a.task_type, a.version_num, a.rec_token, a.remote_db_name,
a.remote_table_sql, a.remote_file_name, a.remote_file_fields, a.local_db_name,
a.local_pre_sql, a.local_table, a.mapping_json, a.default_query_json, a.temp_query_json,
a.attempts_num, a.run_interval_time, a.last_run_date, a.scheduled_cron, a.remarks,
a.order_num, a.data_status, a.create_time, a.update_time, a.rsv1, a.rsv2, a.rsv3
a.attempts_num, a.run_interval_time, a.last_run_date, a.scheduled_cron,
a.error_interval_time, a.remarks, a.order_num, a.data_status, a.create_time, a.update_time,
a.rsv1, a.rsv2, a.rsv3
</sql>
<sql id="_where">
<!-- 默认必有条件 -->
@ -64,6 +65,7 @@
<if test="vo.runIntervalTime != null" >run_interval_time = #{vo.runIntervalTime},</if>
<if test="vo.lastRunDate != null" >last_run_date = #{vo.lastRunDate},</if>
<if test="vo.scheduledCron != null" >scheduled_cron = #{vo.scheduledCron},</if>
<if test="vo.errorIntervalTime != null" >error_interval_time = #{vo.errorIntervalTime},</if>
<if test="vo.remarks != null" >remarks = #{vo.remarks},</if>
<if test="vo.orderNum != null" >order_num = #{vo.orderNum},</if>
<if test="vo.dataStatus != null" >data_status = #{vo.dataStatus},</if>
@ -90,6 +92,7 @@
<where>
and data_status = #{dataStatus}
</where>
order by a.order_num DESC
</select>
<!-- 清空临时条件 -->

@ -5,8 +5,8 @@
<sql id="_select">
a.task_no, a.task_conditions, a.task_type, a.version_num, a.rec_token, a.pre_start_date,
a.pre_end_date, a.pre_total_time, a.curr_start_date, a.load_file, a.load_time,
a.attempts_num, a.run_interval_time, a.read_count, a.bus_status, a.data_status,
a.create_time, a.update_time, a.rsv1, a.rsv2, a.rsv3
a.attempts_num, a.run_interval_time, a.error_interval_time, a.read_count, a.bus_status,
a.data_status, a.create_time, a.update_time, a.rsv1, a.rsv2, a.rsv3
</sql>
<sql id="_where">
<!-- 默认必有条件 -->
@ -44,6 +44,7 @@
<if test="loadTime != null" >load_time,</if>
<if test="attemptsNum != null" >attempts_num,</if>
<if test="runIntervalTime != null" >run_interval_time,</if>
<if test="errorIntervalTime != null" >error_interval_time,</if>
<if test="readCount != null" >read_count,</if>
<if test="busStatus != null" >bus_status,</if>
<if test="dataStatus != null" >data_status,</if>
@ -67,6 +68,7 @@
<if test="loadTime != null" >#{loadTime},</if>
<if test="attemptsNum != null" >#{attemptsNum},</if>
<if test="runIntervalTime != null" >#{runIntervalTime},</if>
<if test="errorIntervalTime != null" >#{errorIntervalTime},</if>
<if test="readCount != null" >#{readCount},</if>
<if test="busStatus != null" >#{busStatus},</if>
<if test="dataStatus != null" >#{dataStatus},</if>
@ -94,6 +96,7 @@
<if test="vo.loadTime != null" >load_time = #{vo.loadTime},</if>
<if test="vo.attemptsNum != null" >attempts_num = #{vo.attemptsNum},</if>
<if test="vo.runIntervalTime != null" >run_interval_time = #{vo.runIntervalTime},</if>
<if test="vo.errorIntervalTime != null" >error_interval_time = #{vo.errorIntervalTime},</if>
<if test="vo.readCount != null" >read_count = #{vo.readCount},</if>
<if test="vo.busStatus != null" >bus_status = #{vo.busStatus},</if>
<if test="vo.dataStatus != null" >data_status = #{vo.dataStatus},</if>
@ -133,6 +136,17 @@
</where>
</select>
<!-- 查询运行任务集合 -->
<select id="selectRunningTaskList" resultType="com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO" parameterType="com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO">
select
a.task_no, a.task_conditions, a.curr_start_date, a.error_interval_time
from tbl_datacenter_task_status a
<where>
AND a.data_status = '00'
AND a.bus_status = #{busStatus}
</where>
</select>
<!-- 更新错误待重启任务记录 -->
<update id="updateErrorByMap">
update tbl_datacenter_task_status
@ -162,14 +176,11 @@
update_time = #{vo.updateTime},
</set>
<where>
attempts_num > 0
AND data_status = '00'
<choose>
<when test="map.overdateType=='m'">
AND curr_start_date &lt; NOW() - INTERVAL '${map.overdate} minutes'
</when>
<when test="map.overdateType=='h'">
AND curr_start_date &lt; NOW() - INTERVAL '${map.overdate} hour'
<when test="map.taskNo!=null">
and task_no = #{map.taskNo}
and task_conditions = #{map.taskConditions}
</when>
<otherwise>
AND curr_start_date &lt; #{map.currStartDate}
@ -179,4 +190,17 @@
</where>
</update>
<!-- 查询最近运行的任务集合 -->
<select id="selectLastTaskTimeExist" resultType="com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO" parameterType="com.jiuyv.sptccc.agile.business.entity.vo.TblDatacenterTaskStatusVO">
select
a.task_no,
MAX(a.create_time) AS create_time
from tbl_datacenter_task_status a
INNER JOIN tbl_datacenter_task b ON b.task_no = a.task_no
where
b.data_status='00'
AND a.create_time >= #{createTime}
GROUP BY a.task_no
</select>
</mapper>

@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-datacenter</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.3-SNAPSHOT</version>
<packaging>pom</packaging>
<description />
@ -33,8 +33,8 @@
<jsch.version>0.1.55</jsch.version>
<spring-cloud.version>2021.0.5</spring-cloud.version>
<agile-common.version>0.1.10</agile-common.version>
<agile-datacenter-api.version>0.0.1-SNAPSHOT</agile-datacenter-api.version>
<agile-common.version>0.1.13</agile-common.version>
<agile-datacenter-api.version>0.0.3-SNAPSHOT</agile-datacenter-api.version>
<agile-mobile-message-api.version>0.1.7</agile-mobile-message-api.version>
</properties>
@ -126,10 +126,10 @@
</distributionManagement>
<scm>
<connection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/datacenter/src/trunk/agile-datacenter
<connection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile-datacenter/src/trunk/agile-datacenter
</connection>
<developerConnection>
scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/datacenter/src/trunk/agile-datacenter
scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile-datacenter/src/trunk/agile-datacenter
</developerConnection>
</scm>

@ -140,7 +140,6 @@
<name>Internal Releases</name>
<url>http://172.16.12.11:8082/repository/maven-releases/</url>
</repository>
<snapshotRepository>
<id>nexus-snapshots</id>
<name>Internal Snapshots</name>
@ -159,36 +158,14 @@
</snapshots>
</repository>
<repository>
<id>jboss</id>
<name>jboss</name>
<url>http://repository.jboss.org/maven2/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>geotools</id>
<name>geotools</name>
<url>http://maven.geotools.fr/repository/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>jahia</id>
<name>jahia</name>
<url>http://maven.jahia.org/maven2/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>aliyun-releases</id>
<name>Internal Releases</name>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
</repository>
<repository>
<id>vars</id>
<name>vars</name>
<url>http://vars.sourceforge.net/maven2/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>spring-milestone-releases</id>
<name>Internal Releases</name>
<url>https://repo.spring.io/milestone</url>
</repository>
</repositories>
@ -198,6 +175,11 @@
<name>jiuyv Plugin Repository</name>
<url>http://172.16.12.11:8082/repository/maven-public/</url>
</pluginRepository>
<pluginRepository>
<id>aliyun-releases</id>
<name>Internal Releases</name>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
</pluginRepository>
<pluginRepository>
<id>central</id>
<name>Maven Plugin Repository</name>

@ -0,0 +1,101 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>agile-dms</artifactId>
<groupId>com.jiuyv.sptcc.agile</groupId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>agile-dms-api</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
<version>${agile-common.version}</version>
</dependency>
<!-- SpringWeb模块 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<!-- 自定义验证注解 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<!--常用工具类 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- excel工具 -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/easyexcel -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>3.1.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,51 @@
//package com.jiuyv.sptcc.agile.api.data;
//
//
//import com.jiuyv.sptcc.agile.dms.data.domain.dto.SqlConsoleDto;
//import com.jiuyv.sptcc.agile.dms.data.domain.vo.SqlConsoleConnNumVo;
//import com.jiuyv.sptcc.agile.dms.data.domain.vo.SqlConsoleResVo;
//import com.jiuyv.sptccc.agile.common.core.domain.R;
//import org.springframework.validation.annotation.Validated;
//import org.springframework.web.bind.annotation.GetMapping;
//import org.springframework.web.bind.annotation.PostMapping;
//import org.springframework.web.bind.annotation.RequestBody;
//
//public interface SqlConsoleApi {
//
// String PREFIX_PATH = "/public/agile-data";
//
// /**
// * 运行SQL
// *
// * @param sqlConsoleDto
// * @return
// */
// @PostMapping(PREFIX_PATH+"/run")
// R<SqlConsoleResVo> sqlRun(@RequestBody @Validated SqlConsoleDto sqlConsoleDto);
//
// /**
// * 停止SQL执行
// *
// * @param sqlConsoleDto SQL停止参数
// * @return 成功响应的R对象
// */
// @PostMapping("/stop")
// R<Object> stopSql(@RequestBody SqlConsoleDto sqlConsoleDto);
//
// /**
// * 导出SQL查询结果
// *
// * @param sqlConsoleDto 导出参数
// * @return 成功响应的R对象
// */
// @PostMapping("/export")
// R<Object> exportQueryResult(@Validated @RequestBody SqlConsoleDto sqlConsoleDto);
//
// /**
// * 获取数据库连接池连接数
// *
// * @return 包含连接数信息的R对象
// */
// @GetMapping("/connectNum")
// R<SqlConsoleConnNumVo> getConnectNum();
//}

@ -0,0 +1,134 @@
package com.jiuyv.sptcc.agile.dms.common.core.domain;
import com.fasterxml.jackson.annotation.JsonFormat;
import java.io.Serializable;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* Entity
*
* @author admin
*/
public class BaseEntity implements Serializable {
private static final long serialVersionUID = 1L;
/** 创建者姓名 */
private String createByName;
/** 创建时间 */
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/** 创建者id(或账户) */
private String createBy;
/** 更新者姓名 */
private String updateByName;
/** 更新者id(或账户) */
private String updateBy;
/** 更新时间 */
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/** 搜索值 */
private String searchValue;
/** 备注 */
private String remark;
/** 请求参数 */
private Map<String, Object> params;
/**
* @return the createByName
*/
public String getCreateByName() {
return createByName;
}
/**
* @param createByName the createByName to set
*/
public void setCreateByName(String createByName) {
this.createByName = createByName;
}
/**
* @return the updateByName
*/
public String getUpdateByName() {
return updateByName;
}
/**
* @param updateByName the updateByName to set
*/
public void setUpdateByName(String updateByName) {
this.updateByName = updateByName;
}
public String getSearchValue() {
return searchValue == null ? "" : searchValue;
}
public void setSearchValue(String searchValue) {
this.searchValue = searchValue;
}
public String getCreateBy() {
return createBy == null ? "" : createBy;
}
public void setCreateBy(String createBy) {
this.createBy = createBy;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public String getUpdateBy() {
return updateBy == null ? "" : updateBy;
}
public void setUpdateBy(String updateBy) {
this.updateBy = updateBy;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public String getRemark() {
return remark == null ? "" : remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public Map<String, Object> getParams() {
if (params == null) {
params = new HashMap<>();
}
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
}

@ -0,0 +1,558 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-dms</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>agile-dms-console</artifactId>
<dependencies>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-mobile-message-api</artifactId>
<version>${agile-mobile-message-api.version}</version>
<exclusions>
<exclusion>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-data-api</artifactId>
<version>${agile-data-api.version}</version>
<exclusions>
<exclusion>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--因为devtools的问题所以agile-common必须和里面版本统一.
使用最新的API包里面的即可-->
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
<version>${agile-common.version}</version>
</dependency>
<!-- openfeign还是申明下把 -->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<dependency>
<groupId>io.github.openfeign</groupId>
<artifactId>feign-okhttp</artifactId>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp3.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-loadbalancer</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-bootstrap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<!-- 自定义验证注解 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<!-- SpringWeb模块 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- spring security 安全认证 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-sleuth</artifactId>
</dependency>
<!-- 解析客户端操作系统、浏览器等 -->
<dependency>
<groupId>eu.bitwalker</groupId>
<artifactId>UserAgentUtils</artifactId>
<version>${bitwalker.version}</version>
</dependency>
<!-- SpringBoot集成mybatis框架 -->
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${mybatis-spring-boot.version}</version>
</dependency>
<!-- pagehelper 分页插件 -->
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper-spring-boot-starter</artifactId>
<version>${pagehelper.boot.version}</version>
</dependency>
<!-- 获取系统信息 -->
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
<version>${oshi.version}</version>
</dependency>
<!-- io常用工具类 -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons.io.version}</version>
</dependency>
<!-- 文件上传工具类 -->
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>${commons.fileupload.version}</version>
</dependency>
<!-- excel工具 -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<!-- velocity代码生成使用模板 -->
<dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity-engine-core</artifactId>
<version>${velocity.version}</version>
</dependency>
<!-- collections工具类 -->
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>${commons.collections.version}</version>
</dependency>
<!-- Token生成与解析 -->
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId>
<version>${jwt.version}</version>
</dependency>
<!-- 验证码 -->
<dependency>
<groupId>com.github.penggle</groupId>
<artifactId>kaptcha</artifactId>
<version>${kaptcha.version}</version>
<exclusions>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
</exclusions>
</dependency>
<!-- spring-boot-devtools -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<optional>true</optional> <!-- 表示依赖不会传递 -->
</dependency>
<!-- json logstash encoder -->
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>6.4</version>
</dependency>
<!-- Spring框架基本的核心工具 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</dependency>
<!--常用工具类 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- yml解析器 -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<!-- Jaxb -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<!-- pool 对象池 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
</dependency>
<!-- servlet包 -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/easyexcel -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>3.1.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.aspectj/aspectjweaver -->
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<!-- SpringBoot 拦截器 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-quartz</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<!-- <dependency>
<groupId>com.baomidou</groupId>
<artifactId>dynamic-datasource-spring-boot-starter</artifactId>
<version>3.5.1</version>
</dependency>-->
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.55</version>
</dependency>
<dependency>
<groupId>org.apache.axis</groupId>
<artifactId>axis</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.70</version>
</dependency>
<!-- https://mvnrepository.com/artifact/net.sf.opencsv/opencsv -->
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<!--整合prometheus-->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<!-- Hive,要测试的时候注释掉devtools-->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<artifactId>*</artifactId>
<groupId>*</groupId>
</exclusion>
</exclusions>
</dependency>
<!-- Hive-JDBC里面有 ,要测试的时候注释掉devtools -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-upgrade-acid</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-llap-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.orbit</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
</exclusion>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
<exclusion>
<groupId>org.pentaho</groupId>
<artifactId>pentaho-aggdesigner-algorithm</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.vaadin.external.google</groupId>
<artifactId>android-json</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-contract-wiremock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.3</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.github.javaparser</groupId>
<artifactId>javaparser-core</artifactId>
<version>3.14.16</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.10</version>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<!-- <start-class>com.jiuyv.sptcc.market.order.OrderBatchApplication</start-class>-->
</configuration>
</execution>
</executions>
<configuration>
<fork>true</fork> <!-- 如果没有该配置devtools不会生效 -->
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<compilerArguments>
<extdirs>lib</extdirs>
</compilerArguments>
</configuration>
</plugin>
<!--前后端合并-->
<!--clean插件 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>${maven.clean.version}</version>
<configuration>
<filesets>
<fileset>
<directory>src/main/resources/static</directory>
</fileset>
</filesets>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven.resource.version}</version>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,42 @@
package com.jiuyv.sptcc.agile.dms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration;
import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.ImportResource;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.transaction.annotation.EnableTransactionManagement;
/**
*
*
* @author admin
*/
@EnableFeignClients
@EnableEurekaClient
@EnableScheduling
@EnableTransactionManagement
@SpringBootApplication(exclude = {GroovyTemplateAutoConfiguration.class})
@ImportResource({ "classpath:spring/applicationContext.xml" })//事务控制
public class SptccConsoleApplication {
private static final Logger LOGGER = LoggerFactory.getLogger(SptccConsoleApplication.class);
public static void main(String[] args) {
SpringApplication.run(SptccConsoleApplication.class, args);
LOGGER.info("(♥◠‿◠)ノ゙ 模块启动成功゙ \n"+
" ___ ___ ___ \n"+
" |\\ \\ |\\ \\ / /| \n"+
" \\ \\ \\ \\ \\ \\/ / / \n"+
" __ \\ \\ \\ \\ \\ / / \n"+
" |\\ \\\\_\\ \\ \\/ / / \n"+
" \\ \\________\\ __/ / / \n"+
" \\|________| |\\___/ / \n"+
" \\|___|/ \n"
);
}
}

@ -0,0 +1,16 @@
package com.jiuyv.sptcc.agile.dms;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
/**
* web
*
* @author admin
*/
public class SptccConsoleServletInitializer extends SpringBootServletInitializer {
@Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
return application.sources(SptccConsoleApplication.class);
}
}

@ -0,0 +1,19 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* 访
*
* @author admin
*/
@Target({ ElementType.METHOD, ElementType.TYPE })
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface Anonymous
{
}

@ -0,0 +1,28 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
*
*
* @author admin
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface DataScope
{
/**
*
*/
String deptAlias() default "";
/**
*
*/
String userAlias() default "";
}

@ -0,0 +1,29 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.jiuyv.sptcc.agile.dms.common.enums.DataSourceType;
/**
*
*
*
*
* @author admin
*/
@Target({ ElementType.METHOD, ElementType.TYPE })
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
public @interface DataSource
{
/**
*
*/
DataSourceType value() default DataSourceType.MASTER;
}

@ -0,0 +1,24 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import javax.validation.Constraint;
import javax.validation.Payload;
import com.jiuyv.sptcc.agile.dms.common.utils.bean.EnumStringValidator;
/**
*
*/
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
@Constraint(validatedBy = EnumStringValidator.class)
public @interface EnumCheckValue {
String value();
String message() default "值不在枚举范围内";
Class<?>[] groups() default {};
Class<? extends Payload>[] payload() default {};
}

@ -0,0 +1,184 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.math.BigDecimal;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.apache.poi.ss.usermodel.IndexedColors;
import com.jiuyv.sptcc.agile.dms.common.utils.poi.ExcelHandlerAdapter;
/**
* Excel
*
* @author admin
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface Excel
{
/**
* excel
*/
int sort() default Integer.MAX_VALUE;
/**
* Excel.
*/
String name() default "";
/**
* , : yyyy-MM-dd
*/
String dateFormat() default "";
/**
* type (: sys_user_sex)
*/
String dictType() default "";
/**
* (: 0=,1=,2=)
*/
String readConverterExp() default "";
/**
*
*/
String separator() default ",";
/**
* BigDecimal :-1(BigDecimal)
*/
int scale() default -1;
/**
* BigDecimal :BigDecimal.ROUND_HALF_EVEN
*/
int roundingMode() default BigDecimal.ROUND_HALF_EVEN;
/**
* excel
*/
double height() default 14;
/**
* excel
*/
double width() default 16;
/**
* ,% 90 90%
*/
String suffix() default "";
/**
* ,
*/
String defaultValue() default "";
/**
*
*/
String prompt() default "";
/**
* .
*/
String[] combo() default {};
/**
* ,:,.
*/
boolean isExport() default true;
/**
* ,,
*/
String targetAttr() default "";
/**
* ,
*/
boolean isStatistics() default false;
/**
* 0 1
*/
ColumnType cellType() default ColumnType.STRING;
/**
*
*/
IndexedColors color() default IndexedColors.BLACK;
/**
*
*/
HorizontalAlignment align() default HorizontalAlignment.CENTER;
/**
*
*/
Class<?> handler() default ExcelHandlerAdapter.class;
/**
*
*/
String[] args() default {};
enum Align
{
AUTO(0), LEFT(1), CENTER(2), RIGHT(3);
private final int value;
Align(int value)
{
this.value = value;
}
int value()
{
return this.value;
}
}
/**
* 012
*/
Type type() default Type.ALL;
enum Type
{
ALL(0), EXPORT(1), IMPORT(2);
private final int value;
Type(int value)
{
this.value = value;
}
int value()
{
return this.value;
}
}
enum ColumnType
{
NUMERIC(0), STRING(1), IMAGE(2);
private final int value;
ColumnType(int value)
{
this.value = value;
}
int value()
{
return this.value;
}
}
}

@ -0,0 +1,18 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Excel
*
* @author admin
*/
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Excels
{
Excel[] value();
}

@ -0,0 +1,47 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.jiuyv.sptcc.agile.dms.common.enums.BusinessType;
import com.jiuyv.sptcc.agile.dms.common.enums.OperatorType;
/**
*
*
* @author admin
*
*/
@Target({ ElementType.PARAMETER, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface Log
{
/**
*
*/
String title() default "";
/**
*
*/
BusinessType businessType() default BusinessType.OTHER;
/**
*
*/
OperatorType operatorType() default OperatorType.MANAGE;
/**
*
*/
boolean isSaveRequestData() default true;
/**
*
*/
boolean isSaveResponseData() default true;
}

@ -0,0 +1,19 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* 使
* @author zhouliang
*
*/
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface LogIgnore {
}

@ -0,0 +1,19 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
*
* 使
* @author zhouliang
*
*/
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface LogSimpleResult {
}

@ -0,0 +1,23 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
*
*
* @author admin
*
*/
@Inherited
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface NoRepeatSubmit
{
}

@ -0,0 +1,41 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.jiuyv.sptcc.agile.dms.common.constant.CacheConstants;
import com.jiuyv.sptcc.agile.dms.common.enums.LimitType;
/**
*
*
* @author admin
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface RateLimiter
{
/**
* key
*/
String key() default CacheConstants.RATE_LIMIT_KEY;
/**
* ,
*/
int time() default 60;
/**
*
*/
int count() default 100;
/**
*
*/
LimitType limitType() default LimitType.DEFAULT;
}

@ -0,0 +1,31 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
*
*
* @author admin
*
*/
@Inherited
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface RepeatSubmit
{
/**
* (ms)
*/
int interval() default 5000;
/**
*
*/
String message() default "不允许重复提交,请稍候再试";
}

@ -0,0 +1,23 @@
package com.jiuyv.sptcc.agile.dms.common.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* *便
* password
*
* @author zhouliang
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface SensitiveData {
char defaultMark() default '*';//默认使用*
int firstLength() default 3;//保留前三位
int endLength() default 3;//保留后三位
}

@ -0,0 +1,265 @@
package com.jiuyv.sptcc.agile.dms.common.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
*
*
* @author admin
*/
@Component
@ConfigurationProperties(prefix = "console")
public class ConsoleConfig {
/** 项目名称 */
private String name;
/** 版本 */
private String version;
/** 版权年份 */
private String copyrightYear;
/** 实例演示开关 */
private boolean demoEnabled;
/** 上传路径 */
private static String profile;
/** 获取地址开关 */
private static boolean addressEnabled;
/** 验证码类型 */
private static String captchaType;
private static String encryptKey;
private static String encryptIv;
private static String encryptDbIv;
/** 测试类标志,为了屏蔽一些远程结果 */
private static boolean testFlag=false;
//密码过期时间,登陆前修改密码
private static int passwordExpireDays=90;
//密码快期天数提醒
private static int passwordExpireReminderDays=7;
//密码过期后,修改错误次数
private static int passwordErrorNum=3;
//登陆错误/密码修改错误锁定时间(分)
private static int loginErrorLockTime=30;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getCopyrightYear() {
return copyrightYear;
}
public void setCopyrightYear(String copyrightYear) {
this.copyrightYear = copyrightYear;
}
public boolean isDemoEnabled() {
return demoEnabled;
}
public void setDemoEnabled(boolean demoEnabled) {
this.demoEnabled = demoEnabled;
}
public static String getProfile() {
return profile;
}
public void setProfile(String profile) {
setProfile2(profile);
}
public static void setProfile2(String profile) {
ConsoleConfig.profile = profile;
}
public static boolean isAddressEnabled() {
return addressEnabled;
}
public void setAddressEnabled(boolean addressEnabled) {
setAddressEnabled2(addressEnabled);
}
public static void setAddressEnabled2(boolean addressEnabled) {
ConsoleConfig.addressEnabled = addressEnabled;
}
public static String getCaptchaType() {
return captchaType;
}
public void setCaptchaType(String captchaType) {
setCaptchaType2(captchaType);
}
public static void setCaptchaType2(String captchaType) {
ConsoleConfig.captchaType = captchaType;
}
public static String getEncryptKey() {
return encryptKey;
}
public void setEncryptKey(String encryptKey) {
setEncryptKey2(encryptKey);
}
public static void setEncryptKey2(String encryptKey) {
ConsoleConfig.encryptKey = encryptKey;
}
public static String getEncryptIv() {
return encryptIv;
}
public void setEncryptIv(String encryptIv) {
setEncryptIv2(encryptIv);
}
public static void setEncryptIv2(String encryptIv) {
ConsoleConfig.encryptIv = encryptIv;
}
public static String getEncryptDbIv() {
return encryptDbIv;
}
public void setEncryptDbIv(String encryptDbIv) {
setEncryptDbIv2(encryptDbIv);
}
public static void setEncryptDbIv2(String encryptDbIv) {
ConsoleConfig.encryptDbIv = encryptDbIv;
}
/**
*
*/
public static String getImportPath() {
return getProfile() + "/import";
}
/**
*
*/
public static String getDownloadPath() {
return getProfile() + "/download/";
}
/**
*
*/
public static String getUploadPath() {
return getProfile() + "/upload";
}
/**
*
*/
public static String getUploadPath2() {
return getProfile();
}
public static boolean getTestFlag() {
return testFlag;
}
public void setTestFlag(boolean testFlag) {
setTestFlag2(testFlag);
}
public static void setTestFlag2(boolean testFlag) {
ConsoleConfig.testFlag = testFlag;
}
/**
* @return the passwordExpireDays
*/
public static int getPasswordExpireDays() {
return passwordExpireDays;
}
/**
* @param passwordExpireDays the passwordExpireDays to set
*/
public static void setPasswordExpireDays(int passwordExpireDays) {
setPasswordExpireDays2(passwordExpireDays);
}
public static void setPasswordExpireDays2(int passwordExpireDays) {
ConsoleConfig.passwordExpireDays = passwordExpireDays;
}
/**
* @return the passwordErrorNum
*/
public static int getPasswordErrorNum() {
return passwordErrorNum;
}
/**
* @param passwordErrorNum the passwordErrorNum to set
*/
public static void setPasswordErrorNum(int passwordErrorNum) {
setPasswordErrorNum2(passwordErrorNum);
}
public static void setPasswordErrorNum2(int passwordErrorNum) {
ConsoleConfig.passwordErrorNum = passwordErrorNum;
}
/**
* @return the loginErrorLockTime
*/
public static int getLoginErrorLockTime() {
return loginErrorLockTime;
}
/**
* @param loginErrorLockTime the loginErrorLockTime to set
*/
public static void setLoginErrorLockTime(int loginErrorLockTime) {
setLoginErrorLockTime2(loginErrorLockTime);
}
public static void setLoginErrorLockTime2(int loginErrorLockTime) {
ConsoleConfig.loginErrorLockTime = loginErrorLockTime;
}
/**
* @return the passwordExpireReminderDays
*/
public static int getPasswordExpireReminderDays() {
return passwordExpireReminderDays;
}
/**
* @param passwordExpireReminderDays the passwordExpireReminderDays to set
*/
public static void setPasswordExpireReminderDays(int passwordExpireReminderDays) {
setPasswordExpireReminderDays2(passwordExpireReminderDays);
}
public static void setPasswordExpireReminderDays2(int passwordExpireReminderDays) {
ConsoleConfig.passwordExpireReminderDays = passwordExpireReminderDays;
}
}

@ -0,0 +1,169 @@
package com.jiuyv.sptcc.agile.dms.common.config;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.Charsets;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import com.jiuyv.sptcc.agile.dms.common.utils.sm4.Sm4Util;
import com.jiuyv.sptcc.agile.dms.common.utils.uuid.IdUtils;
import com.jiuyv.sptcc.agile.dms.framework.security.model.OperationTokenSet;
import com.jiuyv.sptcc.agile.dms.framework.security.model.TokenNode;
/**
* ,便使
* @author Administrator
*
*/
@Component
@ConfigurationProperties(prefix = "operationtoken")
public class ConsoleOprTokenProperties {
/** 操作令牌标识(放具体值) */
private String headerVal="tokenSet";
/** 操作令牌标识 */
private String header="operationToken";
/** 操作令牌标识 */
private String headerKey="operationTokenKey";
/** 操作令牌次数 */
private Integer tokenTimes=100;
/** 操作令牌跳过的url */
private List<String> skipUrls;
@PostConstruct
public void init() {
Assert.notNull(tokenTimes, "openrationtToken times null");
}
/**
* @return the headerVal
*/
public String getHeaderVal() {
return headerVal;
}
/**
* @param headerVal the headerVal to set
*/
public void setHeaderVal(String headerVal) {
this.headerVal = headerVal;
}
/**
* @return the header
*/
public String getHeader() {
return header;
}
/**
* @param header the header to set
*/
public void setHeader(String header) {
this.header = header;
}
/**
* @return the headerKey
*/
public String getHeaderKey() {
return headerKey;
}
/**
* @param headerKey the headerKey to set
*/
public void setHeaderKey(String headerKey) {
this.headerKey = headerKey;
}
/**
* @return the tokenTimes
*/
public Integer getTokenTimes() {
return tokenTimes;
}
/**
* @param tokenTimes the tokenTimes to set
*/
public void setTokenTimes(Integer tokenTimes) {
this.tokenTimes = tokenTimes;
}
/**
* @return the skipUrls
*/
public List<String> getSkipUrls() {
return skipUrls;
}
/**
* @param skipUrls the skipUrls to set
*/
public void setSkipUrls(List<String> skipUrls) {
this.skipUrls = skipUrls;
}
//加密token
public String encryptToken(String token, String sm4Key) throws Exception {
return Base64.encodeBase64String(Sm4Util.encrypt_ECB_Padding(sm4Key.getBytes(Charsets.UTF_8), token.getBytes(Charsets.UTF_8)));
}
//解密token
public String decryptToken(String token, String sm4Key) throws Exception {
return Base64.encodeBase64String(Sm4Util.decrypt_ECB_Padding(Hex.decodeHex(sm4Key), Base64.decodeBase64(token)));
}
public ImmutablePair<TokenNode, String> calculateNewToken(HttpServletRequest request) throws Exception {
//
String sm4key = IdUtils.fastSimpleUUID().substring(0, 16);
String hexSm4Key = Hex.encodeHexString(sm4key.getBytes(Charsets.UTF_8));
//token
String newToken = IdUtils.fastSimpleUUID().substring(0, 16);
String encryptToken = this.encryptToken(newToken, sm4key);
TokenNode newNode = new TokenNode(encryptToken, newToken, hexSm4Key, sm4key, tokenTimes);
// String tokenSetStr = String.valueOf(request.getSession().getAttribute(this.headerVal));
// OperationTokenSet set = JsonUtil.json2Bean(tokenSetStr, OperationTokenSet.class);
OperationTokenSet set = (OperationTokenSet) request.getSession().getAttribute(this.headerVal);
if (null == set) {
set = new OperationTokenSet(newNode, sm4key);
// request.getSession().setAttribute(this.headerVal, JsonUtil.toJSONString(set));
request.getSession().setAttribute(this.headerVal, set);
} else {
set.getCurNode().setTimes(set.getCurNode().getTimes() - 1);
//当前node节点 放入 旧node map
Map<String, TokenNode> oldNodeMap = set.getTokenNodeMap();
if (null == oldNodeMap) {
oldNodeMap = new ConcurrentHashMap<>();
}
oldNodeMap.put(set.getCurNode().getEncryptToken(), set.getCurNode());
set.setTokenNodeMap(oldNodeMap);
//新node 替换
set.setCurNode(newNode);
//request.getSession().setAttribute(this.headerVal, JsonUtil.toJSONString(set));
request.getSession().setAttribute(this.headerVal, set);
}
return ImmutablePair.of(newNode, set.getCurNode().getHexSM4Key());
}
}

@ -0,0 +1,68 @@
package com.jiuyv.sptcc.agile.dms.common.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* ,便使
* @author Administrator
*
*/
@Component
@ConfigurationProperties(prefix = "token")
public class ConsoleTokenProperties {
/** 令牌自定义标识 */
//@Value("${token.header}")
private static String header;
/** 令牌秘钥 */
//@Value("${token.secret}")
private static String secret;
/** 令牌有效期默认30分钟 */
//@Value("${token.expireTime:30}")
private static int expireTime;
public static String getHeader() {
return header;
}
public void setHeader(String header) {
setHeader2(header);
}
public static void setHeader2(String header) {
ConsoleTokenProperties.header = header;
}
public static String getSecret() {
return secret;
}
public void setSecret(String secret) {
setSecret2(secret);
}
public static void setSecret2(String secret) {
ConsoleTokenProperties.secret = secret;
}
public static int getExpireTime() {
return expireTime;
}
public void setExpireTime(int expireTime) {
setExpireTime2(expireTime);
}
public static void setExpireTime2(int expireTime) {
if(expireTime<=0) {
ConsoleTokenProperties.expireTime = 30;
return;
}
ConsoleTokenProperties.expireTime = expireTime;
}
}

@ -0,0 +1,64 @@
package com.jiuyv.sptcc.agile.dms.common.constant;
/**
* key
*
* @author admin
*/
public class CacheConstants
{
/**
* redis key
*/
public static final String LOGIN_TOKEN_KEY = "login_tokens:";
/**
* redis key
*/
public static final String CAPTCHA_CODE_KEY = "";//置为空了独立缓存不用key减少空间
public static final String CAPTCHA_CODE_SIGN_KEY = "SIGN";//秘钥的key
/**
* cache key
*/
public static final String SYS_CONFIG_KEY = "sys_config:";
/**
* cache key
*/
public static final String SYS_DICT_KEY = "sys_dict:";
/**
* redis key
*/
public static final String REPEAT_SUBMIT_KEY = "repeat_submit:";
/**
* redis key
*/
public static final String RATE_LIMIT_KEY = "rate_limit:";
/**
* redis key
*/
public static final String METADATA_SOURCE_KEY = "data:metadata:sources";
/**
* redis key
*/
public static final String METADATA_TABLE_KEY = "data:metadata:tables";
/**
* redis key
*/
public static final String METADATA_COLUMN_KEY = "data:metadata:columns";
/**
* redis key
*/
public static final String METADATA_AUTHORIZE_KEY = "data:metadata:authorizes";
/**
*
*/
public static final String CAPTCHA_SMS_NUMBER_KEY = "sms_number";//秘钥的key
}

@ -0,0 +1,43 @@
package com.jiuyv.sptcc.agile.dms.common.constant;
/**
*
*
*/
public class CacheNameConstants
{
private CacheNameConstants() {
throw new IllegalStateException("Utility class");
}
/**
*
*/
public static final String CACHE_SYS_DICT = "cache_sys_dict";
/**
* ,
*/
public static final String CACHE_CAPTCHA_CODE = "cache_captcha_code";
/**
* 5s
*/
public static final String CACHE_REPEAT_SUBMIT_5S = "cache_repeat_submit_5s";
/**
* 30s
*/
public static final String CACHE_REPEAT_SUBMIT_30S = "cache_repeat_submit_30s";
/**
*
*/
public static final String CACHE_USER_DATA_SECRET_KEY = "cache_user_data_secret_key";
/**
* 1
*/
public static final String CACHE_ONEDAY = "cache_oneday";
}

@ -0,0 +1,183 @@
package com.jiuyv.sptcc.agile.dms.common.constant;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import io.jsonwebtoken.Claims;
/**
*
*
* @author admin
*/
public class Constants {
/**
* UTF-8
*/
public static final String UTF8 = "UTF-8";
/**
* GBK
*/
public static final String GBK = "GBK";
/**
* http
*/
public static final String HTTP = "http://";
/**
* https
*/
public static final String HTTPS = "https://";
/**
*
*/
public static final String SUCCESS = "0";
/**
*
*/
public static final String FAIL = "1";
/**
*
*/
public static final String LOGIN_SUCCESS = "Success";
/**
*
*/
public static final String LOGOUT = "Logout";
/**
*
*/
public static final String REGISTER = "Register";
/**
*
*/
public static final String LOGIN_FAIL = "Error";
/**
*
*/
public static final Integer CAPTCHA_EXPIRATION = 1;
/**
*
*/
public static final String TOKEN = "token";
/**
*
*/
public static final String TOKEN_PREFIX = "Bearer ";
/**
*
*/
public static final String LOGIN_USER_KEY = "login_user_key";
/**
* ID
*/
public static final String JWT_USERID = "userid";
/**
*
*/
public static final String JWT_USERNAME = Claims.SUBJECT;
/**
*
*/
public static final String JWT_AVATAR = "avatar";
/**
*
*/
public static final String JWT_CREATED = "created";
/**
*
*/
public static final String JWT_AUTHORITIES = "authorities";
/**
*
*/
public static final String RESOURCE_PREFIX = "/profile";
/**
* RMI
*/
public static final String LOOKUP_RMI = "rmi:";
/**
* LDAP
*/
public static final String LOOKUP_LDAP = "ldap:";
/**
* LDAPS
*/
public static final String LOOKUP_LDAPS = "ldaps:";
/**
* 访
*/
private static final String[] JOB_WHITELIST_STR = { "com.jiuyv.sptcc.agile" };
public static List<String> getJobWhiteListStr() {
return Collections.unmodifiableList(Arrays.asList(JOB_WHITELIST_STR));
}
/**
*
*/
private static final String[] JOB_ERROR_STR = { "java.net.URL", "javax.naming.InitialContext", "org.yaml.snakeyaml",
"org.springframework", "org.apache", "com.jiuyv.sptcc.agile.dms.common.utils.file" };
public static List<String> getJobErrorStr() {
return Collections.unmodifiableList(Arrays.asList(JOB_ERROR_STR));
}
public static final String REPEAT_PARAMS = "repeatParams";
public static final String REPEAT_TIME = "repeatTime";
public static final String SESSION_NAME = "JSESSIONID";
/** IP变量名*/
public static final String IP_ADDR = "ip_addr";
/** 后缀获取*/
public static final Pattern SUFFIX_RULE=Pattern.compile("\\.([A-Z0-9]+$)",Pattern.CASE_INSENSITIVE);
//图片的前缀固定
public static final String IMAGE_URL_PATH="images";
//文件的前缀固定
public static final String FILE_URL_PATH="files";
//文件分类路径写死,不允许随意传。要细分自己加。
public static final String FILE_TYPE_PATH_PORTAL= "portal";
public static final String FILE_TYPE_PATH_CONSOLE= "console";
private static final String[] FILE_TYPE_PATHS = {FILE_TYPE_PATH_PORTAL, FILE_TYPE_PATH_CONSOLE};
public static String[] getFileTypePaths() {
return FILE_TYPE_PATHS;
}
/** FEIGN_CLIENT标志 */
public final static String FEIGN_CLIENT_FLAG="gateway-source";
/** FEIGN_CLIENT标志 */
public final static String FEIGN_CLIENT_VAL="cabcc6d21b22a06d1";
//变量名不要version字段
public static final String NO_VERSION_FLAG="noVersionFlag";
/** 一个固定的密钥,用于配置文件内容加密/解密 */
public static final String SM4_SECERT_KEY = "a14751855ccb428d982c33dfa3535a57";
}

@ -0,0 +1,53 @@
package com.jiuyv.sptcc.agile.dms.common.constant;
public class DataConstant {
/**
*
*/
public enum TrueOrFalse {
FALSE("0",false),
TRUE("1",true);
TrueOrFalse(String key, boolean val){
this.key = key;
this.val = val;
}
private final String key;
private final boolean val;
public String getKey() {
return key;
}
public boolean getVal() {
return val;
}
}
/**
*
*/
public enum EnableState {
DISABLE("1","禁用"),
ENABLE("0","启用");
EnableState(String key, String val){
this.key = key;
this.val = val;
}
private final String key;
private final String val;
public String getKey() {
return key;
}
public String getVal() {
return val;
}
}
}

@ -0,0 +1,20 @@
package com.jiuyv.sptcc.agile.dms.common.constant;
/**
* type
*/
public class DictTypeNameConstants
{
private DictTypeNameConstants() {
throw new IllegalStateException("Utility class");
}
/**
* ,
*/
public class TBL_SYS_MENU{
//是否可见
public static final String VISIBLE = "sys_show_hide";
}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save