2023-12-29更新

master
RENCHAO 11 months ago
parent 26172f3340
commit a4f96eaf5c

@ -3,8 +3,14 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.7</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<groupId>com.renchao</groupId>
<artifactId>MyMaven</artifactId>
<version>1.0-SNAPSHOT</version>
@ -151,6 +157,18 @@
<version>1.36</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

@ -0,0 +1,12 @@
package com.renchao.unit;
/**
* @author ren_chao
*/
public class MyUtil {
public int test(String str) {
return Integer.parseInt(str);
}
}

@ -0,0 +1,32 @@
package com.renchao.unit;
import org.springframework.util.StringUtils;
import java.util.List;
/**
* @author ren_chao
*/
public class UnitTestDemo {
private MyUtil myUtil;
public int add(int a, int b) {
return a + b;
}
public String str(Integer i) {
return i + "a";
}
public int testUtil() {
return myUtil.test("11");
}
public int testList(List<String> list) {
return list.size() + 5;
}
}

@ -0,0 +1,39 @@
package com.renchao.unit;
import org.junit.Test;
import org.junit.jupiter.api.BeforeEach;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import java.util.Collections;
public class UnitTestDemoTest {
@Spy
UnitTestDemo unitTestDemo;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
public void testAdd() {
UnitTestDemo mock = Mockito.mock(UnitTestDemo.class);
Mockito.when(mock.add(Mockito.any(Integer.class), Mockito.any(Integer.class))).thenReturn(88);
System.out.println(mock.add(3, 2));
Mockito.when(mock.str(66)).thenReturn("66666666");
Mockito.when(mock.str(Mockito.any(Integer.class))).thenReturn("CCCCCCC");
System.out.println(mock.str(55));
Mockito.when(mock.testList(Mockito.anyList())).thenReturn(9900);
System.out.println(mock.testList(Collections.emptyList()));
}
}

@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>agile-bacth</artifactId>
<groupId>com.jiuyv.sptcc.agile.batch</groupId>
<version>1.0-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>agile-bacth</artifactId>
<groupId>com.jiuyv.sptcc.agile.batch</groupId>
<version>1.0-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -149,10 +147,42 @@
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.10</version>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>

@ -7,7 +7,6 @@ import com.jiuyv.sptcc.agile.batch.domain.ReqClearTableDTO;
import com.jiuyv.sptcc.agile.batch.domain.ReqSyncTableDTO;
import com.jiuyv.sptcc.agile.batch.service.BatchService;
import com.jiuyv.sptcc.agile.batch.service.ClearService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -33,24 +32,6 @@ public class BatchController {
this.clearService = clearService;
}
/**
*
*/
@GetMapping("/syncAll")
public AjaxResult<Void> syncAll() {
getService().syncAll();
return AjaxResult.success("同步完成");
}
/**
*
*/
@PostMapping("/syncByDateAndTableList")
public AjaxResult<Void> syncByDate(@RequestBody List<ReqSyncTableDTO> tableList) {
getService().syncByDate(tableList);
return AjaxResult.success("同步完成");
}
/**
*
*/

@ -1,47 +0,0 @@
package com.jiuyv.sptcc.agile.batch.dao;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* DAO
*
* @author ren_chao
*/
public abstract class BaseDAO {
protected final DataSource dataSource;
public BaseDAO(DataSource dataSource) {
this.dataSource = dataSource;
}
/**
*
*
*/
public Integer getCountByDate(String table, String whereColumn, String date) throws SQLException {
return getCount(String.format("select count(*) from %s where %s = '%s'", table, whereColumn, date));
}
/**
*
*
*/
public Integer getCountByDate(String table) throws SQLException {
return getCount("select count(*) from " + table);
}
private Integer getCount(String sql) throws SQLException {
try (Connection connection = dataSource.getConnection();
ResultSet resultSet = connection.prepareStatement(sql).executeQuery()) {
resultSet.next();
return resultSet.getInt(1);
}
}
}

@ -18,10 +18,12 @@ import java.util.Map;
* @author ren_chao
*/
@Repository
public class HiveDAO extends BaseDAO {
public class HiveDAO {
private final DataSource dataSource;
public HiveDAO(DataSource hiveDataSource) {
super(hiveDataSource);
this.dataSource = hiveDataSource;
}
/**
@ -43,22 +45,6 @@ public class HiveDAO extends BaseDAO {
}
}
/**
* Hive
*/
public List<String> getDates(String hiveTable) throws SQLException {
String sql = "show partitions " + hiveTable;
try (Connection connection = dataSource.getConnection();
ResultSet resultSet = connection.prepareStatement(sql).executeQuery()) {
List<String> list = new ArrayList<>();
while (resultSet.next()) {
list.add(resultSet.getString(1).split("=")[1]);
}
return list;
}
}
/**
* Hive
*/

@ -25,14 +25,15 @@ import java.util.stream.Collectors;
* @author ren_chao
*/
@Repository
public class PostgreDAO extends BaseDAO {
public class PostgreDAO {
private static final Pattern SQL_INSERT_PATTERN = Pattern.compile("\\(([^)]+)\\)[ ]*VALUES\\(([^)]+)\\)", Pattern.CASE_INSENSITIVE);
private final DataSource dataSource;
public PostgreDAO(DataSource pgDataSource) {
super(pgDataSource);
this.dataSource = pgDataSource;
}
/**
*
*/
@ -71,22 +72,6 @@ public class PostgreDAO extends BaseDAO {
}
}
/**
*
*/
public String getLastDate(String column, String table) throws SQLException {
String sql = String.format("SELECT DISTINCT %s FROM %s ORDER BY %s DESC LIMIT 1", column, table, column);
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
if (resultSet.next()) {
return resultSet.getString(1);
}
return null;
}
}
public void clearTable(String table, String date, String dateColumn) throws SQLException {
String sql = String.format("DELETE FROM %s WHERE %s = '%s'", table, dateColumn, date);
try (Connection connection = dataSource.getConnection();

@ -14,6 +14,7 @@ import com.jiuyv.sptcc.agile.batch.exception.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.task.TaskExecutor;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
@ -27,7 +28,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Date;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
@ -76,27 +76,6 @@ public class BatchService {
this.executor = tableProperties.getExecutor();
}
/**
*
*
*/
public void syncAll() {
initConsumer();
try {
sync(new ArrayList<>(tableMapByHive.values()));
} catch (IOException | SQLException e) {
LOGGER.error("同步时发生异常:{}", e.getMessage(), e);
throw new ServiceException("同步异常,详情查看日志");
} catch (InterruptedException e) {
LOGGER.error("同步时中断异常:{}", e.getMessage(), e);
Thread.currentThread().interrupt();
throw new ServiceException("同步异常,详情查看日志");
} finally {
keepRunning = false;
}
}
/**
*
*
@ -116,22 +95,13 @@ public class BatchService {
for (ReqSyncTableDTO tableDTO : tableList) {
String hiveTable = tableDTO.getHiveTable();
TableInfo tableInfo = tableMapByHive.get(hiveTable);
String dateColumn = tableInfo.getDateColumn();
String date = tableDTO.getDate();
if (tableInfo.isPart()) {
Integer count = hiveDAO.getCountByDate(hiveTable, dateColumn, date);
selectData(tableInfo, date, count, 0);
} else {
selectData(tableInfo, date);
}
selectData(tableInfo, date);
}
// 等待完成
waitCompletion();
} catch (SQLException e) {
} catch (SQLException | InterruptedException e) {
LOGGER.error("同步时发生异常:{}", e.getMessage(), e);
throw new ServiceException("同步异常,详情查看日志");
} catch (InterruptedException e) {
LOGGER.error("同步时中断异常:{}", e.getMessage(), e);
Thread.currentThread().interrupt();
throw new ServiceException("同步异常,详情查看日志");
} finally {
@ -157,23 +127,6 @@ public class BatchService {
}
}
/**
*
*
*/
private void sync(List<TableInfo> tables) throws SQLException, InterruptedException, IOException {
for (TableInfo table : tables) {
LOGGER.info("====开始同步表:{}", table.getHiveTable());
if (table.isPart()) {
syncByPart(table);
} else {
syncNoPart(table);
}
}
// 等待完成
waitCompletion();
}
/**
*
*
@ -196,39 +149,6 @@ public class BatchService {
}
}
/**
*
*/
private void syncByPart(TableInfo tableInfo) throws SQLException, InterruptedException {
String hiveTable = tableInfo.getHiveTable();
String pgTable = tableInfo.getPgTable();
String dateColumn = tableInfo.getDateColumn();
// 准备同步
List<String> dates = hiveDAO.getDates(tableInfo.getHiveTable());
String pgLastDate = postgreDAO.getLastDate(dateColumn, pgTable);
List<String> syncParts;
if (pgLastDate == null) {
syncParts = dates;
} else {
syncParts = dates.stream().filter(date -> toDate(date).after(toDate(pgLastDate))).collect(Collectors.toList());
// 查询hive最后同步分区总记录数
Integer hiveCount = hiveDAO.getCountByDate(hiveTable, dateColumn, pgLastDate);
Integer pgCount = postgreDAO.getCountByDate(pgTable, dateColumn, pgLastDate);
if (hiveCount > pgCount) {
// 同步该区上次未完成的
selectData(tableInfo, pgLastDate, hiveCount, pgCount);
}
}
// 逐个分区进行同步
syncParts = syncParts.stream().sorted().collect(Collectors.toList());
for (String part : syncParts) {
Integer count = hiveDAO.getCountByDate(hiveTable, dateColumn, part);
selectData(tableInfo, part, count, 0);
}
}
/**
* dataQueue
*
@ -286,7 +206,6 @@ public class BatchService {
/**
* fileQueueSFTP
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
private void uploadSFTP() {
while (keepRunning || !fileQueue.isEmpty()) {
File file = null;
@ -309,8 +228,8 @@ public class BatchService {
LOGGER.debug("线程中断异常:{}", e.getMessage(), e);
Thread.currentThread().interrupt();
} finally {
if (file != null) {
file.delete();
if (file != null && !file.delete()) {
LOGGER.info("文件删除失败:{}", file);
}
}
}
@ -318,75 +237,29 @@ public class BatchService {
}
/**
*
*/
private void syncNoPart(TableInfo tableInfo) throws SQLException, InterruptedException {
long hiveCount = hiveDAO.getCountByDate(tableInfo.getHiveTable());
long pgCount = postgreDAO.getCountByDate(tableInfo.getPgTable());
if (hiveCount <= pgCount) {
return;
}
// 未分区一次查出全部【因为带条件查询count和查询日期信息时效率非常慢】
String sql = String.format(TableProperties.HIVE_SQL_TEMPLATE, tableInfo.getHiveTable(), "");
List<Map<String, Object>> list = hiveDAO.selectList(sql, pgCount, Integer.MAX_VALUE);
Map<String, List<Map<String, Object>>> dateMap = list.stream()
.collect(Collectors.groupingBy(m -> m.get(tableInfo.getDateColumn()).toString()));
for (Map.Entry<String, List<Map<String, Object>>> entry : dateMap.entrySet()) {
String date = entry.getKey();
List<Map<String, Object>> mapList = entry.getValue();
DataInfoContainer container = new DataInfoContainer(tableInfo, mapList, mapList.size(), date);
putDataQueue(container);
LOGGER.info("====表{}放入队列,总数:{},本次范围:{},日期:{}", tableInfo.getHiveTable(), mapList.size(), "0-" + mapList.size(), date);
}
}
/**
* Date
*
*/
private Date toDate(String str) {
if (str.length() <= 7) {
return Date.valueOf(str + "-01");
}
return Date.valueOf(str);
}
/**
* hive
*
*/
private void selectData(TableInfo tableInfo, String part, Integer count, Integer start) throws SQLException, InterruptedException {
Integer offset = start;
while (offset < count) {
private void selectData(TableInfo tableInfo, String part) throws SQLException, InterruptedException {
int count = 0;
int offset = 0;
boolean keepOn = true;
while (keepOn) {
List<Map<String, Object>> list = hiveDAO.selectList(tableInfo.getHiveSql(), part, offset, BATCH_QUANTITY);
DataInfoContainer container = new DataInfoContainer(tableInfo, list, null, part);
putDataQueue(container);
offset += BATCH_QUANTITY;
if (offset >= count) {
container.setTotal(count);
if (CollectionUtils.isEmpty(list)) {
putDataQueue(new DataInfoContainer(tableInfo, list, count, part));
keepOn = false;
} else {
putDataQueue(new DataInfoContainer(tableInfo, list, null, part));
offset += BATCH_QUANTITY;
count = count + list.size();
LOGGER.info("====表{}放入队列,记录数:{},日期:{}", tableInfo.getHiveTable(), offset - BATCH_QUANTITY + "-" + count, part);
}
LOGGER.info("====表{}放入队列,总数:{},本次范围:{},日期:{}", tableInfo.getHiveTable(), count, offset - BATCH_QUANTITY + "-" + offset, part);
}
}
/**
* hive
*
*
*/
private void selectData(TableInfo tableInfo, String date) throws SQLException, InterruptedException {
List<Map<String, Object>> list = hiveDAO.selectList(tableInfo.getHiveSql(), date, 0, Integer.MAX_VALUE);
int count = list.size();
DataInfoContainer container = new DataInfoContainer(tableInfo, list, count, date);
putDataQueue(container);
LOGGER.info("====表{}放入队列,总数:{},本次范围:{},日期:{}", tableInfo.getHiveTable(), count, 0 + "-" + count, date);
}
private void putDataQueue(DataInfoContainer container) throws InterruptedException {
while (!dataQueue.offer(container, 500, TimeUnit.MILLISECONDS)) {
if (!keepRunning) {

@ -26,4 +26,4 @@ eureka:
register-with-eureka: true
fetch-registry: true
service-url:
defaultZone: http://172.16.12.107:8761/eureka/
defaultZone: http://172.16.12.109:8761/eureka/

@ -0,0 +1,143 @@
package com.jiuyv.sptcc.agile.batch.service;
import com.jiuyv.sptcc.agile.batch.config.TableProperties;
import com.jiuyv.sptcc.agile.batch.config.sftp.SftpChannelPool;
import com.jiuyv.sptcc.agile.batch.config.sftp.SftpProperties;
import com.jiuyv.sptcc.agile.batch.dao.HiveDAO;
import com.jiuyv.sptcc.agile.batch.dao.PostgreDAO;
import com.jiuyv.sptcc.agile.batch.domain.DataInfoContainer;
import com.jiuyv.sptcc.agile.batch.domain.ReqSyncTableDTO;
import com.jiuyv.sptcc.agile.batch.domain.TableInfo;
import com.jiuyv.sptcc.agile.batch.exception.ServiceException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.task.TaskExecutor;
import org.springframework.test.util.ReflectionTestUtils;
import java.io.File;
import java.lang.reflect.Field;
import java.sql.SQLException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.*;
@ExtendWith(MockitoExtension.class)
class BatchServiceTest {
@Mock
private LinkedBlockingQueue<DataInfoContainer> dataQueue;
@Mock
private LinkedBlockingQueue<File> fileQueue;
@Mock
private Map<String, TableInfo> tableMapByPg;
@Mock
private Map<String, TableInfo> tableMapByHive;
@Spy
@InjectMocks
private TableProperties tableProperties;
@Mock
private SftpProperties sftpProperties;
@Mock
private HiveDAO hiveDAO;
@Mock
private PostgreDAO postgreDAO;
@Mock
private SftpChannelPool sftpChannelPool;
@Mock
TaskExecutor executor;
@Test
void syncByDate() throws Exception {
List<ReqSyncTableDTO> tableList = Collections.singletonList(new ReqSyncTableDTO());
TableInfo tableInfo = mock(TableInfo.class);
when(tableMapByPg.get(null)).thenReturn(tableInfo);
when(tableMapByHive.get(null)).thenReturn(tableInfo);
when(dataQueue.offer(any(DataInfoContainer.class),anyLong(),any(TimeUnit.class))).thenReturn(true);
BatchService batchService = getBatchService();
new Thread(()->endWaiting(batchService)).start();
batchService.syncByDate(tableList);
when(hiveDAO.selectList(any(), any()))
.thenReturn(Collections.singletonList(new HashMap<>()))
.thenReturn(null);
new Thread(()->endWaiting(batchService)).start();
batchService.syncByDate(tableList);
ReflectionTestUtils.setField(batchService, "exception", mock(ServiceException.class));
new Thread(()->endWaiting(batchService)).start();
try {
batchService.syncByDate(tableList);
fail();
} catch (Exception e) {
assertTrue(e instanceof ServiceException);
}
when(hiveDAO.selectList(any(), any())).thenThrow(SQLException.class);
try {
batchService.syncByDate(tableList);
fail();
} catch (Exception e) {
assertTrue(e instanceof ServiceException);
}
doThrow(SQLException.class).when(postgreDAO).clearTable(any(), any(), any());
try {
batchService.syncByDate(tableList);
fail();
} catch (Exception e) {
assertTrue(e instanceof ServiceException);
}
}
@Test
void clearTable() {
System.out.println(System.getProperty("os.PATHEXT"));
}
private BatchService getBatchService() {
BatchService batchService = new BatchService(tableProperties);
ReflectionTestUtils.setField(batchService, "tableMapByPg", tableMapByPg);
ReflectionTestUtils.setField(batchService, "tableMapByHive", tableMapByHive);
ReflectionTestUtils.setField(batchService, "dataQueue", dataQueue);
ReflectionTestUtils.setField(batchService, "fileQueue", fileQueue);
return batchService;
}
@SuppressWarnings("all")
private void endWaiting(BatchService batchService) {
try {
Field sync = batchService.getClass().getDeclaredField("sync");
sync.setAccessible(true);
byte end = 3;
while (true) {
Byte b = (Byte) sync.get(batchService);
if (b == 1) {
sync.set(batchService, end);
break;
}
Thread.sleep(200);
}
} catch (ReflectiveOperationException | InterruptedException e) {
e.printStackTrace();
}
}
}

@ -1,16 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>agile-bacth</artifactId>
<groupId>com.jiuyv.sptcc.agile.batch</groupId>
<version>1.0-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>agile-batch-service</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
<name>agile-batch-service</name>
<description>agile-batch-service</description>
@ -73,6 +72,16 @@
<version>0.1.55</version>
</dependency>
<!--整合prometheus-->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-sleuth</artifactId>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>

@ -1,5 +1,7 @@
package com.jiuyv.sptcc.agile.batch.batchTask.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@ -24,5 +26,8 @@ public interface TblBatchTaskMapper{
/** 重置全部任务 */
void updateResetAllBusStatus(@Param("vo") TblBatchTask record,@Param("map") TblBatchTaskVO paramMap);
List<TblBatchTaskVO> selectListByMap(TblBatchTaskVO paramMap);
}

@ -173,4 +173,12 @@ public class BatchTaskServiceImpl implements IBatchTaskService {
//不报错
}
}
@Override
public List<TblBatchTaskVO> getRunningBatchTaskList() throws Exception {
TblBatchTaskVO batchTaskParamMap = new TblBatchTaskVO();
batchTaskParamMap.setBusStatus(TblBatchTaskEnum.BUS_STATUS.RUNING.getCode());
return tblBatchTaskMapper.selectListByMap(batchTaskParamMap);
}
}

@ -54,4 +54,9 @@ public interface IBatchTaskService {
* @throws Exception
*/
public void doBatchTaskUnFinish(TblBatchTask task) throws Exception;
/** 获取正则运行的任务*/
public List<TblBatchTaskVO> getRunningBatchTaskList() throws Exception;
}

@ -0,0 +1,109 @@
package com.jiuyv.sptcc.agile.batch.common.constant;
/**
*
* @author zhouliang
*
*/
public class TblPublicFilesEnum {
/** 文件类型 */
public enum FILE_TYPE {
NORMAL("normal", "常规文件"),
FLINK("flink", "flink组件"),//和docker组件类型统一的
PYTHON("python", "python组件"),//和docker组件类型统一的
DATA("data", "数据文件"),//和docker组件类型统一的
;
private String code;
private String msg;
FILE_TYPE(String code, String msg) {
this.code = code;
this.msg = msg;
}
public String getCode() {
return code;
}
public String getMsg() {
return msg;
}
}
/** 文件来源*/
public enum FILE_SOURCE_TYPE {
//在文件表绑定关系只用作快捷条件建议仍在各自业务表中绑定文件id关系
DOCKERLIB("dockerlib", "实验室组件表"),
USER("user", "用户表表"),
;
private String code;
private String msg;
FILE_SOURCE_TYPE(String code, String msg) {
this.code = code;
this.msg = msg;
}
public String getCode() {
return code;
}
public String getMsg() {
return msg;
}
}
/**
*
*/
public enum SYS_TYPE {
CONSOLE("console", "控制台"),
PORTAL("portal", "门户"),
;
private String code;
private String msg;
SYS_TYPE(String code, String msg) {
this.code = code;
this.msg = msg;
}
public String getCode() {
return code;
}
public String getMsg() {
return msg;
}
}
/** 数据状态*/
public enum DATA_STATUS {
NORMAL("00", "正常"), //业务数据正常使用中
CLEAR("96", "已清理"),//真实的文件已删除了
TRANSFERRING("97", "传输中"),//表示与业务数据还没绑定的,且没有传输完成(表示在分块/续传),可以直接清理的数据,
TEMPORARY("98", "临时"),//表示与业务数据还没绑定的,可以直接清理的数据
DELETED("99", "删除"),//表示与业务数据绑定,但是已删除,按需要选择清理
;
private String code;
private String msg;
DATA_STATUS(String code, String msg) {
this.code = code;
this.msg = msg;
}
public String getCode() {
return code;
}
public String getMsg() {
return msg;
}
}
}

@ -33,13 +33,12 @@ public class SFTPChannel {
private static final Logger LOG = LoggerFactory.getLogger(SFTPChannel.class);
/**
*
*
* @param sftpDetails
* @param timeout
* @return
* @throws JSchException
*/
public ChannelSftp getChannel(SFTPConfig sftpDetails, int timeout) throws JSchException {
public void setConnect(SFTPConfig sftpDetails, int timeout) throws JSchException {
String ftpHost = sftpDetails.getHost();
int ftpPort = sftpDetails.getPort();
String ftpUserName = sftpDetails.getUsername();
@ -57,10 +56,6 @@ public class SFTPChannel {
session.setConfig(config); // 为Session对象设置properties
session.setTimeout(timeout); // 设置timeout时间
session.connect(); // 通过Session建立链接
channel = session.openChannel("sftp"); // 打开SFTP通道
channel.connect(); // 建立SFTP通道的连接
LOG.debug("Connected successfully to ftpHost = " + ftpHost + ",as ftpUserName = " + ftpUserName);
return (ChannelSftp) channel;
}
/**

@ -1,21 +1,35 @@
package com.jiuyv.sptcc.agile.batch.controller;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.jcraft.jsch.ChannelSftp;
import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
import com.jiuyv.sptcc.agile.batch.common.BaseTime;
import com.jiuyv.sptcc.agile.batch.common.R;
import com.jiuyv.sptcc.agile.batch.common.constant.TblPublicFilesEnum;
import com.jiuyv.sptcc.agile.batch.common.util.sftp.SFTPChannel;
import com.jiuyv.sptcc.agile.batch.common.util.sftp.model.SFTPConfig;
import com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles;
import com.jiuyv.sptcc.agile.batch.framework.SftpConfigProperties;
import com.jiuyv.sptcc.agile.batch.service.ISftpTempFileClearService;
/**
@ -31,9 +45,22 @@ public class SftpTempFileClearController {
@Autowired
private SftpConfigProperties sftpConfigProperties;
@Autowired
private ISftpTempFileClearService sftpTempFileClearService;
@Value("${docker.dockerDataPath:}")
private String dockerDataPath;
@Value("${syncdata.clearLogFileDays:7}")
private int clearLogFileDays;
/* 查找docker的待销毁的文件 */
public static final String DOCKER_SHELL_STAT_GREP = "ls '%s' | xargs -I {} stat -c \"%%n %%Y\" '%s'{} |grep .pid";
/* 删除文件 */
public static final String SHELL_RM = "rm -rf '%s'";
/**
* 98
* 98,97
* 99
* @return
* @throws Exception
*/
@ -41,32 +68,78 @@ public class SftpTempFileClearController {
public R<String> clearSftpTempFile() throws Exception{
//查询临时文件
String excludeFileSourceType=sftpConfigProperties.getExcludeFileSourceType();
List<String> excludeFileSourceTypes=null;
if(StringUtils.isNotBlank(excludeFileSourceType)) {
String[] excludeFileSourceTypeArr=excludeFileSourceType.trim().split(" *, *");
List<String> list=new ArrayList<>();
excludeFileSourceTypes=new ArrayList<>();
for(String fileSourceType:excludeFileSourceTypeArr) {
if(StringUtils.isNotBlank(fileSourceType)) {
list.add(fileSourceType);
excludeFileSourceTypes.add(fileSourceType);
}
}
}
sftpConfigProperties.getClearTempFileDays();
BaseTime timeVO = sftpTempFileClearService.getSysDate();
int days1 = sftpConfigProperties.getClearTempFileDays();
int days2 = sftpConfigProperties.getClearTempFileDays();
//临时文件
List<TblPublicFiles> flist1 = sftpTempFileClearService.getTempFileList(getCurrDateReduceDay(timeVO.getUtcTime()
, days1),excludeFileSourceTypes);
//已删除文件
List<TblPublicFiles> flist2 = sftpTempFileClearService.getDeletedFileList(getCurrDateReduceDay(timeVO.getUtcTime()
, days2));
if(flist1.isEmpty() && flist2.isEmpty()) {
return R.ok("finish");
}
boolean flag=true;
List<Long> filedIds=new ArrayList<>();
//连接文件服务器清理批处理
SFTPChannel channelx = new SFTPChannel();
try {
//创建容器前先创建目录挂载目录(后续要使用sftp操作)
ChannelSftp chSftp = channelx.getChannel(createSftpCfg(), 60000);
chSftp.quit();//直接退
channelx.commitCurrShellCmd(String.format("rm -rf %s",""));
channelx.setConnect(createSftpCfg(), 60000);
for(TblPublicFiles x:flist1) {
filedIds.add(x.getFileId());
String path="";
if(TblPublicFilesEnum.DATA_STATUS.TEMPORARY.getCode().equals( x.getDataStatus())) {
path=x.getFactPath(sftpConfigProperties.getUploadPath());
}else if(TblPublicFilesEnum.DATA_STATUS.TRANSFERRING.getCode().equals( x.getDataStatus())) {
path=sftpConfigProperties.getUploadPath()+"/"+x.getSysType()+"-tmp/"+x.getUuid();
}
channelx.commitCurrShellCmd(String.format(SHELL_RM,path));
x=null;
}
for(TblPublicFiles x:flist2) {
filedIds.add(x.getFileId());
String path=x.getFactPath(sftpConfigProperties.getUploadPath());
channelx.commitCurrShellCmd(String.format(SHELL_RM,path));
x=null;
}
} catch (Exception e) {
LOGGER.info("initDocker error>>" ,e);
flag=false;
}finally {
channelx.closeChannel();
}
flist1=null;
flist2=null;
if(!filedIds.isEmpty()) {
int groupSize=1000;
for (int i = 0; i < filedIds.size(); i += groupSize) {
int endIndex = Math.min(i + groupSize, filedIds.size());
List<Long> sublist = filedIds.subList(i, endIndex);
sftpTempFileClearService.doUpdateDataStatus(sublist);
}
}
if(!flag) {
return R.ok("failed");
}
return R.ok("finish");
}
@ -84,4 +157,63 @@ public class SftpTempFileClearController {
cfg.setHost(sftpConfigProperties.getHost());
return cfg;
}
public String getCurrDateReduceDay(Instant tm,int days) throws Exception {
Instant ntime = tm.plus(-days, ChronoUnit.DAYS);
Date date =new Date(ntime.toEpochMilli());
return new SimpleDateFormat("yyyy-MM-dd").format(date);
}
/**
*
*/
@GetMapping("clearDockerLogFile")
public R<String> clearDockerLogFile() throws Exception{
BaseTime timeVO = sftpTempFileClearService.getSysDate();
boolean flag=true;
//连接文件服务器清理批处理
SFTPChannel channelx = new SFTPChannel();
try {
//创建容器前先创建目录挂载目录(后续要使用sftp操作)
channelx.setConnect(createSftpCfg(), 60000);
//因为实验室总数量就几个,直接扫描实验室总目录下的文件
if(!dockerDataPath.endsWith("/")) {
dockerDataPath+="/";
}
String lscommand= String.format(DOCKER_SHELL_STAT_GREP, dockerDataPath,dockerDataPath);
List<String> list =channelx.commitCurrShellCmd(lscommand);
if(!list.isEmpty()) {
for(String x:list) {
if(StringUtils.isBlank(x)) {
continue;
}
String[] arrs= x.split(" +");
if(arrs[1].length()==10) {//没有毫秒,补全
arrs[1]=arrs[1]+"000";
}
// 获取文件的修改时间
Date modifiedDate = new Date(Long.valueOf(arrs[1]));
long d= timeVO.getDate().getTime()-modifiedDate.getTime();
Duration duration = Duration.of(d, ChronoUnit.MILLIS);
if(duration.toDays()>=clearLogFileDays) {
channelx.commitCurrShellCmd(String.format(SHELL_RM, arrs[0])
+" && "+ String.format(SHELL_RM, arrs[0].replace(".pid", "")));
}
}
}
} catch (Exception e) {
LOGGER.info("initDocker error>>" ,e);
flag=false;
}finally {
channelx.closeChannel();
}
if(!flag) {
return R.ok("failed");
}
return R.ok("finish");
}
}

@ -0,0 +1,25 @@
package com.jiuyv.sptcc.agile.batch.dao;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles;
import com.jiuyv.sptcc.agile.batch.entity.vo.TblPublicFilesVO;
/**
*
* @author zhouliang
* @date 2023-06-15
*/
@Mapper
public interface TblPublicFilesMapper{
/** 查询分页 */
List<TblPublicFiles> selectTempFileList(TblPublicFilesVO paramMap);
/** 更新数据状态 */
void updateFileDataStatus(@Param("vo") TblPublicFiles record,@Param("map") TblPublicFilesVO paramMap);
}

@ -0,0 +1,393 @@
package com.jiuyv.sptcc.agile.batch.entity;
import java.io.File;
import java.util.Date;
/**
*
* @author zhouliang
* @date 2023-06-15
*/
public class TblPublicFiles implements java.io.Serializable {
private static final long serialVersionUID = 1L;
/** 文件主键id */
private Long fileId;
/** 版本号 */
private Long versionNum;
/** 随机码 */
private String recToken;
/** uuid */
private String uuid;
/** 文件名称 */
private String fileName;
/** 文件后缀名 */
private String fileExtension;
/** 文件大小 */
private String fileSize;
/** 文件分类路径 */
private String fileCategoryPath;
/** 文件类型 */
private String fileType;
/** 文件来源 */
private String fileSourceType;
/** 系统类型 */
private String sysType;
/** 文件备注 */
private String remarks;
/** 关联业务id */
private String fileBusiId;
/** 关联业务id2 */
private String fileBusiId2;
/** 关联业务id3 */
private String fileBusiId3;
/** 已上传大小 */
private String uploadSize;
/** 数据状态 */
private String dataStatus;
/** 创建用户id */
private String createBy;
/** 创建用户 */
private String createByName;
/** 创建时间 */
private Date createTime;
/** 更新用户id */
private String updateBy;
/** 更新用户 */
private String updateByName;
/** 更新时间 */
private Date updateTime;
/**
* Getid
*/
public Long getFileId(){
return fileId;
}
/**
* Setid
*/
public void setFileId(Long fileId){
this.fileId = fileId;
}
/**
* Get
*/
public Long getVersionNum(){
return versionNum;
}
/**
* Set
*/
public void setVersionNum(Long versionNum){
this.versionNum = versionNum;
}
/**
* Get
*/
public String getRecToken(){
return recToken;
}
/**
* Set
*/
public void setRecToken(String recToken){
this.recToken = recToken;
}
/**
* Getuuid
*/
public String getUuid(){
return uuid;
}
/**
* Setuuid
*/
public void setUuid(String uuid){
this.uuid = uuid;
}
/**
* Get
*/
public String getFileName(){
return fileName;
}
/**
* Set
*/
public void setFileName(String fileName){
this.fileName = fileName;
}
/**
* Get
*/
public String getFileExtension(){
return fileExtension;
}
/**
* Set
*/
public void setFileExtension(String fileExtension){
this.fileExtension = fileExtension;
}
/**
* Get
*/
public String getFileSize(){
return fileSize;
}
/**
* Set
*/
public void setFileSize(String fileSize){
this.fileSize = fileSize;
}
/**
* Get
*/
public String getFileCategoryPath(){
return fileCategoryPath;
}
/**
* Set
*/
public void setFileCategoryPath(String fileCategoryPath){
this.fileCategoryPath = fileCategoryPath;
}
/**
* Get
*/
public String getFileType(){
return fileType;
}
/**
* Set
*/
public void setFileType(String fileType){
this.fileType = fileType;
}
/**
* Get
*/
public String getFileSourceType(){
return fileSourceType;
}
/**
* Set
*/
public void setFileSourceType(String fileSourceType){
this.fileSourceType = fileSourceType;
}
/**
* Get
*/
public String getSysType(){
return sysType;
}
/**
* Set
*/
public void setSysType(String sysType){
this.sysType = sysType;
}
/**
* Get
*/
public String getRemarks(){
return remarks;
}
/**
* Set
*/
public void setRemarks(String remarks){
this.remarks = remarks;
}
/**
* Getid
*/
public String getFileBusiId(){
return fileBusiId;
}
/**
* Setid
*/
public void setFileBusiId(String fileBusiId){
this.fileBusiId = fileBusiId;
}
/**
* Getid2
*/
public String getFileBusiId2(){
return fileBusiId2;
}
/**
* Setid2
*/
public void setFileBusiId2(String fileBusiId2){
this.fileBusiId2 = fileBusiId2;
}
/**
* Getid3
*/
public String getFileBusiId3(){
return fileBusiId3;
}
/**
* Setid3
*/
public void setFileBusiId3(String fileBusiId3){
this.fileBusiId3 = fileBusiId3;
}
/**
* Get
*/
public String getUploadSize(){
return uploadSize;
}
/**
* Set
*/
public void setUploadSize(String uploadSize){
this.uploadSize = uploadSize;
}
/**
* Get
*/
public String getDataStatus(){
return dataStatus;
}
/**
* Set
*/
public void setDataStatus(String dataStatus){
this.dataStatus = dataStatus;
}
/**
* Getid
*/
public String getCreateBy(){
return createBy;
}
/**
* Setid
*/
public void setCreateBy(String createBy){
this.createBy = createBy;
}
/**
* Get
*/
public String getCreateByName(){
return createByName;
}
/**
* Set
*/
public void setCreateByName(String createByName){
this.createByName = createByName;
}
/**
* Get
*/
public Date getCreateTime(){
return createTime;
}
/**
* Set
*/
public void setCreateTime(Date createTime){
this.createTime = createTime;
}
/**
* Getid
*/
public String getUpdateBy(){
return updateBy;
}
/**
* Setid
*/
public void setUpdateBy(String updateBy){
this.updateBy = updateBy;
}
/**
* Get
*/
public String getUpdateByName(){
return updateByName;
}
/**
* Set
*/
public void setUpdateByName(String updateByName){
this.updateByName = updateByName;
}
/**
* Get
*/
public Date getUpdateTime(){
return updateTime;
}
/**
* Set
*/
public void setUpdateTime(Date updateTime){
this.updateTime = updateTime;
}
/*
*
*/
public String getFactPath(String rootPath) {
return rootPath+"/"+fileCategoryPath+"/"+uuid+"."+fileExtension;
}
}

@ -0,0 +1,71 @@
package com.jiuyv.sptcc.agile.batch.entity.vo;
import java.util.List;
import com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles;
/**
*
* @author zhouliang
* @date 2023-06-15
*/
public class TblPublicFilesVO extends TblPublicFiles implements java.io.Serializable {
private static final long serialVersionUID = 1L;
/** 条件:文件来源集合 */
private List<String> fileSourceTypes;
/** 条件:数据状态集合 */
private List<String> dataStatuss;
/** 条件文件id集合 */
private List<Long> fileIds;
/**
* @return the fileSourceTypes
*/
public List<String> getFileSourceTypes() {
return fileSourceTypes;
}
/**
* @param fileSourceTypes the fileSourceTypes to set
*/
public void setFileSourceTypes(List<String> fileSourceTypes) {
this.fileSourceTypes = fileSourceTypes;
}
/**
* @return the dataStatuss
*/
public List<String> getDataStatuss() {
return dataStatuss;
}
/**
* @param dataStatuss the dataStatuss to set
*/
public void setDataStatuss(List<String> dataStatuss) {
this.dataStatuss = dataStatuss;
}
/**
* @return the fileIds
*/
public List<Long> getFileIds() {
return fileIds;
}
/**
* @param fileIds the fileIds to set
*/
public void setFileIds(List<Long> fileIds) {
this.fileIds = fileIds;
}
}

@ -39,9 +39,14 @@ public class SftpConfigProperties implements java.io.Serializable{
/** 全局上传主路径 */
private String uploadPath;
/** 自动清理任务排除文件类型fileSourceType */
/** 自动清理任务:排除文件类型fileSourceType */
private String excludeFileSourceType;
/** 自动清理任务:临时文件清理过期天数 */
private int clearTempFileDays=1;
/** 自动清理任务:删除文件清理过期天数 */
private int clearDelFileDays=3;
/**
* @return the host
@ -115,4 +120,32 @@ public class SftpConfigProperties implements java.io.Serializable{
public void setExcludeFileSourceType(String excludeFileSourceType) {
this.excludeFileSourceType = excludeFileSourceType;
}
/**
* @return the clearTempFileDays
*/
public int getClearTempFileDays() {
return clearTempFileDays;
}
/**
* @param clearTempFileDays the clearTempFileDays to set
*/
public void setClearTempFileDays(int clearTempFileDays) {
this.clearTempFileDays = clearTempFileDays;
}
/**
* @return the clearDelFileDays
*/
public int getClearDelFileDays() {
return clearDelFileDays;
}
/**
* @param clearDelFileDays the clearDelFileDays to set
*/
public void setClearDelFileDays(int clearDelFileDays) {
this.clearDelFileDays = clearDelFileDays;
}
}

@ -0,0 +1,28 @@
package com.jiuyv.sptcc.agile.batch.service;
import java.util.List;
import com.jiuyv.sptcc.agile.batch.common.BaseTime;
import com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles;
/**
*
* @author zhouliang
* @date 2023-07-05
*/
public interface ISftpTempFileClearService {
/** 获取时间 */
public BaseTime getSysDate() throws Exception;
/** 获取临时文件98和传输中的97*/
public List<TblPublicFiles> getTempFileList(String dateStr,List<String> excludeFileSourceTypes) throws Exception;
/** 获取已删除的文件99 */
public List<TblPublicFiles> getDeletedFileList(String dateStr) throws Exception;
/** 更新文件状态*/
public void doUpdateDataStatus(List<Long> filedIds) throws Exception;
}

@ -0,0 +1,76 @@
package com.jiuyv.sptcc.agile.batch.service;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.jiuyv.sptcc.agile.batch.common.BaseTime;
import com.jiuyv.sptcc.agile.batch.common.constant.TblPublicFilesEnum;
import com.jiuyv.sptcc.agile.batch.dao.ISysTimeBaseMapper;
import com.jiuyv.sptcc.agile.batch.dao.TblPublicFilesMapper;
import com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles;
import com.jiuyv.sptcc.agile.batch.entity.vo.TblPublicFilesVO;
/**
*
* @author zhouliang
* @date 2023-07-05
*/
@Service("sftpTempFileClearService")
public class SftpTempFileClearImpl implements ISftpTempFileClearService {
@Autowired
private TblPublicFilesMapper tblPublicFilesMapper;
@Autowired
private ISysTimeBaseMapper sysTimeBaseMapper;
@Override
public List<TblPublicFiles> getTempFileList(String dateStr,List<String> excludeFileSourceTypes) throws Exception {
TblPublicFilesVO paramMap=new TblPublicFilesVO();
Date currdate=new SimpleDateFormat("yyyy-MM-dd").parse(dateStr);
paramMap.setCreateTime(DateUtils.addDays(currdate, -3));
paramMap.setUpdateTime(DateUtils.addDays(currdate, 1));//因为不含等于所以加一天
List<String> dataStatuss=new ArrayList<>();
dataStatuss.add(TblPublicFilesEnum.DATA_STATUS.TEMPORARY.getCode());
dataStatuss.add(TblPublicFilesEnum.DATA_STATUS.TRANSFERRING.getCode());
paramMap.setDataStatuss(dataStatuss);
paramMap.setFileSourceTypes(excludeFileSourceTypes);
return tblPublicFilesMapper.selectTempFileList(paramMap);
}
@Override
public List<TblPublicFiles> getDeletedFileList(String dateStr) throws Exception {
TblPublicFilesVO paramMap=new TblPublicFilesVO();
Date currdate=new SimpleDateFormat("yyyy-MM-dd").parse(dateStr);
paramMap.setCreateTime(DateUtils.addDays(currdate, -3));//预留几天,防止任务没有正常运行
paramMap.setUpdateTime(DateUtils.addDays(currdate, 1));//因为不含等于所以加一天
List<String> dataStatuss=new ArrayList<>();
dataStatuss.add(TblPublicFilesEnum.DATA_STATUS.DELETED.getCode());
paramMap.setDataStatuss(dataStatuss);
return tblPublicFilesMapper.selectTempFileList(paramMap);
}
@Override
public void doUpdateDataStatus(List<Long> filedIds) throws Exception {
BaseTime tmieVO = getSysDate();
TblPublicFiles record=new TblPublicFiles();
record.setUpdateTime(tmieVO.getDate());
record.setDataStatus(TblPublicFilesEnum.DATA_STATUS.CLEAR.getCode());
TblPublicFilesVO paramMap=new TblPublicFilesVO();
paramMap.setFileIds(filedIds);
tblPublicFilesMapper.updateFileDataStatus(record,paramMap);
}
//获取系统时间
@Override
public BaseTime getSysDate() throws Exception {
return sysTimeBaseMapper.selectSysCurrentTime();
}
}

@ -59,6 +59,8 @@ public class JsyHiveJDBCBuilder {
String zooKeeperNamespace = clientInfo.getProperty("zooKeeperNamespace");
String serviceDiscoveryMode = clientInfo.getProperty("serviceDiscoveryMode");
String principal = clientInfo.getProperty("principal");
String socketTimeout = clientInfo.getProperty("socketTimeout");
String connectTimeout = clientInfo.getProperty("connectTimeout");
String userKeytabFile=null;
String krb5File=null;
@ -95,8 +97,19 @@ public class JsyHiveJDBCBuilder {
.append(serviceDiscoveryMode)
.append(";zooKeeperNamespace=")
.append(zooKeeperNamespace)
.append(";auth=none");
.append(";auth=none;");
}
if(StringUtils.isNotBlank(socketTimeout)) {
sBuilder.append("socketTimeout="+socketTimeout+";");
}else {
sBuilder.append("socketTimeout=12000;");
}
if(StringUtils.isNotBlank(connectTimeout)) {
sBuilder.append("connectTimeout="+connectTimeout+";");
}else {
sBuilder.append("connectTimeout=6000;");
}
String url = sBuilder.toString();
// System.out.print(url);
return url;
@ -168,8 +181,9 @@ public class JsyHiveJDBCBuilder {
.append(serviceDiscoveryMode)
.append(";zooKeeperNamespace=")
.append(zooKeeperNamespace)
.append(";auth=none");
.append(";auth=none;");
}
sBuilder.append("hive2.server.execute.timeout=12000;");
String url = sBuilder.toString();
// System.out.print(url);

@ -36,6 +36,13 @@ public class JdbcBaseReader {
LOGGER.info("Create connection url={}, username={}",url,username);
// 加载JDBC驱动
Class.forName(driverClassName);
if(!url.contains("socketTimeout")) {
// socketTimeout=12000
//loginTimeout=60
}else if(!url.contains("loginTimeout")) {
// socketTimeout=12000
//loginTimeout=60
}
// 获取JDBC连接
connection = DriverManager.getConnection(url, username!=null?username:"", password!=null?password:"");

@ -215,7 +215,7 @@ public class ReaderWriterHelper {
LOGGER.info("writeData exec selectSql>>time={}",(ttime2-ttime));
long count=0;
if(resultSet!=null) {
LOGGER.info("writeData Task Progress>>start");
LOGGER.info("writeData Task Progress>>start, each num={}",singleWriteNumber);
//解决使用*的返回字段全路径问题
int n=resultSet.getMetaData().getColumnCount();
Map<String,Integer> colMap=new LinkedHashMap<>();

@ -1,13 +1,19 @@
package com.jiuyv.sptcc.agile.batch.syncJiushiData.controller;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
@ -28,6 +34,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.jiuyv.sptcc.agile.batch.batchTask.common.TblBatchTaskEnum;
import com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO;
import com.jiuyv.sptcc.agile.batch.batchTask.service.IBatchTaskService;
import com.jiuyv.sptcc.agile.batch.common.BaseTime;
import com.jiuyv.sptcc.agile.batch.common.R;
import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.DDsProperties;
import com.jiuyv.sptcc.agile.batch.syncJiushiData.common.SyncDataConstants;
@ -51,8 +58,10 @@ public class SyncDataReadTaskController {
private IBatchTaskService batchTaskService;
@Value("${syncdata.clearTempFileDs:}")
private String defaultClearTempFileDs;
@Value("${syncdata.clearTempFileDays:15}")
@Value("${syncdata.clearTempFileDays:7}")
private int defaultClearTempFileDays;
@Value("${syncdata.resetTaskStatusTime:30}")
private int resetTaskStatusTime;
//外部任务调度器触发的任务,线程不是同一个,必须保证任务只能有一个执行
//下面实现了通用的按天处理的单表同步模式
@ -195,6 +204,17 @@ public class SyncDataReadTaskController {
printWriter.flush();
int exitCode= process.waitFor();
if (exitCode != 0) {
// 获取错误信息流
InputStream errorStream = process.getErrorStream();
BufferedReader errorReader = new BufferedReader(new InputStreamReader(errorStream));
String line;
String line2="";
while ((line = errorReader.readLine()) != null) {
line2+="\n"+line;
}
LOGGER.error(">>>>>>"+line2);
// 关闭流
errorReader.close();
LOGGER.error("refresh Kerberos TGTCheck if the Kerberos command exists and permissions");
}else {
LOGGER.warn("refresh Kerberos TGT Finish.");
@ -212,15 +232,14 @@ public class SyncDataReadTaskController {
refreshTicket();//启动就初始化一次,这样可以不用先手动创建
}
/**
*
*
* @return
* @throws Exception
*/
@GetMapping("clearTempFile")
@GetMapping("clearSyncTempFile")
public R<String> clearTempFile() throws Exception{
//如果写入的是文件,则会产生临时文件,定时清理1个月之前文件
//如果写入的是文件,则会产生临时文件,定时清理
if(StringUtils.isNotBlank(defaultClearTempFileDs)) {
String[] dsarr=defaultClearTempFileDs.split(",");
for(String dsx:dsarr) {
@ -253,12 +272,33 @@ public class SyncDataReadTaskController {
}
// 获取文件的修改时间
Date modifiedDate = new Date(file.lastModified());
Date date = DateUtils.addDays(modifiedDate, days);
Date currDate = new Date();
int d= DateUtils.truncatedCompareTo(date, currDate, Calendar.DATE);
if(d<0 && file.delete()) {
long d= currDate.getTime()-modifiedDate.getTime();
Duration duration = Duration.of(d, ChronoUnit.MILLIS);
if(duration.toDays()>=days && file.delete()) {
//ok
}
}
}
/**
*
* @throws Exception
*/
@Scheduled(cron = "0 */10 * * * ?")
public void batchTaskReset() throws Exception{
List<TblBatchTaskVO> list = batchTaskService.getRunningBatchTaskList();
if(list==null || list.isEmpty()) {
return;
}
BaseTime timeVO = syncDataReadTaskServiceManager.getSysDate();
for(TblBatchTaskVO x:list) {
if((timeVO.getUtcTime().toEpochMilli()-x.getCurrStartDate().getTime())>=(resetTaskStatusTime*60*1000)) {
LOGGER.error("batchTaskReset>>{}", x.getTaskNo());
batchTaskService.doBatchTaskUnFinish(x);
}
}
}
}

@ -21,11 +21,17 @@ filesftp:
username: flink
password: mIVYkELj5T+4MnD5+V542A==
uploadPath: /home/flink/sysfile
#清理文件时,需要排除文件表的类型,多个用,分隔
excludeFileSourceType: usertest,usertest2
#清理分块上传临时目录,多个用,分隔
includeFileCategoryPath: console-tmp,portal-tmp
clearTempFileDays: 3 #过期天数更新时间超过N天即过期
#清理临时文件时排除文件表的FileSourceType多个用,分隔(要排除的自己定义不同的类型,勿冲突)
excludeFileSourceType: portaltest
#临时文件过期天数更新时间超过N天即过期
clearTempFileDays: 30
#已删除文件过期天数更新时间超过N天即过期(分开控制)
clearDelFileDays: 30
docker:
#实验室主目录
dockerDataPath: /home/flink/docker_data
clearLogFileDays: 3 #日志过期天数销毁超过N天即过期
console:
#readWritePath: /home/flink/read_write_data
readWritePath: F:\ZLworkspace\agilesystem\agile.batch\src\trunk\agile-bacth\agile-batch-service\read_write_data
@ -35,6 +41,7 @@ syncdata:
singleWriteNumber: 20000 #读取一定数量写入一次
clearTempFileDs: klbHiveDs #需清理的数据库,同步任务临时文件、过期文件,多个用,分隔
clearTempFileDays: 7 #过期天数更新时间超过N天即过期
resetTaskStatusTime: 40 #任务超过N分钟没结束就强制终止,最长的算
klbHiveDs: #客流宝hive
url: jdbc:hive2://172.16.12.101:10000/hive;socketTimeout=12000;
@ -44,6 +51,7 @@ klbHiveDs: #客流宝hive
singleWriteNumber: 2000 #读取一定数量写入一次,覆盖默认
readWritePath: ${console.readWritePath}2 #文件存放路径,覆盖默认
fieldSeparator: ',' #文件字段分隔符号,默认就是逗号
clearTempFileDays: 7 #过期天数更新时间超过N天即过期
klbPgDs: #客流宝pg
url: ${spring.datasource.url}
username: ${spring.datasource.username}

@ -7,3 +7,5 @@ sasl.qop = auth-conf
zooKeeperNamespace = hiveserver2
serviceDiscoveryMode = zooKeeper
instanceNo = 0
socketTimeout=12000
connectTimeout=6000

@ -2,6 +2,6 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.jiuyv.sptcc.agile.batch.dao.ISysTimeBaseMapper">
<select id="selectSysCurrentTime" resultType="com.jiuyv.sptcc.agile.batch.common.BaseTime">
SELECT now() AT TIME ZONE 'UTC' AS utcTime, now() AS date
SELECT now() AS utcTime, now() AS date
</select>
</mapper>

@ -49,7 +49,7 @@
<if test="vo.preEndDate != null" >pre_end_date = #{vo.preEndDate},</if>
<if test="vo.preTotalTime != null and vo.preTotalTime != ''" >pre_total_time = #{vo.preTotalTime},</if>
<if test="vo.currStartDate != null" >curr_start_date = #{vo.currStartDate},</if>
<if test="vo.failureConditions != null and vo.failureConditions != ''" >failure_conditions = #{vo.failureConditions},</if>
<if test="vo.failureConditions != null" >failure_conditions = #{vo.failureConditions},</if>
<if test="vo.busStatus != null and vo.busStatus != ''" >bus_status = #{vo.busStatus},</if>
<if test="vo.dataStatus != null and vo.dataStatus != ''" >data_status = #{vo.dataStatus},</if>
<if test="vo.updateTime != null" >update_time = #{vo.updateTime},</if>
@ -79,4 +79,15 @@
</where>
</update>
<!-- 查询集合 -->
<select id="selectListByMap" resultType="com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO" parameterType="com.jiuyv.sptcc.agile.batch.batchTask.entity.vo.TblBatchTaskVO">
select
<include refid="_select"/>
from tbl_batch_task a
<where>
and bus_status = #{busStatus}
</where>
</select>
</mapper>

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<!-- 系统文件表 -->
<mapper namespace="com.jiuyv.sptcc.agile.batch.dao.TblPublicFilesMapper">
<!-- 查询文件集合 -->
<select id="selectTempFileList" resultType="com.jiuyv.sptcc.agile.batch.entity.TblPublicFiles" parameterType="com.jiuyv.sptcc.agile.batch.entity.vo.TblPublicFilesVO">
select
a.file_id,
a.uuid,
a.file_extension,
a.file_category_path,
a.sys_type,
a.data_status
from tbl_public_files a
<where>
and data_status in (<foreach collection="dataStatuss" item="fst" separator=",">#{fst}</foreach>)
and update_time &gt;= #{createTime}::timestamp
and update_time &lt; #{updateTime}::timestamp
<if test="fileSourceTypes != null" >
and file_source_type not in (<foreach collection="fileSourceTypes" item="fst" separator=",">#{fst}</foreach>)
</if>
</where>
</select>
<!-- 更新数据状态 -->
<update id="updateFileDataStatus" parameterType="com.jiuyv.sptcc.agile.batch.entity.vo.TblPublicFilesVO">
update tbl_public_files
set
<if test="vo.updateBy != null" >#{vo.updateBy},</if>
<if test="vo.updateByName != null" >#{vo.updateByName},</if>
data_status = #{vo.dataStatus},
update_time = #{vo.updateTime}
where
file_id in (<foreach collection="map.fileIds" item="fileId" separator=",">#{fileId}</foreach>)
</update>
</mapper>

@ -1,18 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.7</version>
<relativePath/>
<relativePath />
</parent>
<groupId>com.jiuyv.sptcc.agile.batch</groupId>
<artifactId>agile-bacth</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.0.5-SNAPSHOT</version>
<modules>
<module>agile-batch-api</module>
<module>agile-batch-service</module>
@ -68,8 +66,8 @@
</distributionManagement>
<scm>
<connection>scm:svn:http://172.16.12.10/sptcc_agile_etl/src/agile-batch/src/trunk/agile-batch</connection>
<developerConnection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile-batch/src/trunk/agile-batch</developerConnection>
<connection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile.batch/src/trunk/agile-bacth</connection>
<developerConnection>scm:svn:http://172.16.12.10/svn/sptcc_agile_etl/src/agile.batch/src/trunk/agile-bacth</developerConnection>
</scm>
<repositories>

@ -3,7 +3,7 @@
<parent>
<artifactId>agile-portal</artifactId>
<groupId>com.jiuyv.sptcc.agile</groupId>
<version>0.0.5-SNAPSHOT</version>
<version>0.2.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -20,16 +20,59 @@
<artifactId>spring-web</artifactId>
</dependency>
<!--eureka-client-->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<!-- <scope>provided</scope>-->
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<!-- <scope>provided</scope>-->
</dependency>
<dependency>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-common</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.10</version>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.PortalContentDTO;
import com.jiuyv.sptccc.agile.dto.ReqPageDTO;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.DockerApplyReviewDTO;
import com.jiuyv.sptccc.agile.dto.ReqDockerApplyPageDTO;

@ -1,6 +1,6 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.DockerDownloadApplyDTO;
import com.jiuyv.sptccc.agile.dto.ReqDockerDownApplyPageDTO;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.DockerFileDTO;
import com.jiuyv.sptccc.agile.dto.DockerWithUserDTO;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.ReqFileDTO;
import com.jiuyv.sptccc.agile.dto.UploadFileDTO;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.dto.ReqLogDTO;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.domain.TableDataPage;
import com.jiuyv.sptccc.agile.dto.PortalMessageDTO;
import com.jiuyv.sptccc.agile.dto.ReqPageDTO;

@ -1,7 +1,7 @@
package com.jiuyv.sptccc.agile.api;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.core.domain.R;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.dto.PortalUserDTO;
import com.jiuyv.sptccc.agile.dto.ResUserPasswordDTO;
import org.springframework.web.bind.annotation.GetMapping;

@ -1,14 +1,14 @@
package com.jiuyv.sptccc.agile.common.annotation;
import com.jiuyv.sptccc.agile.common.enums.BusinessType;
import com.jiuyv.sptccc.agile.common.enums.OperatorType;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.jiuyv.sptccc.agile.common.enums.BusinessType;
import com.jiuyv.sptccc.agile.common.enums.OperatorType;
/**
*
*

@ -2,18 +2,20 @@ package com.jiuyv.sptccc.agile.common.annotation;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* 使
*
*
* @author admin
*
* @author zhouliang
*/
@Target({ElementType.METHOD})
@Inherited
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface LogIgnore {
}
public @interface NoRepeatSubmit {
}

@ -5,7 +5,7 @@ package com.jiuyv.sptccc.agile.common.constant;
*
* @author admin
*/
public class Constants {
public class ServiceConstant {
/**
* -

@ -20,8 +20,6 @@ public final class ServiceException extends RuntimeException {
/**
*
* <p>
* {@link CommonResult#getDetailMessage()}
*/
private String detailMessage;
@ -44,12 +42,13 @@ public final class ServiceException extends RuntimeException {
return detailMessage;
}
public String getMessage() {
return message;
public ServiceException setDetailMessage(String detailMessage) {
this.detailMessage = detailMessage;
return this;
}
public Integer getCode() {
return code;
public String getMessage() {
return message;
}
public ServiceException setMessage(String message) {
@ -57,8 +56,7 @@ public final class ServiceException extends RuntimeException {
return this;
}
public ServiceException setDetailMessage(String detailMessage) {
this.detailMessage = detailMessage;
return this;
public Integer getCode() {
return code;
}
}

@ -1,10 +1,14 @@
package com.jiuyv.sptccc.agile.common.utils.ip;
import java.net.InetAddress;
import java.net.UnknownHostException;
import javax.servlet.http.HttpServletRequest;
package com.jiuyv.sptccc.agile.common.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.client.RestTemplate;
import javax.servlet.http.HttpServletRequest;
/**
* IP
@ -12,6 +16,36 @@ import org.apache.commons.lang3.StringUtils;
* @author admin
*/
public class IpUtils {
// IP地址查询
public static final String IP_URL = "http://whois.pconline.com.cn/ipJson.jsp";
// 未知地址
public static final String UNKNOWN = "XX XX";
private static final Logger LOGGER = LoggerFactory.getLogger(IpUtils.class);
private static final RestTemplate restTemplate = new RestTemplate();
private static final ObjectMapper objectMapper = new ObjectMapper();
public static String getRealAddressByIP(String ip, boolean isAddressEnabled) {
// 内网不查询
if (IpUtils.internalIp(ip)) {
return "内网IP";
}
if (isAddressEnabled) {
String rspStr = restTemplate.getForEntity(IP_URL + "?ip=" + ip + "&json=true", String.class).getBody();
if (StringUtils.isBlank(rspStr)) {
LOGGER.error("获取地理位置异常 {}", ip);
return UNKNOWN;
}
try {
JsonNode obj = objectMapper.readTree(rspStr);
return obj == null ? UNKNOWN : (obj.get("pro").asText() + " " + obj.get("city").asText());
} catch (JsonProcessingException e) {
LOGGER.error("获取地理位置异常 {}", ip);
return UNKNOWN;
}
}
return UNKNOWN;
}
/**
* IP
*
@ -64,9 +98,6 @@ public class IpUtils {
* @return
*/
private static boolean internalIp(byte[] addr) {
if (addr == null || addr.length < 2) {
return true;
}
final byte b0 = addr[0];
final byte b1 = addr[1];
// 10.x.x.x/8
@ -82,15 +113,9 @@ public class IpUtils {
case SECTION_1:
return true;
case SECTION_2:
if (b1 >= SECTION_3 && b1 <= SECTION_4) {
return true;
}
return false;
return b1 >= SECTION_3 && b1 <= SECTION_4;
case SECTION_5:
if (b1 == SECTION_6) {
return true;
}
return false;
return b1 == SECTION_6;
default:
return false;
}
@ -103,7 +128,7 @@ public class IpUtils {
* @return byte
*/
public static byte[] textToNumericFormatV4(String text) {
if (org.apache.commons.lang3.StringUtils.isBlank(text)) {
if (StringUtils.isBlank(text)) {
return null;
}
@ -170,32 +195,6 @@ public class IpUtils {
return bytes;
}
/**
* IP
*
* @return IP
*/
public static String getHostIp() {
try {
return InetAddress.getLocalHost().getHostAddress();
} catch (UnknownHostException e) {
}
return "127.0.0.1";
}
/**
*
*
* @return
*/
public static String getHostName() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e) {
}
return "未知";
}
/**
* unknown IP
*

@ -1,12 +1,13 @@
package com.jiuyv.sptccc.agile.common.utils;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;
import org.springframework.web.context.request.RequestAttributes;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
/**
*
*
@ -24,6 +25,15 @@ public class ServletUtils {
}
/**
* response
*/
public static HttpServletResponse getResponse() {
ServletRequestAttributes requestAttributes = getRequestAttributes();
assert requestAttributes != null;
return requestAttributes.getResponse();
}
public static ServletRequestAttributes getRequestAttributes() {
RequestAttributes attributes = RequestContextHolder.getRequestAttributes();
return (ServletRequestAttributes) attributes;

@ -1,15 +1,17 @@
package com.jiuyv.sptccc.agile.common.utils;
import org.apache.commons.lang3.StringUtils;
import java.security.SecureRandom;
import java.util.UUID;
/**
*
*
* @author admin
*/
public class StringUtil {
private static final SecureRandom random = new SecureRandom();
/**
*
*/
@ -57,34 +59,6 @@ public class StringUtil {
return str.substring(start, end);
}
/**
* ssizesize
*
* @param s
* @param size
* @param c 使
* @return
*/
public static String align(final String s, final int size, final char c) {
final StringBuilder sb = new StringBuilder(size);
if (s != null) {
final int len = s.length();
if (s.length() <= size) {
for (int i = size - len; i > 0; i--) {
sb.append(c);
}
sb.append(s);
} else {
return s.substring(len - size, len);
}
} else {
for (int i = size; i > 0; i--) {
sb.append(c);
}
}
return sb.toString();
}
public static String getFileExtension(String fileName) {
if (StringUtils.isBlank(fileName)) {
return BLANK_STR;
@ -107,10 +81,40 @@ public class StringUtil {
/**
* RecToken
* @return
*
*/
public static String getRecToken() {
return getSimpleUUID().substring(0, 10);
}
/**
*
*
*/
public static String strHide(String str) {
if (str == null || str.length() <= 2) {
return str;
}
if (str.length() <= 7) {
return str.substring(0, 2) + "***";
}
return str.substring(0, 3) + "****" + str.substring(str.length() - 4);
}
/**
*
*
*/
public static String randomNumber(int length) {
if (length < 4) {
length = 4;
}
StringBuilder bld = new StringBuilder();
for (int i = 0; i < length; i++) {
bld.append(random.nextInt(10));
}
return bld.toString();
}
}

@ -10,7 +10,7 @@ import java.util.List;
*/
public class TableDataPage<T> implements Serializable {
private static final long serialVersionUID = 1L;
private int total;
private Integer total;
private List<T> rows;
private Integer code;
private String msg;
@ -18,16 +18,16 @@ public class TableDataPage<T> implements Serializable {
public TableDataPage() {
}
public TableDataPage(List<T> list, int total) {
public TableDataPage(List<T> list, Integer total) {
this.rows = list;
this.total = total;
}
public int getTotal() {
public Integer getTotal() {
return this.total;
}
public void setTotal(int total) {
public void setTotal(Integer total) {
this.total = total;
}

@ -1,116 +0,0 @@
package com.jiuyv.sptccc.agile.dto;
import com.fasterxml.jackson.annotation.JsonFormat;
import java.io.Serializable;
import java.util.Date;
public class DataApiDTO implements Serializable {
private static final long serialVersionUID = 1L;
/**
*
*/
private String orgNo;
/**
*
*/
private String orgName;
/**
* id
*/
private Long apiId;
/**
*
*/
private String apiName;
/**
*
*/
private String remark;
/**
*
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/**
*
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date dataBegin;
/**
*
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date dataEnd;
public String getOrgNo() {
return orgNo;
}
public void setOrgNo(String orgNo) {
this.orgNo = orgNo;
}
public String getOrgName() {
return orgName;
}
public void setOrgName(String orgName) {
this.orgName = orgName;
}
public Long getApiId() {
return apiId;
}
public void setApiId(Long apiId) {
this.apiId = apiId;
}
public String getApiName() {
return apiName;
}
public void setApiName(String apiName) {
this.apiName = apiName;
}
public String getRemark() {
return remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public Date getDataBegin() {
return dataBegin;
}
public void setDataBegin(Date dataBegin) {
this.dataBegin = dataBegin;
}
public Date getDataEnd() {
return dataEnd;
}
public void setDataEnd(Date dataEnd) {
this.dataEnd = dataEnd;
}
}

@ -1,111 +0,0 @@
package com.jiuyv.sptccc.agile.dto;
import java.io.Serializable;
import java.util.Date;
public class DataApiStatisticsDTO implements Serializable {
private static final long serialVersionUID = 1L;
/**
*
*/
private Object id;
/**
*
*/
private String orgNo;
/**
*
*/
private String orgName;
/**
* API ID
*/
private Object apiId;
/**
*
*/
private String apiName;
/**
*
*/
private Object successTotal;
/**
*
*/
private Integer failTotal;
/**
*
*/
private Date updateTime;
public Object getId() {
return id;
}
public void setId(Object id) {
this.id = id;
}
public String getOrgNo() {
return orgNo;
}
public void setOrgNo(String orgNo) {
this.orgNo = orgNo;
}
public String getOrgName() {
return orgName;
}
public void setOrgName(String orgName) {
this.orgName = orgName;
}
public Object getApiId() {
return apiId;
}
public void setApiId(Object apiId) {
this.apiId = apiId;
}
public Object getSuccessTotal() {
return successTotal;
}
public void setSuccessTotal(Object successTotal) {
this.successTotal = successTotal;
}
public Integer getFailTotal() {
return failTotal;
}
public void setFailTotal(Integer failTotal) {
this.failTotal = failTotal;
}
public String getApiName() {
return apiName;
}
public void setApiName(String apiName) {
this.apiName = apiName;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}

@ -1,297 +0,0 @@
package com.jiuyv.sptccc.agile.dto;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
/**
*
*/
public class DockerApplyInfoDTO implements Serializable {
private static final long serialVersionUID = 1L;
/**
* id
*/
private Long applyId;
/**
*
*/
private String recToken;
/**
* id
*/
private Long dockerServerId;
/**
* id
*/
private String applyUserId;
/**
*
*/
private String applyUserName;
/**
*
*/
private String applyDesc;
/**
*
*/
private String labTitle;
/**
*
*/
private String serviceType;
/**
* 使
*/
private String dockerImageName;
/**
*
*/
private String usageTime;
/**
*
*/
private String usageTimeUnit;
/**
*
*/
private Date startDate;
/**
*
*/
private Date endDate;
/**
*
*/
private String delayTimes;
/**
* cpu
*/
private String cpuLimits;
/**
* cpu
*/
private String cpuSeq;
/**
*
*/
private String memoryLimits;
/**
*
*/
private String discLimits;
/**
*
*/
private String remarks;
/**
*
*/
private String reviewStatus;
/**
*
*/
private String reviewDesc;
/**
*
*/
private List<DockerLibDTO> applyLibList;
public Long getApplyId() {
return applyId;
}
public void setApplyId(Long applyId) {
this.applyId = applyId;
}
public String getRecToken() {
return recToken;
}
public void setRecToken(String recToken) {
this.recToken = recToken;
}
public Long getDockerServerId() {
return dockerServerId;
}
public void setDockerServerId(Long dockerServerId) {
this.dockerServerId = dockerServerId;
}
public String getApplyUserId() {
return applyUserId;
}
public void setApplyUserId(String applyUserId) {
this.applyUserId = applyUserId;
}
public String getApplyUserName() {
return applyUserName;
}
public void setApplyUserName(String applyUserName) {
this.applyUserName = applyUserName;
}
public String getApplyDesc() {
return applyDesc;
}
public void setApplyDesc(String applyDesc) {
this.applyDesc = applyDesc;
}
public String getLabTitle() {
return labTitle;
}
public void setLabTitle(String labTitle) {
this.labTitle = labTitle;
}
public String getServiceType() {
return serviceType;
}
public void setServiceType(String serviceType) {
this.serviceType = serviceType;
}
public String getDockerImageName() {
return dockerImageName;
}
public void setDockerImageName(String dockerImageName) {
this.dockerImageName = dockerImageName;
}
public String getUsageTime() {
return usageTime;
}
public void setUsageTime(String usageTime) {
this.usageTime = usageTime;
}
public String getUsageTimeUnit() {
return usageTimeUnit;
}
public void setUsageTimeUnit(String usageTimeUnit) {
this.usageTimeUnit = usageTimeUnit;
}
public Date getStartDate() {
return startDate;
}
public void setStartDate(Date startDate) {
this.startDate = startDate;
}
public Date getEndDate() {
return endDate;
}
public void setEndDate(Date endDate) {
this.endDate = endDate;
}
public String getDelayTimes() {
return delayTimes;
}
public void setDelayTimes(String delayTimes) {
this.delayTimes = delayTimes;
}
public String getCpuLimits() {
return cpuLimits;
}
public void setCpuLimits(String cpuLimits) {
this.cpuLimits = cpuLimits;
}
public String getCpuSeq() {
return cpuSeq;
}
public void setCpuSeq(String cpuSeq) {
this.cpuSeq = cpuSeq;
}
public String getMemoryLimits() {
return memoryLimits;
}
public void setMemoryLimits(String memoryLimits) {
this.memoryLimits = memoryLimits;
}
public String getDiscLimits() {
return discLimits;
}
public void setDiscLimits(String discLimits) {
this.discLimits = discLimits;
}
public String getRemarks() {
return remarks;
}
public void setRemarks(String remarks) {
this.remarks = remarks;
}
public String getReviewStatus() {
return reviewStatus;
}
public void setReviewStatus(String reviewStatus) {
this.reviewStatus = reviewStatus;
}
public String getReviewDesc() {
return reviewDesc;
}
public void setReviewDesc(String reviewDesc) {
this.reviewDesc = reviewDesc;
}
public List<DockerLibDTO> getApplyLibList() {
return applyLibList;
}
public void setApplyLibList(List<DockerLibDTO> applyLibList) {
this.applyLibList = applyLibList;
}
}

@ -0,0 +1,113 @@
package com.jiuyv.sptccc.agile.common.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import javax.servlet.http.HttpServletRequest;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import static com.jiuyv.sptccc.agile.common.utils.IpUtils.UNKNOWN;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* IpUtils
*
* @author ren_chao
*/
@ExtendWith(MockitoExtension.class)
class IpUtilsTest {
@Mock
private RestTemplate restTemplate;
@Mock
private ObjectMapper objectMapper;
@Mock
private ResponseEntity<String> response;
@BeforeEach
void setUp() throws ReflectiveOperationException {
Field modifier = Field.class.getDeclaredField("modifiers");
modifier.setAccessible(true);
Field restTemplateField = IpUtils.class.getDeclaredField("restTemplate");
restTemplateField.setAccessible(true);
modifier.setInt(restTemplateField, restTemplateField.getModifiers() & ~Modifier.FINAL);
restTemplateField.set(null, restTemplate);
Field objectMapperField = IpUtils.class.getDeclaredField("objectMapper");
objectMapperField.setAccessible(true);
modifier.setInt(objectMapperField, objectMapperField.getModifiers() & ~Modifier.FINAL);
objectMapperField.set(null, objectMapper);
}
@Test
void getRealAddressByIP() throws JsonProcessingException {
String json = "{}";
assertEquals("内网IP", IpUtils.getRealAddressByIP("192.168.0.1", true));
when(restTemplate.getForEntity(anyString(), eq(String.class))).thenReturn(response);
assertEquals(UNKNOWN, IpUtils.getRealAddressByIP("localhost", true));
when(response.getBody()).thenReturn(json);
assertEquals(UNKNOWN, IpUtils.getRealAddressByIP("localhost", true));
when(objectMapper.readTree(anyString())).thenThrow(JsonProcessingException.class);
assertEquals(UNKNOWN, IpUtils.getRealAddressByIP("localhost", true));
assertEquals("内网IP", IpUtils.getRealAddressByIP("10.0.0.1", false));
assertEquals("内网IP", IpUtils.getRealAddressByIP("172.16.0.1", false));
assertEquals(UNKNOWN, IpUtils.getRealAddressByIP("1", false));
}
@Test
void getIpAddr() {
assertEquals("unknown", IpUtils.getIpAddr(null));
HttpServletRequest request = mock(HttpServletRequest.class);
assertNull(IpUtils.getIpAddr(request));
}
@Test
void textToNumericFormatV4() {
try {
IpUtils.textToNumericFormatV4("127");
IpUtils.textToNumericFormatV4("127.0");
IpUtils.textToNumericFormatV4("127.0.0");
IpUtils.textToNumericFormatV4("127.0.0.1");
IpUtils.textToNumericFormatV4("127.0.0.1.1");
IpUtils.textToNumericFormatV4("-1");
IpUtils.textToNumericFormatV4("-1.0");
IpUtils.textToNumericFormatV4("-1.0.0");
IpUtils.textToNumericFormatV4("-1.0.0.1");
IpUtils.textToNumericFormatV4("127.-1");
IpUtils.textToNumericFormatV4("127.0.-1");
IpUtils.textToNumericFormatV4("");
} catch (Exception e) {
fail();
}
}
@Test
void getMultistageReverseProxyIp() {
String ip = "127.0.0.1,localhost";
assertEquals("127.0.0.1", IpUtils.getMultistageReverseProxyIp(ip));
}
}

@ -0,0 +1,71 @@
package com.jiuyv.sptccc.agile.common.utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.when;
/**
* ServletUtils
*
* @author ren_chao
*/
@ExtendWith(MockitoExtension.class)
class ServletUtilsTest {
@Mock
private MockedStatic<RequestContextHolder> mockStatic;
@Mock
private ServletRequestAttributes attributes;
@Mock
private HttpServletRequest request;
@Mock
private HttpServletResponse response;
@Mock
private HttpSession session;
@BeforeEach
void setUp() {
mockStatic.when(RequestContextHolder::getRequestAttributes).thenReturn(attributes);
}
@AfterEach
void tearDown() {
mockStatic.close();
}
@Test
void getRequest() {
when(attributes.getRequest()).thenReturn(request);
assertEquals(request, ServletUtils.getRequest());
}
@Test
void getResponse() {
when(attributes.getResponse()).thenReturn(response);
assertEquals(response, ServletUtils.getResponse());
}
@Test
void getRequestAttributes() {
assertEquals(attributes, ServletUtils.getRequestAttributes());
}
@Test
void getSession() {
when(attributes.getRequest()).thenReturn(request);
when(request.getSession()).thenReturn(session);
assertEquals(session, ServletUtils.getSession());
}
}

@ -0,0 +1,48 @@
package com.jiuyv.sptccc.agile.common.utils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* StringUtil
*
* @author ren_chao
*/
class StringUtilTest {
@Test
void substring() {
String str = "testSubstring";
assertEquals("", StringUtil.substring(str, -1, -1));
assertEquals("g", StringUtil.substring(str, -1, 20));
assertEquals("", StringUtil.substring(str, 1, 0));
assertEquals("", StringUtil.substring(str, -20, -20));
assertEquals("", StringUtil.substring(null, 0, 0));
}
@Test
void getFileExtension() {
assertEquals("", StringUtil.getFileExtension(""));
assertEquals("", StringUtil.getFileExtension("a"));
assertEquals("txt", StringUtil.getFileExtension("a.txt"));
}
@Test
void getRecToken() {
assertEquals(10, StringUtil.getRecToken().length());
}
@Test
void strHide() {
assertEquals("1", StringUtil.strHide("1"));
assertEquals("13***", StringUtil.strHide("1333333"));
assertEquals("133****3333", StringUtil.strHide("13333333"));
}
@Test
void randomNumber() {
assertEquals(4, StringUtil.randomNumber(2).length());
}
}

@ -0,0 +1,56 @@
package com.jiuyv.sptccc.agile.domain;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.platform.commons.util.ReflectionUtils;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
/**
* Domain
*
* @author ren_chao
*/
@ExtendWith(MockitoExtension.class)
class TblDomainTest {
@Test
void domainTest() {
String packageName = "com.jiuyv.sptccc.agile.domain";
try {
invoke(packageName);
} catch (Exception e) {
Assertions.fail();
}
}
private void invoke(String packageName) throws InstantiationException, IllegalAccessException, InvocationTargetException {
List<Class<?>> classList = ReflectionUtils.findAllClassesInPackage(packageName, aClass -> true, s -> !s.endsWith("Test"));
for (Class<?> aClass : classList) {
Method[] methods = aClass.getDeclaredMethods();
Object o = aClass.newInstance();
for (Method method : methods) {
String name = method.getName();
if (name.startsWith("get")) {
method.invoke(o);
} else if (name.startsWith("set")) {
Class<?>[] parameterTypes = method.getParameterTypes();
Object[] objects = Arrays.stream(parameterTypes).map(p -> {
try {
return p.newInstance();
} catch (ReflectiveOperationException e) {
return null;
}
}).toArray();
method.invoke(o, objects);
}
}
}
}
}

@ -0,0 +1,57 @@
package com.jiuyv.sptccc.agile.dto;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.platform.commons.util.ReflectionUtils;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
/**
* DTO
*
* @author ren_chao
*/
@ExtendWith(MockitoExtension.class)
class PortalDTOTest {
@Test
void dtoTest() {
String packageName = "com.jiuyv.sptccc.agile.dto";
try {
invoke(packageName);
} catch (Exception e) {
Assertions.fail();
}
}
private void invoke(String packageName) throws InstantiationException, IllegalAccessException, InvocationTargetException {
List<Class<?>> classList = ReflectionUtils.findAllClassesInPackage(packageName, aClass -> true, s -> !s.endsWith("Test"));
for (Class<?> aClass : classList) {
Method[] methods = aClass.getDeclaredMethods();
Object o = aClass.newInstance();
for (Method method : methods) {
String name = method.getName();
if (name.startsWith("get")) {
method.invoke(o);
} else if (name.startsWith("set")) {
Class<?>[] parameterTypes = method.getParameterTypes();
Object[] objects = Arrays.stream(parameterTypes).map(p -> {
try {
return p.newInstance();
} catch (ReflectiveOperationException e) {
return null;
}
}).toArray();
method.invoke(o, objects);
}
}
}
}
}

@ -3,7 +3,7 @@
<parent>
<groupId>com.jiuyv.sptcc.agile</groupId>
<artifactId>agile-portal</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.2.4-SNAPSHOT</version>
</parent>
<artifactId>agile-portal-gateway</artifactId>
@ -85,6 +85,11 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
@ -114,14 +119,7 @@
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-bootstrap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-sleuth</artifactId>
@ -141,10 +139,42 @@
<artifactId>feign-okhttp</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.10</version>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>default-report</id>
<phase>prepare-package</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
@ -186,7 +216,7 @@
<configuration>
<filesets>
<fileset>
<directory>src/main/resources/public</directory>
<directory>src/main/resources/view</directory>
</fileset>
</filesets>
</configuration>
@ -203,7 +233,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources/public</outputDirectory>
<outputDirectory>src/main/resources/view</outputDirectory>
<overwrite>true</overwrite>
<resources>
<resource>

@ -7,7 +7,6 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.cloud.openfeign.EnableFeignClients;
/**
@ -16,7 +15,6 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
* @author admin
*/
@EnableFeignClients
@EnableDiscoveryClient
@EnableCaching
@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
public class PortalGatewayApplication {

@ -1,23 +0,0 @@
package com.jiuyv.sptccc.agile.common.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* *便
* password
*
* @author zhouliang
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface SensitiveData {
char defaultMark() default '*';//默认使用*
int firstLength() default 3;//保留前三位
int endLength() default 3;//保留后三位
}

@ -1,18 +0,0 @@
package com.jiuyv.sptccc.agile.common.config;
import org.springframework.boot.web.servlet.ServletContextInitializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class WebConfig {
/**
* JSESSIONIDPath"/"
*
*/
@Bean
public ServletContextInitializer servletContextInitializer() {
return servletContext -> servletContext.getSessionCookieConfig().setPath("/");
}
}

@ -1,152 +0,0 @@
package com.jiuyv.sptccc.agile.common.core.domain;
import java.io.Serializable;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonFormat;
/**
* Entity
*
* @author admin
*/
public class BaseEntity implements Serializable {
private static final long serialVersionUID = 1L;
/**
*
*/
private String createByName;
/**
*
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* id()
*/
private String createBy;
/**
*
*/
private String updateByName;
/**
* id()
*/
private String updateBy;
/**
*
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/**
*
*/
private String searchValue;
/**
*
*/
private String remark;
/**
*
*/
private Map<String, Object> params;
/**
* @return the createByName
*/
public String getCreateByName() {
return createByName;
}
/**
* @param createByName the createByName to set
*/
public void setCreateByName(String createByName) {
this.createByName = createByName;
}
/**
* @return the updateByName
*/
public String getUpdateByName() {
return updateByName;
}
/**
* @param updateByName the updateByName to set
*/
public void setUpdateByName(String updateByName) {
this.updateByName = updateByName;
}
public String getSearchValue() {
return searchValue == null ? "" : searchValue;
}
public void setSearchValue(String searchValue) {
this.searchValue = searchValue;
}
public String getCreateBy() {
return createBy == null ? "" : createBy;
}
public void setCreateBy(String createBy) {
this.createBy = createBy;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public String getUpdateBy() {
return updateBy == null ? "" : updateBy;
}
public void setUpdateBy(String updateBy) {
this.updateBy = updateBy;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
public String getRemark() {
return remark == null ? "" : remark;
}
public void setRemark(String remark) {
this.remark = remark;
}
public Map<String, Object> getParams() {
if (params == null) {
params = new HashMap<>();
}
return params;
}
public void setParams(Map<String, Object> params) {
this.params = params;
}
}

@ -1,32 +0,0 @@
package com.jiuyv.sptccc.agile.common.enums;
import java.util.HashMap;
import java.util.Map;
import org.springframework.lang.Nullable;
/**
*
*
* @author admin
*/
public enum HttpMethod {
GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS, TRACE;
private static final Map<String, HttpMethod> mappings = new HashMap<>(16);
static {
for (HttpMethod httpMethod : values()) {
mappings.put(httpMethod.name(), httpMethod);
}
}
@Nullable
public static HttpMethod resolve(@Nullable String method) {
return (method != null ? mappings.get(method) : null);
}
public boolean matches(String method) {
return (this == resolve(method));
}
}

@ -1,273 +0,0 @@
package com.jiuyv.sptccc.agile.common.utils;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonGenerator.Feature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.introspect.Annotated;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jiuyv.sptccc.agile.common.annotation.SensitiveData;
import com.jiuyv.sptccc.agile.common.utils.jackson.MaskSensitiveDataSerializerProvider;
/**
* @author zhouliang
*/
public abstract class JsonUtil {
/**
* The Constant LOGGER.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(JsonUtil.class);
private JsonUtil() {
throw new IllegalStateException("Utility class");
}
/**
* The object mapper.
*/
private static ObjectMapper objectMapper = new ObjectMapper();
//专门使用一个独立的,处理敏感字段等
private static ObjectMapper objectMapper2 = new ObjectMapper();
static {
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.setSerializationInclusion(Include.NON_NULL);
objectMapper.configure(Feature.WRITE_BIGDECIMAL_AS_PLAIN, true);
objectMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
objectMapper2.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper2.setSerializationInclusion(Include.NON_NULL);
objectMapper2.configure(Feature.WRITE_BIGDECIMAL_AS_PLAIN, true);
objectMapper2.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
// 启用自定义注解功能
objectMapper2.setAnnotationIntrospector(new JsonUtil.SensitiveDataIntrospector());
objectMapper2.setSerializerProvider(new MaskSensitiveDataSerializerProvider());
}
public static ObjectMapper JsonMapper() {
return objectMapper;
}
public static ObjectMapper JsonMapper2() {
return objectMapper2;
}
/**
* ,
*
* @author zhouliang
*/
public static class SensitiveDataIntrospector extends JacksonAnnotationIntrospector {
private static final long serialVersionUID = 1L;
@Override
public Object findSerializer(Annotated annotated) {
if (annotated.hasAnnotation(SensitiveData.class)) {
SensitiveData sensitiveDataProperty = annotated.getAnnotation(SensitiveData.class);
return new JsonSerializer<String>() {
@Override
public void serialize(String value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
if (value != null) {
String val = StringUtil.padl("", 6, sensitiveDataProperty.defaultMark());
//大于允许截取
if (value.length() * 2 / 3 >= (sensitiveDataProperty.firstLength() + sensitiveDataProperty.endLength())) {
gen.writeString(value.substring(0, sensitiveDataProperty.firstLength()) + val + value.substring(value.length() - sensitiveDataProperty.endLength()));
} else {
gen.writeString(val);
}
}
}
};
}
return super.findSerializer(annotated);
}
}
/**
* json
*
* @param object the object
* @return the string
*/
public static String toJSONString(Object object) {
try {
return objectMapper.writeValueAsString(object);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return "";
}
}
/**
* json
*
* @param <T>
* @param json
* @param clz
* @return
*/
public static <T> T json2Bean(String json, Class<T> clz) {
try {
return objectMapper.readValue(json, clz);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return null;
}
}
/**
* json
*
* @param <T>
* @param json
* @param clz
* @return
*/
public static <T> T json2Bean(String json, JavaType clz) {
try {
return objectMapper.readValue(json, clz);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return null;
}
}
/**
*
*
* @param <T>
* @param json
* @param clz
* @return
*/
public static <T> List<T> json2List(String json, Class<T> clz) {
try {
JavaType javaType = getCollectionType(ArrayList.class, clz);
return objectMapper.readValue(json, javaType);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return new ArrayList<>();
}
}
/**
* Collection Type
*
* @param collectionClass Collection
* @param elementClasses
* @return JavaType Java
* @since 1.0
*/
public static JavaType getCollectionType(Class<?> collectionClass, Class<?>... elementClasses) {
return objectMapper.getTypeFactory().constructParametricType(collectionClass, elementClasses);
}
/**
* jsonArrayNode
*
* @param json
* @return
*/
public static ArrayNode parseArray(String json) {
try {
return (ArrayNode) objectMapper.readTree(json);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return null;
}
}
/**
* jsonObjectNode
*
* @param json
* @return
*/
public static ObjectNode parseObject(String json) {
try {
return (ObjectNode) objectMapper.readTree(json);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return null;
}
}
/**
* ArrayNode
*
* @param json
* @return
*/
public static ArrayNode createArray() {
return objectMapper.createArrayNode();
}
/**
* ObjectNode
*
* @param json
* @return
*/
public static ObjectNode createObject() {
return objectMapper.createObjectNode();
}
/**
* ArrayNode
*
* @param json
* @return
*/
public static ArrayNode toArray(Object obj) {
try {
return objectMapper.valueToTree(obj);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return createArray();
}
}
/**
* ObjectNode
*
* @param json
* @return
*/
public static ObjectNode toObject(Object obj) {
try {
return objectMapper.valueToTree(obj);
} catch (Exception e) {
LOGGER.error("convert failed", e);
return createObject();
}
}
/**
*
* Node
*/
public static <K, V> void copyProperties(K source, V target) {
try {
objectMapper.updateValue(target, source);
} catch (Exception e) {
LOGGER.error("convert failed", e);
}
}
}

@ -1,126 +0,0 @@
package com.jiuyv.sptccc.agile.common.utils;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Random;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.CollectionUtils;
/**
*
*
* @author admin
*/
public class StringUtil {
private static final Random random = new Random();
/**
*
*
* @param str
* @param strs
* @return
*/
public static boolean matches(String str, List<String> strs) {
if (StringUtils.isBlank(str) || CollectionUtils.isEmpty(strs)) {
return false;
}
for (String pattern : strs) {
if (isMatch(pattern, str)) {
return true;
}
}
return false;
}
/**
* url:
* ? ;
* * ;
* ** ;
*
* @param pattern
* @param url url
*/
public static boolean isMatch(String pattern, String url) {
AntPathMatcher matcher = new AntPathMatcher();
return matcher.match(pattern, url);
}
/**
* ssizesize
*
* @param s
* @param size
* @param c
* @return
*/
public static String padl(final String s, final int size, final char c) {
final StringBuilder sb = new StringBuilder(size);
if (s != null) {
final int len = s.length();
if (s.length() <= size) {
for (int i = size - len; i > 0; i--) {
sb.append(c);
}
sb.append(s);
} else {
return s.substring(len - size, len);
}
} else {
for (int i = size; i > 0; i--) {
sb.append(c);
}
}
return sb.toString();
}
/**
*
*
*/
public static String strHide(String str) {
if (str == null || str.length() <= 2) {
return str;
}
if (str.length() <= 7) {
return str.substring(0, 2) + "***";
}
return str.substring(0, 3) + "****" + str.substring(str.length() - 4);
}
/**
*
*
*/
public static String randomNumber(int length) {
if (length < 4) {//最少4位
length = 4;
}
StringBuilder bld = new StringBuilder();
for (int i = 0; i < length; i++) {
bld.append(random.nextInt(9)); //生成随机数
}
return bld.toString();
}
/**
* URL
*
*/
public static String encoderURL(String str) {
try {
return URLEncoder.encode(str, StandardCharsets.UTF_8.toString()).replace("+", "%20");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
throw new ServiceException("编码异常");
}
}
}

@ -1,75 +0,0 @@
package com.jiuyv.sptccc.agile.common.utils;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 线.
*
* @author admin
*/
public class Threads {
private static final Logger logger = LoggerFactory.getLogger(Threads.class);
/**
* sleep,
*/
public static void sleep(long milliseconds) {
try {
Thread.sleep(milliseconds);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
/**
* 线 使shutdown, . , shutdownNow,
* workQueuePending,. 退. shutdown线.
*/
public static void shutdownAndAwaitTermination(ExecutorService pool) {
if (pool != null && !pool.isShutdown()) {
pool.shutdown();
try {
if (!pool.awaitTermination(120, TimeUnit.SECONDS)) {
pool.shutdownNow();
if (!pool.awaitTermination(120, TimeUnit.SECONDS)) {
logger.info("Pool did not terminate");
}
}
} catch (InterruptedException ie) {
pool.shutdownNow();
Thread.currentThread().interrupt();
}
}
}
/**
* 线
*/
public static void printException(Runnable r, Throwable t) {
if (t == null && r instanceof Future<?>) {
try {
Future<?> future = (Future<?>) r;
if (future.isDone()) {
future.get();
}
} catch (CancellationException ce) {
t = ce;
} catch (ExecutionException ee) {
t = ee.getCause();
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
}
if (t != null) {
logger.error(t.getMessage(), t);
}
}
}

@ -1,49 +0,0 @@
package com.jiuyv.sptccc.agile.common.utils.ip;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jiuyv.sptccc.agile.common.config.ConsoleConfig;
import com.jiuyv.sptccc.agile.common.utils.JsonUtil;
import com.jiuyv.sptccc.agile.common.utils.SpringUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.web.client.RestTemplate;
/**
*
*
* @author admin
*/
public class AddressUtils {
// IP地址查询
public static final String IP_URL = "http://whois.pconline.com.cn/ipJson.jsp";
// 未知地址
public static final String UNKNOWN = "XX XX";
private static final Logger log = LoggerFactory.getLogger(AddressUtils.class);
private static final ConsoleConfig consoleProperties;
private static final RestTemplate restTemplate;
static {
consoleProperties = SpringUtils.getBean(ConsoleConfig.class);
restTemplate = SpringUtils.getBean(RestTemplateBuilder.class).build();
}
public static String getRealAddressByIP(String ip) {
// 内网不查询
if (IpUtils.internalIp(ip)) {
return "内网IP";
}
if (consoleProperties.isAddressEnabled()) {
String rspStr = restTemplate.getForEntity(IP_URL + "?ip=" + ip + "&json=true", String.class).getBody();
if (StringUtils.isBlank(rspStr)) {
log.error("获取地理位置异常 {}", ip);
return UNKNOWN;
}
ObjectNode obj = JsonUtil.parseObject(rspStr);
assert obj != null;
return obj.get("pro").asText() + " " + obj.get("city").asText();
}
return UNKNOWN;
}
}

@ -1,168 +0,0 @@
package com.jiuyv.sptccc.agile.common.utils.jackson;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.jiuyv.sptccc.agile.common.utils.StringUtil;
import org.apache.commons.lang3.ObjectUtils;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.BeanProperty;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializationConfig;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.DefaultSerializerProvider;
import com.fasterxml.jackson.databind.ser.SerializerFactory;
/**
* *便
*
*
*
* @author zhouliang
*/
public class MaskSensitiveDataSerializerProvider extends DefaultSerializerProvider {
private static final long serialVersionUID = 1L;
//开始或结尾是,就认为是敏感字段​
private static final String LIKE_FILEDS = "password,email,phone";
//明确敏感字段
private static final String STATIC_FILEDS = "idCard,socialCreditCode";
private static final Set<String> STATIC_FILED_SET = new HashSet<>(
Arrays.stream(STATIC_FILEDS.split(","))
.map(String::trim)
.filter(o -> !ObjectUtils.isEmpty(o))
.collect(Collectors.toSet())
);
private static final Set<String> LIKE_FILED_SET = new HashSet<>(
Arrays.stream(LIKE_FILEDS.split(","))
.map(String::trim)
.filter(o -> !ObjectUtils.isEmpty(o))
.collect(Collectors.toSet())
);
public MaskSensitiveDataSerializerProvider() {
super();
}
public MaskSensitiveDataSerializerProvider(MaskSensitiveDataSerializerProvider src) {
super(src);
}
protected MaskSensitiveDataSerializerProvider(SerializerProvider src, SerializationConfig config, SerializerFactory f) {
super(src, config, f);
}
@Override
public DefaultSerializerProvider copy() {
if (this.getClass() != MaskSensitiveDataSerializerProvider.class) {
return super.copy();
}
return new MaskSensitiveDataSerializerProvider(this);
}
@Override
public MaskSensitiveDataSerializerProvider createInstance(SerializationConfig config, SerializerFactory jsf) {
return new MaskSensitiveDataSerializerProvider(this, config, jsf);
}
@Override
public JsonSerializer<Object> findValueSerializer(Class<?> valueType, BeanProperty property)
throws JsonMappingException {
JsonSerializer<Object> target = findSerializerByBeanProperty(property);
if (target != null) {
return target;
}
return super.findValueSerializer(valueType, property);
}
@Override
public JsonSerializer<Object> findValueSerializer(JavaType valueType, BeanProperty property)
throws JsonMappingException {
JsonSerializer<Object> target = findSerializerByBeanProperty(property);
if (target != null) {
return target;
}
return super.findValueSerializer(valueType, property);
}
@Override
public JsonSerializer<Object> findPrimaryPropertySerializer(JavaType valueType, BeanProperty property)
throws JsonMappingException {
JsonSerializer<Object> target = findSerializerByBeanProperty(property);
if (target != null) {
return target;
}
return super.findPrimaryPropertySerializer(valueType, property);
}
@Override
public JsonSerializer<Object> findPrimaryPropertySerializer(Class<?> valueType, BeanProperty property)
throws JsonMappingException {
JsonSerializer<Object> target = findSerializerByBeanProperty(property);
if (target != null) {
return target;
}
return super.findPrimaryPropertySerializer(valueType, property);
}
private JsonSerializer<Object> findSerializerByBeanProperty(BeanProperty property) {
String propertyName = Optional.ofNullable(property).map(BeanProperty::getName).orElse("");
if (STATIC_FILED_SET.contains(propertyName)) {
return MaskSerializer.INSTANCE;
} else {
for (String f : LIKE_FILED_SET) {
Pattern p = Pattern.compile("(^" + f + ")|(" + f + "$)", Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(propertyName);
if (m.find()) {
return MaskSerializer.INSTANCE;
}
}
}
return null;
}
static class MaskSerializer extends JsonSerializer<Object> {
public static final MaskSerializer INSTANCE = new MaskSerializer();
private MaskSerializer() {
}
@Override
public void serialize(Object value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
if (value instanceof String) {
String value2 = (String) value;
String val = StringUtil.padl("", 6, '*');
//大于允许截取
if (value2.length() * 2 / 3 >= 6) {
gen.writeString(value2.substring(0, 3) + val + value2.substring(value2.length() - 3));
} else {
gen.writeString(val);
}
}
}
}
public static Set<String> getStaticFields() {
return STATIC_FILED_SET;
}
public static Set<String> getLikeFields() {
return LIKE_FILED_SET;
}
}

@ -5,6 +5,6 @@ import org.springframework.cloud.openfeign.FeignClient;
import static com.jiuyv.sptccc.agile.api.DockerApplyFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "dockerApplyFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface DockerApplyFeign extends DockerApplyFeignApi {
}

@ -8,9 +8,9 @@ import org.springframework.web.bind.annotation.PathVariable;
import static com.jiuyv.sptccc.agile.api.DockerDownloadApplyFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "dockerDownloadApplyFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface DockerDownloadApplyFeign extends DockerDownloadApplyFeignApi {
@GetMapping("/download/{downloadApplyId}")
Response download(@PathVariable("downloadApplyId") Long downloadApplyId);
@GetMapping("/download/{downloadApplyId}/{isQuery}")
Response download(@PathVariable("downloadApplyId") Long downloadApplyId, @PathVariable("isQuery") Boolean isQuery);
}

@ -5,6 +5,6 @@ import org.springframework.cloud.openfeign.FeignClient;
import static com.jiuyv.sptccc.agile.api.DockerWithUserFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "dockerWithUserFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface DockerWithUserFeign extends DockerWithUserFeignApi {
}

@ -4,6 +4,6 @@ package com.jiuyv.sptccc.agile.feign.portal;
import com.jiuyv.sptccc.agile.feign.portal.transit.ExternalInterface;
import org.springframework.cloud.openfeign.FeignClient;
@FeignClient("${remoteUrl.gateway}")
@FeignClient(name = "externalInterfaceFeign", url = "${remoteUrl.gateway}")
public interface ExternalInterfaceFeign extends ExternalInterface {
}

@ -8,7 +8,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import static com.jiuyv.sptccc.agile.api.ContentFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "PortalContentFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface PortalContentFeign extends ContentFeignApi {
@GetMapping("/sdkDownload")

@ -5,6 +5,6 @@ import org.springframework.cloud.openfeign.FeignClient;
import static com.jiuyv.sptccc.agile.api.PortalLogFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "portalLogFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface PortalLogFeign extends PortalLogFeignApi {
}

@ -10,6 +10,6 @@ import static com.jiuyv.sptccc.agile.api.PortalMessageApi.API_PATH_PREFIX;
*
* @author ren_chao
*/
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "portalMessageFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface PortalMessageFeign extends PortalMessageApi {
}

@ -5,6 +5,6 @@ import org.springframework.cloud.openfeign.FeignClient;
import static com.jiuyv.sptccc.agile.api.PortalUserFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "portalUserFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface PortalUserFeign extends PortalUserFeignApi {
}

@ -6,6 +6,6 @@ import org.springframework.cloud.openfeign.FeignClient;
import static com.jiuyv.sptccc.agile.api.FileFeignApi.API_PATH_PREFIX;
@FeignClient("${remoteUrl.gateway}" + API_PATH_PREFIX)
@FeignClient(name = "resourceFeign", url = "${remoteUrl.gateway}", path = API_PATH_PREFIX)
public interface ResourceFeign extends FileFeignApi {
}

@ -1,10 +1,10 @@
package com.jiuyv.sptccc.agile.feign.portal.config;
import com.anji.captcha.util.MD5Util;
import com.jiuyv.sptccc.agile.common.core.domain.model.LoginUser;
import com.jiuyv.sptccc.agile.common.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
import com.jiuyv.sptccc.agile.common.utils.SecurityUtils;
import com.jiuyv.sptccc.agile.constant.FeignApiConstant;
import com.jiuyv.sptccc.agile.portal.domain.LoginUser;
import com.jiuyv.sptccc.agile.portal.utils.SecurityUtils;
import feign.RequestInterceptor;
import feign.Target;
import feign.codec.ErrorDecoder;

@ -1,13 +1,12 @@
package com.jiuyv.sptccc.agile.framework.aspectj;
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
import com.jiuyv.sptccc.agile.common.utils.IpUtils;
import com.jiuyv.sptccc.agile.common.utils.ServletUtils;
import com.jiuyv.sptccc.agile.portal.domain.LoginUser;
import com.jiuyv.sptccc.agile.portal.utils.SecurityUtils;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.AfterThrowing;
@ -17,18 +16,17 @@ import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.validation.BindingResult;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.HandlerMapping;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.jiuyv.sptccc.agile.common.core.domain.model.LoginUser;
import com.jiuyv.sptccc.agile.common.enums.HttpMethod;
import com.jiuyv.sptccc.agile.common.utils.JsonUtil;
import com.jiuyv.sptccc.agile.common.utils.SecurityUtils;
import com.jiuyv.sptccc.agile.common.utils.ServletUtils;
import com.jiuyv.sptccc.agile.common.utils.ip.IpUtils;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.Map;
/**
*
@ -42,6 +40,12 @@ public class GatewayGlobalLogAspect {
private static final ThreadLocal<Long> timeTreadLocal = new ThreadLocal<>();
private static final double THOUSAND = 1000.0;
private final ObjectMapper objectMapper;
public GatewayGlobalLogAspect(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
//请求Controller输出日志
@Pointcut("execution(* *..*.*Controller.*(..)) && !@annotation(com.jiuyv.sptccc.agile.common.annotation.LogIgnore)")
public void controllerAspect() {}
@ -81,7 +85,7 @@ public class GatewayGlobalLogAspect {
String returnstr = null;
if (result != null) {
try {
returnstr = JsonUtil.JsonMapper2().writeValueAsString(result);
returnstr = objectMapper.writeValueAsString(result);
} catch (JsonProcessingException e) {
LOGGER.debug("返回转换异常", e);
returnstr = "返回转换异常";
@ -143,7 +147,7 @@ public class GatewayGlobalLogAspect {
for (Object o : paramsArray) {
if (o != null && !isFilterObject(o)) {
try {
Object jsonObj = JsonUtil.JsonMapper2().writeValueAsString(o);
Object jsonObj = objectMapper.writeValueAsString(o);
params.append(jsonObj.toString()).append(" ");
} catch (JsonProcessingException e) {
LOGGER.debug("参数拼装出错:{}", e.getMessage(), e);

@ -0,0 +1,89 @@
package com.jiuyv.sptccc.agile.framework.aspectj;
import com.jiuyv.sptccc.agile.common.constant.CacheNames;
import com.jiuyv.sptccc.agile.common.exception.ServiceException;
import com.jiuyv.sptccc.agile.common.utils.ServletUtils;
import com.jiuyv.sptccc.agile.framework.config.LocalCache;
import com.jiuyv.sptccc.agile.portal.domain.LoginUser;
import com.jiuyv.sptccc.agile.portal.utils.SecurityUtils;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.AfterThrowing;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import javax.servlet.http.HttpServletRequest;
/**
*
* 30
*
* @author zhouliang
*/
@Aspect
@Order(2)
@Component
public class RepeatSubmitAspect {
private static final Logger LOGGER = LoggerFactory.getLogger(RepeatSubmitAspect.class);
private static final ThreadLocal<String> repeatSubmitTreadLocal = new ThreadLocal<>();
@Autowired
private LocalCache localCache;
@Pointcut("execution(* *..*.*Controller.*(..)) && !@annotation(com.jiuyv.sptccc.agile.common.annotation.NoRepeatSubmit)")
public void controllerAspect() {}
//请求日志记录
@Before("controllerAspect()")
public void before() {
//获取请求的request
HttpServletRequest request = ServletUtils.getRequest();
if (HttpMethod.GET.name().equals(request.getMethod())) {
return;
}
String userName = "none";
try {
LoginUser loginuser = SecurityUtils.getLoginUser();
if (loginuser != null) {
userName = loginuser.getUserId().toString();
}
} catch (Exception e) {
//没有用户的直接跳过
return;
}
String repeatSubmitKey = userName + "_" + request.getMethod() + "_" + request.getRequestURI();
String lockFlag = localCache.getValueOfCacheName(CacheNames.CACHE_30S, repeatSubmitKey, String.class);
if (StringUtils.isNotBlank(lockFlag)) {
LOGGER.info("RepeatSubmitAspect={}", repeatSubmitKey);
throw new ServiceException("请勿连续提交,请求正在处理中...");
}
repeatSubmitTreadLocal.set(repeatSubmitKey);
localCache.setValueOfCacheName(CacheNames.CACHE_30S, repeatSubmitKey, "1");
}
//后置返回
@AfterReturning("controllerAspect()")
public void doAfterReturning() {
if (repeatSubmitTreadLocal.get() != null) {
localCache.removeValueOfCacheName(CacheNames.CACHE_30S, repeatSubmitTreadLocal.get());
repeatSubmitTreadLocal.remove();
}
}
//后置异常返回
@AfterThrowing("controllerAspect()")
public void doAfterThrowing() {
if (repeatSubmitTreadLocal.get() != null) {
localCache.removeValueOfCacheName(CacheNames.CACHE_30S, repeatSubmitTreadLocal.get());
repeatSubmitTreadLocal.remove();
}
}
}

@ -1,4 +1,4 @@
package com.jiuyv.sptccc.agile.common.config;
package com.jiuyv.sptccc.agile.framework.config;
import com.anji.captcha.model.common.Const;
import com.anji.captcha.service.CaptchaCacheService;

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save