msb_6557
/
opsli-boot
mirror of https://github.com/hiparker/opsli-boot.git
1
1
Code Issues Projects Releases Wiki Activity

导入Excel 整体性能优化

Browse Source
v1.4.1
Parker 4 years ago
parent 536f6feb9d
commit a106b6e28c
12 changed files with 259 additions and 57 deletions
Show Stats Download Patch File Download Diff File

3
opsli-base-support/opsli-core/src/main/java/org/opsli/core/autoconfigure/properties/GlobalProperties.java
Unescape View File

@ -127,9 +127,6 @@ public class GlobalProperties {
@EqualsAndHashCode(callSuper = false)
public static class Excel {
/** 最大导入操作数 */
private Integer importMaxCount;
/** 最大导出操作数 */
private Integer exportMaxCount;

65
opsli-base-support/opsli-core/src/main/java/org/opsli/core/base/controller/BaseRestController.java
Unescape View File

@ -21,6 +21,8 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.util.CollectionUtils;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@ -33,6 +35,7 @@ import org.opsli.api.wrapper.system.user.UserModel;
import org.opsli.common.annotation.RequiresPermissionsCus;
import org.opsli.common.annotation.hotdata.EnableHotData;
import org.opsli.common.constants.CacheConstants;
import org.opsli.common.enums.ExcelOperate;
import org.opsli.common.exception.ServiceException;
import org.opsli.common.exception.TokenException;
import org.opsli.common.msg.CommonMsg;
@ -49,6 +52,7 @@ import org.opsli.core.utils.DistributedLockUtil;
import org.opsli.core.utils.ExcelUtil;
import org.opsli.core.utils.UserUtil;
import org.opsli.plugins.excel.exception.ExcelPluginException;
import org.opsli.plugins.excel.listener.BatchExcelListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestParam;
@ -59,6 +63,7 @@ import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
import java.lang.reflect.Method;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@ -193,43 +198,39 @@ public abstract class BaseRestController <T extends BaseEntity, E extends ApiWra
return ResultVo.error(CoreMsg.EXCEL_FILE_NULL.getCode(),
CoreMsg.EXCEL_FILE_NULL.getMessage());
}
ResultVo<?> resultVo;
String msgInfo;
ResultVo<?> resultVo ;
String msgInfo = "";
try {
List<E> modelList = ExcelUtil.getInstance().readExcel(files.get(0), modelClazz);
if(CollUtil.isNotEmpty(modelList)){
// 导入数量限制 -1 为无限制
Integer importMaxCount = globalProperties.getExcel().getImportMaxCount();
if(importMaxCount != null && importMaxCount > -1){
if(modelList.size() > importMaxCount){
String maxError = StrUtil.format(CoreMsg.EXCEL_HANDLE_MAX.getMessage(), modelList.size(),
importMaxCount);
// 清空 list
modelList.clear();
// 超出最大导出数量
throw new ExcelPluginException(CoreMsg.EXCEL_HANDLE_MAX.getCode(), maxError);
UserModel user = UserUtil.getUser();
Date currDate = DateUtil.date();
// 导入优化为 监听器 模式 超过一定阈值直接释放资源 防止导入数据导致系统 OOM
ExcelUtil.getInstance().readExcelByListener(files.get(0), modelClazz, new BatchExcelListener<E>() {
@Override
public void saveData(List<E> dataList) {
// 处理字典数据
List<E> disposeData = ExcelUtil.getInstance().handleDatas(dataList, modelClazz, ExcelOperate.READ);
// 手动赋值 必要数据 防止频繁开启Redis网络IO
for (E model : disposeData) {
model.setIzManual(true);
model.setCreateBy(user.getId());
model.setUpdateBy(user.getId());
model.setCreateTime(currDate);
model.setUpdateTime(currDate);
}
// 数据库插入数据
IService.insertBatch(disposeData);
}
});
// 花费毫秒数
long timerCount = timer.interval();
// 提示信息
msgInfo = StrUtil.format(CoreMsg.EXCEL_IMPORT_SUCCESS.getMessage(), DateUtil.formatBetween(timerCount));
// 导出成功
resultVo = ResultVo.success(msgInfo);
resultVo.setCode(CoreMsg.EXCEL_IMPORT_SUCCESS.getCode());
boolean ret = IService.insertBatch(modelList);
if(!ret){
// 清空 list
modelList.clear();
throw new ExcelPluginException(CoreMsg.EXCEL_IMPORT_NO);
}
// 清空 list
modelList.clear();
// 花费毫秒数
long timerCount = timer.interval();
// 提示信息
msgInfo = StrUtil.format(CoreMsg.EXCEL_IMPORT_SUCCESS.getMessage(), DateUtil.formatBetween(timerCount));
// 导出成功
resultVo = ResultVo.success(msgInfo);
resultVo.setCode(CoreMsg.EXCEL_IMPORT_SUCCESS.getCode());
}else {
throw new ExcelPluginException(CoreMsg.EXCEL_FILE_NULL);
}
} catch (ExcelPluginException e) {
// 花费毫秒数
long timerCount = timer.interval();

23
opsli-base-support/opsli-core/src/main/java/org/opsli/core/filters/interceptor/MybatisAutoFillInterceptor.java
Unescape View File

@ -30,6 +30,7 @@ import org.apache.ibatis.mapping.SqlCommandType;
import org.apache.ibatis.plugin.*;
import org.opsli.common.constants.MyBatisConstants;
import org.opsli.core.utils.UserUtil;
import org.opsli.core.utils.excel.factory.AbstractModelHelper;
import org.springframework.stereotype.Component;
import java.lang.reflect.Field;
@ -58,6 +59,9 @@ public class MybatisAutoFillInterceptor implements Interceptor {
private static final String ET = "et";
/** 实体类字段 */
static private final Map<Class<?>, Field[]> ENTITY_FIELD_MAP = new HashMap<>();
@Override
public Object intercept(Invocation invocation) throws IllegalAccessException, InvocationTargetException {
fillField(invocation);
@ -110,7 +114,14 @@ public class MybatisAutoFillInterceptor implements Interceptor {
// 当前时间
Date currDate = DateUtil.date();
Field[] fields = ReflectUtil.getFields(arg.getClass());
// 字段缓存 减少每次更新 反射
Field[] fields = ENTITY_FIELD_MAP.get(arg.getClass());
if(fields == null){
fields = ReflectUtil.getFields(arg.getClass());
ENTITY_FIELD_MAP.put(arg.getClass(), fields);
}
for (Field f : fields) {
// 判断是否是排除字段
if(existField.contains(f.getName())){
@ -189,7 +200,6 @@ public class MybatisAutoFillInterceptor implements Interceptor {
// 2020-09-19
// 修改这儿 有可能会拿到一个 MapperMethod，需要特殊处理
Field[] fields;
if (arg instanceof MapperMethod.ParamMap) {
MapperMethod.ParamMap<?> paramMap = (MapperMethod.ParamMap<?>) arg;
if (paramMap.containsKey(ET)) {
@ -201,7 +211,14 @@ public class MybatisAutoFillInterceptor implements Interceptor {
return;
}
}
fields = ReflectUtil.getFields(arg.getClass());
// 字段缓存 减少每次更新 反射
Field[] fields = ENTITY_FIELD_MAP.get(arg.getClass());
if(fields == null){
fields = ReflectUtil.getFields(arg.getClass());
ENTITY_FIELD_MAP.put(arg.getClass(), fields);
}
for (Field f : fields) {
// 判断是否是排除字段
if(existField.contains(f.getName())){

22
opsli-base-support/opsli-core/src/main/java/org/opsli/core/utils/ExcelUtil.java
Unescape View File

@ -34,6 +34,7 @@ import org.opsli.core.utils.excel.factory.ModelFactoryHelper;
import org.opsli.plugins.excel.ExcelPlugin;
import org.opsli.plugins.excel.annotation.ExcelInfo;
import org.opsli.plugins.excel.exception.ExcelPluginException;
import org.opsli.plugins.excel.listener.BatchExcelListener;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
@ -93,6 +94,25 @@ public final class ExcelUtil {
return this.handleDatas(ts, rowModel, ExcelOperate.READ);
}
///////////////////////
public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel,
BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, batchExcelListener);
}
public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName,
BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, sheetName, batchExcelListener);
}
public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName, int headLineNum,
BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, sheetName, headLineNum, batchExcelListener);
}
///////////////////////
public <T> void writeExcel(HttpServletResponse response, List<T> list, String fileName, String sheetName, Class<T> classType, ExcelTypeEnum excelTypeEnum) throws ExcelPluginException {
// 处理数据
List<T> ts = this.handleDatas(list, classType, ExcelOperate.WRITE);
@ -107,7 +127,7 @@ public final class ExcelUtil {
* @param <T> 泛型
* @return List<T>
*/
private <T> List<T> handleDatas(List<T> datas, Class<T> typeClazz, ExcelOperate operate){
public <T> List<T> handleDatas(List<T> datas, Class<T> typeClazz, ExcelOperate operate){
// 计时器
TimeInterval timer = DateUtil.timer();
// 空处理

8
opsli-base-support/opsli-core/src/main/resources/META-INF/additional-spring-configuration-metadata.json
Unescape View File

@ -157,14 +157,6 @@
},
{
"name": "opsli.excel.import-max-count",
"sourceType": "org.opsli.core.autoconfigure.properties.GlobalProperties$Excel",
"type": "java.lang.Integer",
"defaultValue": 20000,
"description": "Excel 导入最大操作数."
},
{
"name": "opsli.excel.export-max-count",
"sourceType": "org.opsli.core.autoconfigure.properties.GlobalProperties$Excel",

76
opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/ExcelPlugin.java
Unescape View File

@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.opsli.common.utils.WrapperUtil;
import org.opsli.plugins.excel.exception.ExcelPluginException;
import org.opsli.plugins.excel.listener.BatchExcelListener;
import org.opsli.plugins.excel.listener.ExcelListener;
import org.opsli.plugins.excel.msg.ExcelMsg;
import org.springframework.web.multipart.MultipartFile;
@ -84,7 +85,7 @@ public class ExcelPlugin {
*/
public <T> List<T> readExcel(MultipartFile excel, Class<T> rowModel, String sheetName,
int headLineNum) throws ExcelPluginException {
ExcelListener excelListener = new ExcelListener();
ExcelListener<T> excelListener = new ExcelListener<>();
InputStream inputStream = null;
try{
if(null != excel){
@ -115,6 +116,79 @@ public class ExcelPlugin {
return getExtendsBeanList(excelListener.getDataList(), rowModel);
}
// ==================================================================================
/**
* 读取 Excel(多个 sheet)
* 将多sheet合并成一个list数据集，通过自定义ExcelReader继承AnalysisEventListener
* 重写invoke doAfterAllAnalysed方法
* getExtendsBeanList 主要是做Bean的属性拷贝 ，可以通过ExcelReader中添加的数据集直接获取
* @param excel 文件
* @param rowModel 实体类映射，继承 BaseRowModel 类
* @param batchExcelListener 监听器
*/
public <T> void readExcelByListener(MultipartFile excel,Class<T> rowModel,
BatchExcelListener<T> batchExcelListener)
throws ExcelPluginException {
readExcelByListener(excel, rowModel, null, 1, batchExcelListener);
}
/**
* 读取某个 sheet 的 Excel
* @param excel 文件
* @param rowModel 实体类映射，继承 BaseRowModel 类
* @param sheetName sheet 的序号 从1开始
* @param batchExcelListener 监听器
*/
public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName,
BatchExcelListener<T> batchExcelListener)
throws ExcelPluginException{
readExcelByListener(excel, rowModel, sheetName, 1, batchExcelListener);
}
/**
* 读取某个 sheet 的 Excel
* @param excel 文件
* @param rowModel 实体类映射，继承 BaseRowModel 类
* @param sheetName sheet 的序号 从1开始
* @param headLineNum 表头行数，默认为1
* @param batchExcelListener 监听器
*/
public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName,
int headLineNum, BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
if(null == batchExcelListener){
return;
}
InputStream inputStream = null;
try{
if(null != excel){
inputStream = excel.getInputStream();
}
}catch (IOException e){
log.error(e.getMessage(),e);
}
if(null == inputStream){
return;
}
ExcelReader excelReader = EasyExcel.read(inputStream, rowModel, batchExcelListener).build();
if (excelReader == null) {
return;
}
ReadSheet readSheet;
if(StringUtils.isEmpty(sheetName)){
readSheet = EasyExcel.readSheet().build();
}else{
readSheet = EasyExcel.readSheet(sheetName).build();
}
readSheet.setHeadRowNumber(headLineNum);
excelReader.read(readSheet);
// 这里千万别忘记关闭，读的时候会创建临时文件，到时磁盘会崩的
excelReader.finish();
}
/**
* 导出 Excel ：一个 sheet，带表头
* 自定义WriterHandler 可以定制行列数据进行灵活化操作

99
opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/listener/BatchExcelListener.java
Unescape View File

@ -0,0 +1,99 @@
/**
* Copyright 2020 OPSLI 快速开发平台 https://www.opsli.com
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.opsli.plugins.excel.listener;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
/**
* Excel 监听
* @author parker
*/
@Slf4j
public abstract class BatchExcelListener<T> extends AnalysisEventListener<T> {
private static final String SERIAL_VERSION_UID = "serialVersionUID";
/**
* 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
*/
private static final int BATCH_COUNT = 2000;
private final List<Object> dataList = new ArrayList<>();
/**
* 钩子函数 需要调用方重写 保存方法
* @param dataList 数据集合
*/
abstract public void saveData(List<T> dataList);
/**
* 通过 AnalysisContext 对象还可以获取当前 sheet，当前行等数据
*/
@Override
public void invoke(Object object, AnalysisContext context) {
if(!checkObjAllFieldsIsNull(object)) {
dataList.add(object);
// 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM
if (dataList.size() >= BATCH_COUNT) {
saveData((List<T>) dataList);
// 存储完成清理 list
dataList.clear();
}
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
// 这里也要保存数据，确保最后遗留的数据也存储到数据库
saveData((List<T>) dataList);
}
/**
* 判断对象中属性值是否全为空
*/
private static boolean checkObjAllFieldsIsNull(Object object) {
if (null == object) {
return true;
}
try {
for (Field f : object.getClass().getDeclaredFields()) {
f.setAccessible(true);
//只校验带ExcelProperty注解的属性
ExcelProperty property = f.getAnnotation(ExcelProperty.class);
if(property == null || SERIAL_VERSION_UID.equals(f.getName())){
continue;
}
if (f.get(object) != null && StringUtils.isNotBlank(f.get(object).toString())) {
return false;
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
//do something
}
return true;
}
}

6
opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/listener/ExcelListener.java
Unescape View File

@ -18,7 +18,9 @@ package org.opsli.plugins.excel.listener;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.formula.functions.T;
import java.lang.reflect.Field;
import java.util.ArrayList;
@ -28,7 +30,8 @@ import java.util.List;
* Excel 监听
* @author parker
*/
public class ExcelListener extends AnalysisEventListener {
@Slf4j
public class ExcelListener<T> extends AnalysisEventListener<T> {
private static final String SERIAL_VERSION_UID = "serialVersionUID";
@ -69,6 +72,7 @@ public class ExcelListener extends AnalysisEventListener {
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
//do something
}
return true;

4
opsli-starter/src/main/resources/application-beta.yaml
Unescape View File

@ -33,13 +33,13 @@ spring:
#primary: master
datasource:
master:
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
username: root
password: MYSQL0p9o8i7u
driver-class-name: com.mysql.cj.jdbc.Driver
# 多数据源配置
#slave-datasource:
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
#username: root
#password: 12345678
#driver-class-name: com.mysql.cj.jdbc.Driver

4
opsli-starter/src/main/resources/application-dev.yaml
Unescape View File

@ -33,13 +33,13 @@ spring:
#primary: master
datasource:
master:
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
username: root
password: 12345678
driver-class-name: com.mysql.cj.jdbc.Driver
# 多数据源配置
#slave-datasource:
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
#username: root
#password: 12345678
#driver-class-name: com.mysql.cj.jdbc.Driver

4
opsli-starter/src/main/resources/application-release.yaml
Unescape View File

@ -33,13 +33,13 @@ spring:
#primary: master
datasource:
master:
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
username: root
password: 12345678
driver-class-name: com.mysql.cj.jdbc.Driver
# 多数据源配置
#slave-datasource:
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
#url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
#username: root
#password: 12345678
#driver-class-name: com.mysql.cj.jdbc.Driver

2
opsli-starter/src/main/resources/application.yaml
Unescape View File

@ -209,7 +209,5 @@ opsli:
# Excel
excel:
# Excel 最大导入操作数量 防止OOM -1为无限制
import-max-count: 5000
# Excel 最大导出操作数量 防止OOM -1为无限制
export-max-count: 50000

Write Preview
Loading…
Cancel
Save