导入Excel 整体性能优化

opsli-base-support/opsli-core/src/main/java/org/opsli/core/autoconfigure/properties/GlobalProperties.java

@@ -127,9 +127,6 @@ public class GlobalProperties {
     @EqualsAndHashCode(callSuper = false)
     public static class Excel {
 
-        /** 最大导入操作数 */
-        private Integer importMaxCount;
-
         /** 最大导出操作数 */
         private Integer exportMaxCount;
 

opsli-base-support/opsli-core/src/main/java/org/opsli/core/base/controller/BaseRestController.java

@@ -21,6 +21,8 @@ import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.date.TimeInterval;
 import cn.hutool.core.util.StrUtil;
+import cn.hutool.json.JSONUtil;
+import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.support.ExcelTypeEnum;
 import com.alibaba.excel.util.CollectionUtils;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -33,6 +35,7 @@ import org.opsli.api.wrapper.system.user.UserModel;
 import org.opsli.common.annotation.RequiresPermissionsCus;
 import org.opsli.common.annotation.hotdata.EnableHotData;
 import org.opsli.common.constants.CacheConstants;
+import org.opsli.common.enums.ExcelOperate;
 import org.opsli.common.exception.ServiceException;
 import org.opsli.common.exception.TokenException;
 import org.opsli.common.msg.CommonMsg;
@@ -49,6 +52,7 @@ import org.opsli.core.utils.DistributedLockUtil;
 import org.opsli.core.utils.ExcelUtil;
 import org.opsli.core.utils.UserUtil;
 import org.opsli.plugins.excel.exception.ExcelPluginException;
+import org.opsli.plugins.excel.listener.BatchExcelListener;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.ModelAttribute;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -59,6 +63,7 @@ import org.springframework.web.multipart.MultipartHttpServletRequest;
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletResponse;
 import java.lang.reflect.Method;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 
@@ -193,43 +198,39 @@ public abstract class BaseRestController <T extends BaseEntity, E extends ApiWra
             return ResultVo.error(CoreMsg.EXCEL_FILE_NULL.getCode(),
                     CoreMsg.EXCEL_FILE_NULL.getMessage());
         }
-        ResultVo<?> resultVo;
+        ResultVo<?> resultVo ;
-        String msgInfo;
+        String msgInfo = "";
         try {
-            List<E> modelList = ExcelUtil.getInstance().readExcel(files.get(0), modelClazz);
+            UserModel user = UserUtil.getUser();
-            if(CollUtil.isNotEmpty(modelList)){
+            Date currDate = DateUtil.date();
-                // 导入数量限制 -1 为无限制
+
-                Integer importMaxCount = globalProperties.getExcel().getImportMaxCount();
+            // 导入优化为 监听器 模式 超过一定阈值直接释放资源 防止导入数据导致系统 OOM
-                if(importMaxCount != null && importMaxCount > -1){
+            ExcelUtil.getInstance().readExcelByListener(files.get(0), modelClazz, new BatchExcelListener<E>() {
-                    if(modelList.size() > importMaxCount){
+                @Override
-                        String maxError = StrUtil.format(CoreMsg.EXCEL_HANDLE_MAX.getMessage(), modelList.size(),
+                public void saveData(List<E> dataList) {
-                                importMaxCount);
+                    // 处理字典数据
-                        // 清空 list
+                    List<E> disposeData = ExcelUtil.getInstance().handleDatas(dataList, modelClazz, ExcelOperate.READ);
-                        modelList.clear();
+                    // 手动赋值 必要数据 防止频繁开启Redis网络IO
-                        // 超出最大导出数量
+                    for (E model : disposeData) {
-                        throw new ExcelPluginException(CoreMsg.EXCEL_HANDLE_MAX.getCode(), maxError);
+                        model.setIzManual(true);
+                        model.setCreateBy(user.getId());
+                        model.setUpdateBy(user.getId());
+                        model.setCreateTime(currDate);
+                        model.setUpdateTime(currDate);
                     }
+                    // 数据库插入数据
+                    IService.insertBatch(disposeData);
                 }
+            });
 
+            // 花费毫秒数
+            long timerCount = timer.interval();
+            // 提示信息
+            msgInfo = StrUtil.format(CoreMsg.EXCEL_IMPORT_SUCCESS.getMessage(), DateUtil.formatBetween(timerCount));
+            // 导出成功
+            resultVo = ResultVo.success(msgInfo);
+            resultVo.setCode(CoreMsg.EXCEL_IMPORT_SUCCESS.getCode());
 
-                boolean ret = IService.insertBatch(modelList);
-                if(!ret){
-                    // 清空 list
-                    modelList.clear();
-                    throw new ExcelPluginException(CoreMsg.EXCEL_IMPORT_NO);
-                }
-                // 清空 list
-                modelList.clear();
-                // 花费毫秒数
-                long timerCount = timer.interval();
-                // 提示信息
-                msgInfo = StrUtil.format(CoreMsg.EXCEL_IMPORT_SUCCESS.getMessage(), DateUtil.formatBetween(timerCount));
-                // 导出成功
-                resultVo = ResultVo.success(msgInfo);
-                resultVo.setCode(CoreMsg.EXCEL_IMPORT_SUCCESS.getCode());
-            }else {
-                throw new ExcelPluginException(CoreMsg.EXCEL_FILE_NULL);
-            }
         } catch (ExcelPluginException e) {
             // 花费毫秒数
             long timerCount = timer.interval();

opsli-base-support/opsli-core/src/main/java/org/opsli/core/filters/interceptor/MybatisAutoFillInterceptor.java

@@ -30,6 +30,7 @@ import org.apache.ibatis.mapping.SqlCommandType;
 import org.apache.ibatis.plugin.*;
 import org.opsli.common.constants.MyBatisConstants;
 import org.opsli.core.utils.UserUtil;
+import org.opsli.core.utils.excel.factory.AbstractModelHelper;
 import org.springframework.stereotype.Component;
 
 import java.lang.reflect.Field;
@@ -58,6 +59,9 @@ public class MybatisAutoFillInterceptor implements Interceptor {
 
     private static final String ET = "et";
 
+    /** 实体类字段 */
+    static private final Map<Class<?>, Field[]> ENTITY_FIELD_MAP = new HashMap<>();
+
     @Override
     public Object intercept(Invocation invocation) throws IllegalAccessException, InvocationTargetException {
         fillField(invocation);
@@ -110,7 +114,14 @@ public class MybatisAutoFillInterceptor implements Interceptor {
 
         // 当前时间
         Date currDate = DateUtil.date();
-        Field[] fields = ReflectUtil.getFields(arg.getClass());
+
+        // 字段缓存 减少每次更新 反射
+        Field[] fields = ENTITY_FIELD_MAP.get(arg.getClass());
+        if(fields == null){
+            fields = ReflectUtil.getFields(arg.getClass());
+            ENTITY_FIELD_MAP.put(arg.getClass(), fields);
+        }
+
         for (Field f : fields) {
             // 判断是否是排除字段
             if(existField.contains(f.getName())){
@@ -189,7 +200,6 @@ public class MybatisAutoFillInterceptor implements Interceptor {
 
         // 2020-09-19
         // 修改这儿 有可能会拿到一个 MapperMethod，需要特殊处理
-        Field[] fields;
         if (arg instanceof MapperMethod.ParamMap) {
             MapperMethod.ParamMap<?> paramMap = (MapperMethod.ParamMap<?>) arg;
             if (paramMap.containsKey(ET)) {
@@ -201,7 +211,14 @@ public class MybatisAutoFillInterceptor implements Interceptor {
                 return;
             }
         }
-        fields = ReflectUtil.getFields(arg.getClass());
+
+        // 字段缓存 减少每次更新 反射
+        Field[] fields = ENTITY_FIELD_MAP.get(arg.getClass());
+        if(fields == null){
+            fields = ReflectUtil.getFields(arg.getClass());
+            ENTITY_FIELD_MAP.put(arg.getClass(), fields);
+        }
+
         for (Field f : fields) {
             // 判断是否是排除字段
             if(existField.contains(f.getName())){

opsli-base-support/opsli-core/src/main/java/org/opsli/core/utils/ExcelUtil.java

@@ -34,6 +34,7 @@ import org.opsli.core.utils.excel.factory.ModelFactoryHelper;
 import org.opsli.plugins.excel.ExcelPlugin;
 import org.opsli.plugins.excel.annotation.ExcelInfo;
 import org.opsli.plugins.excel.exception.ExcelPluginException;
+import org.opsli.plugins.excel.listener.BatchExcelListener;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletResponse;
@@ -93,6 +94,25 @@ public final class ExcelUtil {
         return this.handleDatas(ts, rowModel, ExcelOperate.READ);
     }
 
+    ///////////////////////
+
+    public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel,
+                                        BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
+        ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, batchExcelListener);
+    }
+
+    public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName,
+                                        BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
+        ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, sheetName, batchExcelListener);
+    }
+
+    public <T> void readExcelByListener(MultipartFile excel, Class<T> rowModel, String sheetName, int headLineNum,
+                                        BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
+        ExcelUtilSingletonHolder.EXCEL_PLUGIN.readExcelByListener(excel, rowModel, sheetName, headLineNum, batchExcelListener);
+    }
+
+    ///////////////////////
+
     public <T> void writeExcel(HttpServletResponse response, List<T> list, String fileName, String sheetName, Class<T> classType, ExcelTypeEnum excelTypeEnum) throws ExcelPluginException {
         // 处理数据
         List<T> ts = this.handleDatas(list, classType, ExcelOperate.WRITE);
@@ -107,7 +127,7 @@ public final class ExcelUtil {
      * @param <T> 泛型
      * @return List<T>
      */
-    private <T> List<T> handleDatas(List<T> datas, Class<T> typeClazz, ExcelOperate operate){
+    public <T> List<T> handleDatas(List<T> datas, Class<T> typeClazz, ExcelOperate operate){
         // 计时器
         TimeInterval timer = DateUtil.timer();
         // 空处理

opsli-base-support/opsli-core/src/main/resources/META-INF/additional-spring-configuration-metadata.json

@@ -157,14 +157,6 @@
     },
 
 
-
-    {
-      "name": "opsli.excel.import-max-count",
-      "sourceType": "org.opsli.core.autoconfigure.properties.GlobalProperties$Excel",
-      "type": "java.lang.Integer",
-      "defaultValue": 20000,
-      "description": "Excel 导入最大操作数."
-    },
     {
       "name": "opsli.excel.export-max-count",
       "sourceType": "org.opsli.core.autoconfigure.properties.GlobalProperties$Excel",

opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/ExcelPlugin.java

@@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.opsli.common.utils.WrapperUtil;
 import org.opsli.plugins.excel.exception.ExcelPluginException;
+import org.opsli.plugins.excel.listener.BatchExcelListener;
 import org.opsli.plugins.excel.listener.ExcelListener;
 import org.opsli.plugins.excel.msg.ExcelMsg;
 import org.springframework.web.multipart.MultipartFile;
@@ -84,7 +85,7 @@ public class ExcelPlugin {
      */
     public <T> List<T> readExcel(MultipartFile excel, Class<T>  rowModel, String sheetName,
                                                              int headLineNum) throws ExcelPluginException {
-        ExcelListener excelListener = new ExcelListener();
+        ExcelListener<T> excelListener = new ExcelListener<>();
         InputStream inputStream = null;
         try{
             if(null != excel){
@@ -115,6 +116,79 @@ public class ExcelPlugin {
         return getExtendsBeanList(excelListener.getDataList(), rowModel);
     }
 
+
+    // ==================================================================================
+
+
+    /**
+     * 读取 Excel(多个 sheet)
+     * 将多sheet合并成一个list数据集，通过自定义ExcelReader继承AnalysisEventListener
+     * 重写invoke doAfterAllAnalysed方法
+     * getExtendsBeanList 主要是做Bean的属性拷贝 ，可以通过ExcelReader中添加的数据集直接获取
+     * @param excel    文件
+     * @param rowModel 实体类映射，继承 BaseRowModel 类
+     * @param batchExcelListener 监听器
+     */
+    public <T> void readExcelByListener(MultipartFile excel,Class<T>  rowModel,
+                                           BatchExcelListener<T> batchExcelListener)
+            throws ExcelPluginException {
+        readExcelByListener(excel, rowModel, null, 1, batchExcelListener);
+    }
+
+    /**
+     * 读取某个 sheet 的 Excel
+     * @param excel    文件
+     * @param rowModel 实体类映射，继承 BaseRowModel 类
+     * @param sheetName  sheet 的序号 从1开始
+     * @param batchExcelListener 监听器
+     */
+    public <T> void readExcelByListener(MultipartFile excel, Class<T>  rowModel, String sheetName,
+                                           BatchExcelListener<T> batchExcelListener)
+            throws ExcelPluginException{
+        readExcelByListener(excel, rowModel, sheetName, 1, batchExcelListener);
+    }
+
+    /**
+     * 读取某个 sheet 的 Excel
+     * @param excel       文件
+     * @param rowModel    实体类映射，继承 BaseRowModel 类
+     * @param sheetName     sheet 的序号 从1开始
+     * @param headLineNum 表头行数，默认为1
+     * @param batchExcelListener 监听器
+     */
+    public <T> void readExcelByListener(MultipartFile excel, Class<T>  rowModel, String sheetName,
+                                 int headLineNum, BatchExcelListener<T> batchExcelListener) throws ExcelPluginException {
+        if(null == batchExcelListener){
+            return;
+        }
+        InputStream inputStream = null;
+        try{
+            if(null != excel){
+                inputStream = excel.getInputStream();
+            }
+        }catch (IOException e){
+            log.error(e.getMessage(),e);
+        }
+        if(null == inputStream){
+            return;
+        }
+
+        ExcelReader excelReader = EasyExcel.read(inputStream, rowModel, batchExcelListener).build();
+        if (excelReader == null) {
+            return;
+        }
+        ReadSheet readSheet;
+        if(StringUtils.isEmpty(sheetName)){
+            readSheet = EasyExcel.readSheet().build();
+        }else{
+            readSheet = EasyExcel.readSheet(sheetName).build();
+        }
+        readSheet.setHeadRowNumber(headLineNum);
+        excelReader.read(readSheet);
+        // 这里千万别忘记关闭，读的时候会创建临时文件，到时磁盘会崩的
+        excelReader.finish();
+    }
+
     /**
      * 导出 Excel ：一个 sheet，带表头
      * 自定义WriterHandler 可以定制行列数据进行灵活化操作

opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/listener/BatchExcelListener.java

@@ -0,0 +1,99 @@
+/**
+ * Copyright 2020 OPSLI 快速开发平台 https://www.opsli.com
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.opsli.plugins.excel.listener;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Excel 监听
+ * @author parker
+ */
+@Slf4j
+public abstract class BatchExcelListener<T> extends AnalysisEventListener<T> {
+
+    private static final String SERIAL_VERSION_UID = "serialVersionUID";
+
+    /**
+     * 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
+     */
+    private static final int BATCH_COUNT = 2000;
+
+    private final List<Object> dataList = new ArrayList<>();
+
+    /**
+     * 钩子函数 需要调用方重写 保存方法
+     * @param dataList 数据集合
+     */
+    abstract public void saveData(List<T> dataList);
+
+    /**
+     * 通过 AnalysisContext 对象还可以获取当前 sheet，当前行等数据
+     */
+    @Override
+    public void invoke(Object object, AnalysisContext context) {
+        if(!checkObjAllFieldsIsNull(object)) {
+            dataList.add(object);
+            // 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM
+            if (dataList.size() >= BATCH_COUNT) {
+                saveData((List<T>) dataList);
+                // 存储完成清理 list
+                dataList.clear();
+            }
+        }
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+        // 这里也要保存数据，确保最后遗留的数据也存储到数据库
+        saveData((List<T>) dataList);
+    }
+
+
+    /**
+     * 判断对象中属性值是否全为空
+     */
+    private static boolean checkObjAllFieldsIsNull(Object object) {
+        if (null == object) {
+            return true;
+        }
+        try {
+            for (Field f : object.getClass().getDeclaredFields()) {
+                f.setAccessible(true);
+                //只校验带ExcelProperty注解的属性
+                ExcelProperty property = f.getAnnotation(ExcelProperty.class);
+                if(property == null || SERIAL_VERSION_UID.equals(f.getName())){
+                    continue;
+                }
+                if (f.get(object) != null && StringUtils.isNotBlank(f.get(object).toString())) {
+                    return false;
+                }
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            //do something
+        }
+        return true;
+    }
+
+}

opsli-plugins/opsli-plugins-excel/src/main/java/org/opsli/plugins/excel/listener/ExcelListener.java

@@ -18,7 +18,9 @@ package org.opsli.plugins.excel.listener;
 import com.alibaba.excel.annotation.ExcelProperty;
 import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.event.AnalysisEventListener;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.ss.formula.functions.T;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
@@ -28,7 +30,8 @@ import java.util.List;
  * Excel 监听
  * @author parker
  */
-public class ExcelListener extends AnalysisEventListener {
+@Slf4j
+public class ExcelListener<T> extends AnalysisEventListener<T> {
 
     private static final String SERIAL_VERSION_UID = "serialVersionUID";
 
@@ -69,6 +72,7 @@ public class ExcelListener extends AnalysisEventListener {
                 }
             }
         } catch (Exception e) {
+            log.error(e.getMessage(), e);
             //do something
         }
         return true;

opsli-starter/src/main/resources/application-beta.yaml

@@ -33,13 +33,13 @@ spring:
       #primary: master
       datasource:
         master:
-          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
           username: root
           password: MYSQL0p9o8i7u
           driver-class-name: com.mysql.cj.jdbc.Driver
       # 多数据源配置
       #slave-datasource:
-        #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+        #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
         #username: root
         #password: 12345678
         #driver-class-name: com.mysql.cj.jdbc.Driver

opsli-starter/src/main/resources/application-dev.yaml

@@ -33,13 +33,13 @@ spring:
       #primary: master
       datasource:
         master:
-          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
           username: root
           password: 12345678
           driver-class-name: com.mysql.cj.jdbc.Driver
         # 多数据源配置
         #slave-datasource:
-          #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+          #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
           #username: root
           #password: 12345678
           #driver-class-name: com.mysql.cj.jdbc.Driver

opsli-starter/src/main/resources/application-release.yaml

@@ -33,13 +33,13 @@ spring:
       #primary: master
       datasource:
         master:
-          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+          url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
           username: root
           password: 12345678
           driver-class-name: com.mysql.cj.jdbc.Driver
           # 多数据源配置
           #slave-datasource:
-          #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&serverTimezone=Asia/Shanghai
+          #url: jdbc:mysql://127.0.0.1:3306/opsli-boot?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai
           #username: root
           #password: 12345678
           #driver-class-name: com.mysql.cj.jdbc.Driver

opsli-starter/src/main/resources/application.yaml

@@ -209,7 +209,5 @@ opsli:
 
   # Excel
   excel:
-    # Excel 最大导入操作数量 防止OOM  -1为无限制
-    import-max-count: 5000
     # Excel 最大导出操作数量 防止OOM  -1为无限制
     export-max-count: 50000