Merge pull request #610 from dataease/pr@dev@excel

backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java

 package io.dataease.commons.utils;
 import com.google.gson.Gson;
 import io.dataease.datasource.dto.TableFiled;
-
-import java.io.FileInputStream;
 import java.io.InputStream;
 import java.util.List;
 
@@ -46,7 +44,7 @@ public class ExcelReaderUtil {
             ExcelXlsReader excelXls=new ExcelXlsReader();
             excelXls.process(inputStream);
             System.out.println(excelXls.totalSheets.size());
-            System.out.println(excelXls.totalSheets.get(0).getSheetName());
+            System.out.println(excelXls.totalSheets.get(0).getExcelLable());
             for (TableFiled field : excelXls.totalSheets.get(0).getFields()) {
                 System.out.println(new Gson().toJson(field));
             }
@@ -56,7 +54,7 @@ public class ExcelReaderUtil {
             ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
             excelXlsxReader.process(inputStream);
             System.out.println(excelXlsxReader.totalSheets.size());
-            System.out.println(excelXlsxReader.totalSheets.get(0).getSheetName());
+            System.out.println(excelXlsxReader.totalSheets.get(0).getExcelLable());
             for (TableFiled field : excelXlsxReader.totalSheets.get(0).getFields()) {
                 System.out.println(new Gson().toJson(field));
             }
@@ -68,7 +66,7 @@ public class ExcelReaderUtil {
     }
 
     public static void main(String[] args) throws Exception {
-        String file ="下单渠道.xlsx";
-        ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file));
+//        String file ="下单渠道.xlsx";
+//        ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file));
     }
 }

backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java

@@ -300,9 +300,9 @@ public class ExcelXlsReader implements HSSFListener {
             }
             lastColumnNumber = -1;
 
-            if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){
+            if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){
                 ExcelSheetData excelSheetData = new ExcelSheetData();
-                excelSheetData.setSheetName(sheetName);
+                excelSheetData.setExcelLable(sheetName);
                 excelSheetData.setData(new ArrayList<>());
                 excelSheetData.setFields(new ArrayList<>());
                 totalSheets.add(excelSheetData);
@@ -322,10 +322,10 @@ public class ExcelXlsReader implements HSSFListener {
 
 
             if (flag && curRow != 0) { //该行不为空行且该行不是第一行，发送（第一行为列名，不需要）
-                if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){
+                if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){
                     ExcelSheetData excelSheetData = new ExcelSheetData();
                     excelSheetData.setData(new ArrayList<>(data));
-                    excelSheetData.setSheetName(sheetName);
+                    excelSheetData.setExcelLable(sheetName);
                     excelSheetData.setFields(new ArrayList<>(fields));
                     List<String> tmp = new ArrayList<>(cellList);
                     excelSheetData.getData().add(tmp);
@@ -333,7 +333,7 @@ public class ExcelXlsReader implements HSSFListener {
                     totalSheets.add(excelSheetData);
                 }else {
                     List<String> tmp = new ArrayList<>(cellList);
-                    totalSheets.stream().filter(s->s.getSheetName().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp);
+                    totalSheets.stream().filter(s->s.getExcelLable().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp);
                     totalRows++;
                 }
             }

backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java

@@ -157,7 +157,7 @@ public class ExcelXlsxReader extends DefaultHandler {
 
             ExcelSheetData excelSheetData = new ExcelSheetData();
             excelSheetData.setData(new ArrayList<>(data));
-            excelSheetData.setSheetName(sheets.getSheetName());
+            excelSheetData.setExcelLable(sheets.getSheetName());
             excelSheetData.setFields(new ArrayList<>(fields));
             totalSheets.add(excelSheetData);
 

backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java

@@ -7,6 +7,7 @@ import io.dataease.base.domain.DatasetTableIncrementalConfig;
 import io.dataease.controller.request.dataset.DataSetTableRequest;
 import io.dataease.datasource.dto.TableFiled;
 import io.dataease.dto.dataset.DataSetTableDTO;
+import io.dataease.dto.dataset.ExcelFileData;
 import io.dataease.service.dataset.DataSetTableService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -37,8 +38,12 @@ public class DataSetTableController {
 
     @ApiOperation("更新")
     @PostMapping("update")
-    public DatasetTable save(@RequestBody DataSetTableRequest datasetTable) throws Exception {
-        return dataSetTableService.save(datasetTable);
+    public void save(@RequestBody DataSetTableRequest datasetTable) throws Exception {
+        if(datasetTable.getType().equalsIgnoreCase("excel")){
+            dataSetTableService.saveExcel(datasetTable);
+        }else {
+            dataSetTableService.save(datasetTable);
+        }
     }
 
     @ApiOperation("删除")
@@ -121,8 +126,8 @@ public class DataSetTableController {
 
     @ApiOperation("excel上传")
     @PostMapping("excel/upload")
-    public Map<String, Object> excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId) throws Exception {
-        return dataSetTableService.excelSaveAndParse(file, tableId);
+    public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId, @RequestParam("editType") Integer editType ) throws Exception {
+        return dataSetTableService.excelSaveAndParse(file, tableId, editType);
     }
 
     @ApiOperation("检测doris")

backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java

@@ -2,6 +2,7 @@ package io.dataease.controller.request.dataset;
 
 import io.dataease.base.domain.DatasetTable;
 import io.dataease.datasource.dto.TableFiled;
+import io.dataease.dto.dataset.ExcelSheetData;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -23,4 +24,6 @@ public class DataSetTableRequest extends DatasetTable {
     private Boolean isRename;
     private List<String> typeFilter;
     private List<TableFiled> fields;
+    private List<ExcelSheetData> sheets;
+    private boolean mergeSheet = false;
 }

backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java

@@ -14,7 +14,7 @@ import java.util.List;
 public class DataTableInfoDTO {
     private String table;
     private String sql;
-    private List<String> sheets;
+    private List<ExcelSheetData> excelSheetDataList;
     private String data;// file path
     private List<DataTableInfoCustomUnion> list;
 }

backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java

@@ -4,10 +4,18 @@ import io.dataease.datasource.dto.TableFiled;
 import lombok.Data;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 public class ExcelSheetData {
-    private String sheetName;
+    private String excelLable;
     private List<List<String>> data;
     private List<TableFiled> fields;
+    private boolean isSheet = true;
+    private List<Map<String, Object>> jsonArray;
+    private String datasetName;
+    private String sheetExcelId;
+    private String sheetId;
+    private String path;
+    private String fieldsMd5;
 }

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -17,6 +17,7 @@ import io.dataease.datasource.provider.ProviderFactory;
 import io.dataease.datasource.request.DatasourceRequest;
 import io.dataease.datasource.service.DatasourceService;
 import io.dataease.dto.dataset.DataTableInfoDTO;
+import io.dataease.dto.dataset.ExcelSheetData;
 import io.dataease.exception.DataEaseException;
 import io.dataease.listener.util.CacheUtils;
 import io.dataease.provider.QueryProvider;
@@ -192,7 +193,9 @@ public class ExtractDataService {
                     dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
                 } finally {
                     deleteFile("all_scope", datasetTableId);
-//                    deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData());
+                    for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
+                        deleteFile(excelSheetData.getPath());
+                    }
                 }
                 break;
 
@@ -213,6 +216,9 @@ public class ExtractDataService {
                 } finally {
                     deleteFile("incremental_add", datasetTableId);
                     deleteFile("incremental_delete", datasetTableId);
+                    for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
+                        deleteFile(excelSheetData.getPath());
+                    }
                 }
                 break;
         }
@@ -831,18 +837,30 @@ public class ExtractDataService {
 
     private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields){
         DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
-        String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1);
+        List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
+        String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
         ExcelInputMeta excelInputMeta = new ExcelInputMeta();
+
+        List<String> sheetNames = new ArrayList<>();
+        List<String> files = new ArrayList<>();
+        for (ExcelSheetData excelSheetData : excelSheetDataList) {
+            if(!sheetNames.contains(excelSheetData.getExcelLable())){
+                sheetNames.add(excelSheetData.getExcelLable());
+            }
+            if(!files.contains(excelSheetData.getPath())){
+                files.add(excelSheetData.getPath());
+            }
+        }
         if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
             excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
-            excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
+            excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
         }
         if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
             excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL);
-            excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
+            excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
         }
         excelInputMeta.setPassword("Encrypted");
-        excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()});
+        excelInputMeta.setFileName( files.toArray(new String[files.size()]));
         excelInputMeta.setStartsWithHeader(true);
         excelInputMeta.setIgnoreEmptyRows(true);
         ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];

frontend/src/lang/en.js

@@ -1006,7 +1006,8 @@ export default {
     field_manage: 'Field Manage',
     edit_calc_field: 'Edit calc field',
     calc_field: 'Calc Field',
-    show_sql: 'Show SQL'
+    show_sql: 'Show SQL',
+    ple_select_excel: 'Please select excel file to import'
   },
   datasource: {
     datasource: 'Data Source',

frontend/src/lang/tw.js

@@ -1006,7 +1006,8 @@ export default {
     field_manage: '字段管理',
     edit_calc_field: '編輯計算字段',
     calc_field: '計算字段',
-    show_sql: '顯示SQL'
+    show_sql: '顯示SQL',
+    ple_select_excel: '請選擇要導入的 Excel'
   },
   datasource: {
     datasource: '數據源',

frontend/src/lang/zh.js

@@ -1006,7 +1006,8 @@ export default {
     field_manage: '字段管理',
     edit_calc_field: '编辑计算字段',
     calc_field: '计算字段',
-    show_sql: '显示SQL'
+    show_sql: '显示SQL',
+    ple_select_excel: '请选择要导入的 Excel'
   },
   datasource: {
     datasource: '数据源',