Merge remote-tracking branch 'origin/dev' into dev

backend/src/main/java/io/dataease/auth/server/AuthServer.java

@@ -117,6 +117,7 @@ public class AuthServer implements AuthApi {
         // 记录token操作时间
         result.put("token", token);
         ServletUtils.setToken(token);
+        authUserService.clearCache(user.getUserId());
         return result;
     }
 

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -126,6 +126,8 @@ public class DataSetTableService {
     }
 
     public void saveExcel(DataSetTableRequest datasetTable) throws Exception {
+        List<String> datasetIdList = new ArrayList<>();
+
         if (StringUtils.isEmpty(datasetTable.getId())) {
             if (datasetTable.isMergeSheet()) {
                 Map<String, List<ExcelSheetData>> map = datasetTable.getSheets().stream().collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5));
@@ -149,12 +151,13 @@ public class DataSetTableService {
                     DataTableInfoDTO info = new DataTableInfoDTO();
                     info.setExcelSheetDataList(excelSheetDataList);
                     sheetTable.setInfo(new Gson().toJson(info));
-                    int insert = datasetTableMapper.insert(sheetTable);
-                    if (insert == 1) {
-                        saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields(), true);
-                        commonThreadPool.addTask(() -> extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入", null));
-                    }
+                    datasetTableMapper.insert(sheetTable);
+                    saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields(), true);
+                    datasetIdList.add(sheetTable.getId());
                 }
+                datasetIdList.forEach(datasetId ->{
+                    commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList));
+                });
             } else {
                 for (ExcelSheetData sheet : datasetTable.getSheets()) {
                     String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
@@ -175,12 +178,14 @@ public class DataSetTableService {
                     DataTableInfoDTO info = new DataTableInfoDTO();
                     info.setExcelSheetDataList(excelSheetDataList);
                     sheetTable.setInfo(new Gson().toJson(info));
-                    int insert = datasetTableMapper.insert(sheetTable);
-                    if (insert == 1) {
-                        saveExcelTableField(sheetTable.getId(), sheet.getFields(), true);
-                        commonThreadPool.addTask(() -> extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入", null));
-                    }
+                    datasetTableMapper.insert(sheetTable);
+                    saveExcelTableField(sheetTable.getId(), sheet.getFields(), true);
+                    datasetIdList.add(sheetTable.getId());
                 }
+                datasetIdList.forEach(datasetId ->{
+                    commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList));
+                });
+
             }
             return;
         }
@@ -210,13 +215,10 @@ public class DataSetTableService {
         datasetTable.setInfo(new Gson().toJson(info));
         int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
         // 替換時，先不刪除旧字段；同步成功后再删除
-
-        if (update == 1) {
-            if (datasetTable.getEditType() == 0) {
-                commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false)));
-            } else if (datasetTable.getEditType() == 1) {
-                commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", null));
-            }
+        if (datasetTable.getEditType() == 0) {
+            commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false), Arrays.asList(datasetTable.getId())));
+        } else if (datasetTable.getEditType() == 1) {
+            commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", null, Arrays.asList(datasetTable.getId())));
         }
     }
 
@@ -302,6 +304,12 @@ public class DataSetTableService {
         return extDataSetTableMapper.search(dataSetTableRequest);
     }
 
+    public List<DatasetTable> list(List<String> datasetIds) {
+        DatasetTableExample example = new DatasetTableExample();
+        example.createCriteria().andIdIn(datasetIds);
+        return datasetTableMapper.selectByExample(example);
+    }
+
     public List<DataSetTableDTO> listAndGroup(DataSetTableRequest dataSetTableRequest) {
         dataSetTableRequest.setUserId(String.valueOf(AuthUtils.getUser().getUserId()));
         dataSetTableRequest.setTypeFilter(dataSetTableRequest.getTypeFilter());

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -150,7 +150,7 @@ public class ExtractDataService {
         }
     }
 
-    public void extractExcelData(String datasetTableId, String type, String ops, List<DatasetTableField> datasetTableFields) {
+    public void extractExcelData(String datasetTableId, String type, String ops, List<DatasetTableField> datasetTableFields, List<String> datasetTableIds) {
         Datasource datasource = new Datasource();
         datasource.setType("excel");
         DatasetTable datasetTable = getDatasetTable(datasetTableId);
@@ -223,9 +223,7 @@ public class ExtractDataService {
                     dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
                 } finally {
                     deleteFile("all_scope", datasetTableId);
-                    for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
-                        deleteFile(excelSheetData.getPath());
-                    }
+                    deleteExcelFile(datasetTable, datasetTableIds);
                 }
                 break;
 
@@ -1081,6 +1079,27 @@ public class ExtractDataService {
         deleteFile(root_path + transName + ".ktr");
     }
 
+    private void deleteExcelFile(DatasetTable datasetTable, List<String>datasetTableIds){
+        List<DatasetTable> datasetTables = dataSetTableService.list(datasetTableIds);
+        for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
+            Boolean allIsFinished = true;
+            for (DatasetTable table : datasetTables) {
+                for(ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()){
+                    if(data.getPath().equalsIgnoreCase(excelSheetData.getPath())){
+                        if(StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())){
+                            allIsFinished = false;
+                        }
+                    }
+                }
+            }
+            if(allIsFinished){
+                deleteFile(excelSheetData.getPath());
+            }else {
+               try { Thread.sleep(5000); }catch (Exception ignore){}
+                deleteExcelFile(datasetTable, datasetTableIds);
+            }
+        }
+    }
     private void deleteFile(String filePath){
         if(StringUtils.isEmpty(filePath)){
             return;