Merge pull request #2197 from dataease/dev

README.md

@@ -68,7 +68,7 @@ curl -sSL https://github.com/dataease/dataease/releases/latest/download/quick_st
 ```
 
 -   [在线文档](https://dataease.io/docs/)
--   [演示视频](https://www.bilibili.com/video/BV1UB4y1K7jA)
+-   [演示视频](https://www.bilibili.com/video/BV1i34y1v7hq/)
 
 ## 微信群
 

backend/src/main/java/io/dataease/controller/chart/ChartViewController.java

@@ -5,10 +5,7 @@ import io.dataease.auth.annotation.DePermission;
 import io.dataease.auth.annotation.DePermissionProxy;
 import io.dataease.commons.constants.DePermissionType;
 import io.dataease.commons.constants.ResourceAuthLevel;
-import io.dataease.controller.request.chart.ChartCalRequest;
-import io.dataease.controller.request.chart.ChartExtRequest;
-import io.dataease.controller.request.chart.ChartViewCacheRequest;
-import io.dataease.controller.request.chart.ChartViewRequest;
+import io.dataease.controller.request.chart.*;
 import io.dataease.controller.response.ChartDetail;
 import io.dataease.dto.chart.ChartViewDTO;
 import io.dataease.plugins.common.base.domain.ChartViewCacheWithBLOBs;
@@ -23,6 +20,7 @@ import springfox.documentation.annotations.ApiIgnore;
 
 import javax.annotation.Resource;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @Author gin
@@ -112,6 +110,13 @@ public class ChartViewController {
         return chartViewService.chartCopy(id, panelId);
     }
 
+    @DePermission(type = DePermissionType.PANEL, level = ResourceAuthLevel.PANNEL_LEVEL_MANAGE, paramIndex = 1)
+    @ApiOperation("批量复制")
+    @PostMapping("chartBatchCopy/{panelId}")
+    public Map<String,String> chartBatchCopy(@RequestBody ChartCopyBatchRequest request, @PathVariable String panelId) {
+        return chartViewService.chartBatchCopy(request,panelId);
+    }
+
     @ApiIgnore
     @GetMapping("searchAdviceSceneId/{panelId}")
     public String searchAdviceSceneId(@PathVariable String panelId) {

backend/src/main/java/io/dataease/controller/request/chart/ChartCopyBatchRequest.java

+package io.dataease.controller.request.chart;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Author: wangjiahao
+ * Date: 2022/4/26
+ * Description:
+ */
+@Data
+public class ChartCopyBatchRequest {
+
+    private String panelId;
+
+    private Map<String,String> sourceAndTargetIds;
+
+}

backend/src/main/java/io/dataease/service/chart/ChartViewService.java

 package io.dataease.service.chart;
 
+import cn.hutool.core.lang.Assert;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import io.dataease.auth.entity.SysUserEntity;
@@ -1024,13 +1025,35 @@ public class ChartViewService {
         return chartViewMapper.selectByPrimaryKey(id);
     }
 
-    public String chartCopy(String id, String panelId) {
+    public String chartCopy(String sourceViewId,String newViewId, String panelId) {
+        extChartViewMapper.chartCopy(newViewId, sourceViewId, panelId);
+        extChartViewMapper.copyCache(sourceViewId, newViewId);
+        extPanelGroupExtendDataMapper.copyExtendData(sourceViewId, newViewId, panelId);
+        chartViewCacheService.refreshCache(newViewId);
+        return newViewId;
+    }
+
+    public String chartCopy(String sourceViewId, String panelId) {
         String newChartId = UUID.randomUUID().toString();
-        extChartViewMapper.chartCopy(newChartId, id, panelId);
-        extChartViewMapper.copyCache(id, newChartId);
-        extPanelGroupExtendDataMapper.copyExtendData(id, newChartId, panelId);
-        chartViewCacheService.refreshCache(newChartId);
-        return newChartId;
+       return chartCopy(sourceViewId,newChartId,panelId);
+    }
+
+
+    /**
+     * @Description Copy a set of views with a given source ID and target ID
+     * @param request
+     * @param panelId
+     * @return
+     */
+    public Map<String,String> chartBatchCopy(ChartCopyBatchRequest request,String panelId){
+        Assert.notNull(panelId,"panelId should not be null");
+        Map<String,String> sourceAndTargetIds = request.getSourceAndTargetIds();
+        if(sourceAndTargetIds != null && !sourceAndTargetIds.isEmpty()){
+            for(Map.Entry<String,String> entry:sourceAndTargetIds.entrySet()){
+                chartCopy(entry.getKey(),entry.getValue(),panelId);
+            }
+        }
+        return request.getSourceAndTargetIds();
     }
 
     public String searchAdviceSceneId(String panelId) {

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -120,6 +120,19 @@ public class ExtractDataService {
             "  exit 1\n" +
             "fi\n";
 
+    private static final String shellScriptForDeleteFile = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
+            "rm -rf  %s \n" +
+            "if [ $? -eq 0 ] ; then\n" +
+            "  failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
+            "  if [ \"x${failstatus}\" != \"x\" ];then" +
+            "     echo $result\n" +
+            "     exit 1\n" +
+            "  fi\n" +
+            "else\n" +
+            "  echo $result\n" +
+            "  exit 1\n" +
+            "fi\n";
+
     public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) {
         datasetTable.setSyncStatus(JobStatus.Underway.name());
         DatasetTableExample example = new DatasetTableExample();
@@ -445,14 +458,20 @@ public class ExtractDataService {
 
         String dataFile = null;
         String script = null;
+        String streamLoadScript = "";
+        if(kettleFilesKeep){
+            streamLoadScript = shellScript;
+        }else {
+            streamLoadScript = shellScriptForDeleteFile;
+        }
         switch (extractType) {
             case "all_scope":
                 dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
-                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
+                script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
                 break;
             default:
                 dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
-                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
+                script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
                 break;
         }
 
@@ -744,7 +763,6 @@ public class ExtractDataService {
             DataEaseException.throwException(transStatus.getLoggingString());
             return;
         }
-
         executing = true;
         String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null);
         SlaveServerJobStatus jobStatus = null;
@@ -759,7 +777,7 @@ public class ExtractDataService {
         if (jobStatus.getStatusDescription().equals("Finished")) {
             return;
         } else {
-            DataEaseException.throwException((jobStatus.getLoggingString()));
+            DataEaseException.throwException(jobStatus.getLoggingString());
         }
     }
 
@@ -773,20 +791,26 @@ public class ExtractDataService {
         Datasource dorisDatasource = engineService.getDeEngine();
         DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
         String columns = columnFields + ",dataease_uuid";
+        String streamLoadScript = "";
+        if(kettleFilesKeep){
+            streamLoadScript = shellScript;
+        }else {
+            streamLoadScript = shellScriptForDeleteFile;
+        }
         switch (extractType) {
             case "all_scope":
                 outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
                 jobName = "job_" + TableUtils.tableName(datasetTable.getId());
-                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
+                script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
                 break;
             case "incremental_add":
                 outFile = TableUtils.addName(datasetTable.getId());
                 jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
-                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
+                script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
                 break;
             case "incremental_delete":
                 outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
-                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
+                script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
                 jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
                 break;
             default:

backend/src/main/java/io/dataease/service/datasource/DatasourceService.java

@@ -115,6 +115,9 @@ public class DatasourceService {
                     datasourceDTO.setCalculationMode(dataSourceType.getCalculationMode());
                 }
             });
+            if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.mysql.toString())){
+                datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class)) );
+            }
             if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())){
                 JSONArray apiDefinitionList = JSONObject.parseArray(datasourceDTO.getConfiguration());
                 JSONArray apiDefinitionListWithStatus = new JSONArray();