Unverified 提交 dec22613 authored 作者: 王嘉豪's avatar 王嘉豪 提交者: GitHub

Merge pull request #2197 from dataease/dev

Dev
......@@ -68,7 +68,7 @@ curl -sSL https://github.com/dataease/dataease/releases/latest/download/quick_st
```
- [在线文档](https://dataease.io/docs/)
- [演示视频](https://www.bilibili.com/video/BV1UB4y1K7jA)
- [演示视频](https://www.bilibili.com/video/BV1i34y1v7hq/)
## 微信群
......
......@@ -5,10 +5,7 @@ import io.dataease.auth.annotation.DePermission;
import io.dataease.auth.annotation.DePermissionProxy;
import io.dataease.commons.constants.DePermissionType;
import io.dataease.commons.constants.ResourceAuthLevel;
import io.dataease.controller.request.chart.ChartCalRequest;
import io.dataease.controller.request.chart.ChartExtRequest;
import io.dataease.controller.request.chart.ChartViewCacheRequest;
import io.dataease.controller.request.chart.ChartViewRequest;
import io.dataease.controller.request.chart.*;
import io.dataease.controller.response.ChartDetail;
import io.dataease.dto.chart.ChartViewDTO;
import io.dataease.plugins.common.base.domain.ChartViewCacheWithBLOBs;
......@@ -23,6 +20,7 @@ import springfox.documentation.annotations.ApiIgnore;
import javax.annotation.Resource;
import java.util.List;
import java.util.Map;
/**
* @Author gin
......@@ -112,6 +110,13 @@ public class ChartViewController {
return chartViewService.chartCopy(id, panelId);
}
@DePermission(type = DePermissionType.PANEL, level = ResourceAuthLevel.PANNEL_LEVEL_MANAGE, paramIndex = 1)
@ApiOperation("批量复制")
@PostMapping("chartBatchCopy/{panelId}")
public Map<String,String> chartBatchCopy(@RequestBody ChartCopyBatchRequest request, @PathVariable String panelId) {
return chartViewService.chartBatchCopy(request,panelId);
}
@ApiIgnore
@GetMapping("searchAdviceSceneId/{panelId}")
public String searchAdviceSceneId(@PathVariable String panelId) {
......
package io.dataease.controller.request.chart;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* Author: wangjiahao
* Date: 2022/4/26
* Description:
*/
@Data
public class ChartCopyBatchRequest {
private String panelId;
private Map<String,String> sourceAndTargetIds;
}
package io.dataease.service.chart;
import cn.hutool.core.lang.Assert;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import io.dataease.auth.entity.SysUserEntity;
......@@ -1024,13 +1025,35 @@ public class ChartViewService {
return chartViewMapper.selectByPrimaryKey(id);
}
public String chartCopy(String id, String panelId) {
public String chartCopy(String sourceViewId,String newViewId, String panelId) {
extChartViewMapper.chartCopy(newViewId, sourceViewId, panelId);
extChartViewMapper.copyCache(sourceViewId, newViewId);
extPanelGroupExtendDataMapper.copyExtendData(sourceViewId, newViewId, panelId);
chartViewCacheService.refreshCache(newViewId);
return newViewId;
}
public String chartCopy(String sourceViewId, String panelId) {
String newChartId = UUID.randomUUID().toString();
extChartViewMapper.chartCopy(newChartId, id, panelId);
extChartViewMapper.copyCache(id, newChartId);
extPanelGroupExtendDataMapper.copyExtendData(id, newChartId, panelId);
chartViewCacheService.refreshCache(newChartId);
return newChartId;
return chartCopy(sourceViewId,newChartId,panelId);
}
/**
* @Description Copy a set of views with a given source ID and target ID
* @param request
* @param panelId
* @return
*/
public Map<String,String> chartBatchCopy(ChartCopyBatchRequest request,String panelId){
Assert.notNull(panelId,"panelId should not be null");
Map<String,String> sourceAndTargetIds = request.getSourceAndTargetIds();
if(sourceAndTargetIds != null && !sourceAndTargetIds.isEmpty()){
for(Map.Entry<String,String> entry:sourceAndTargetIds.entrySet()){
chartCopy(entry.getKey(),entry.getValue(),panelId);
}
}
return request.getSourceAndTargetIds();
}
public String searchAdviceSceneId(String panelId) {
......
......@@ -120,6 +120,19 @@ public class ExtractDataService {
" exit 1\n" +
"fi\n";
private static final String shellScriptForDeleteFile = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
"rm -rf %s \n" +
"if [ $? -eq 0 ] ; then\n" +
" failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
" if [ \"x${failstatus}\" != \"x\" ];then" +
" echo $result\n" +
" exit 1\n" +
" fi\n" +
"else\n" +
" echo $result\n" +
" exit 1\n" +
"fi\n";
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) {
datasetTable.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
......@@ -445,14 +458,20 @@ public class ExtractDataService {
String dataFile = null;
String script = null;
String streamLoadScript = "";
if(kettleFilesKeep){
streamLoadScript = shellScript;
}else {
streamLoadScript = shellScriptForDeleteFile;
}
switch (extractType) {
case "all_scope":
dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
break;
default:
dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
break;
}
......@@ -744,7 +763,6 @@ public class ExtractDataService {
DataEaseException.throwException(transStatus.getLoggingString());
return;
}
executing = true;
String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null);
SlaveServerJobStatus jobStatus = null;
......@@ -759,7 +777,7 @@ public class ExtractDataService {
if (jobStatus.getStatusDescription().equals("Finished")) {
return;
} else {
DataEaseException.throwException((jobStatus.getLoggingString()));
DataEaseException.throwException(jobStatus.getLoggingString());
}
}
......@@ -773,20 +791,26 @@ public class ExtractDataService {
Datasource dorisDatasource = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = columnFields + ",dataease_uuid";
String streamLoadScript = "";
if(kettleFilesKeep){
streamLoadScript = shellScript;
}else {
streamLoadScript = shellScriptForDeleteFile;
}
switch (extractType) {
case "all_scope":
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
jobName = "job_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
break;
case "incremental_add":
outFile = TableUtils.addName(datasetTable.getId());
jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
break;
case "incremental_delete":
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
break;
default:
......
......@@ -115,6 +115,9 @@ public class DatasourceService {
datasourceDTO.setCalculationMode(dataSourceType.getCalculationMode());
}
});
if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.mysql.toString())){
datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class)) );
}
if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())){
JSONArray apiDefinitionList = JSONObject.parseArray(datasourceDTO.getConfiguration());
JSONArray apiDefinitionListWithStatus = new JSONArray();
......
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论