Skip to content

D
dataease
提交 08a193d4 authored 作者: taojinlong's avatar taojinlong
浏览文件
操作

feat: 收取数据后，删掉 kettle 文件

上级 943d753b
隐藏空白字符变更
内嵌 并排
正在显示 包含 210 行增加 和 128 行删除
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java
浏览文件 @ 08a193d4
...@@ -69,6 +69,7 @@ public class DataSetTableTaskLogService { ...@@ -69,6 +69,7 @@ public class DataSetTableTaskLogService {
if(StringUtils.isNotEmpty(datasetTableTaskLog.getTaskId())){ if(StringUtils.isNotEmpty(datasetTableTaskLog.getTaskId())){
criteria.andTaskIdEqualTo(datasetTableTaskLog.getTaskId()); criteria.andTaskIdEqualTo(datasetTableTaskLog.getTaskId());
} }
example.setOrderByClause("create_time desc");
return datasetTableTaskLogMapper.selectByExampleWithBLOBs(example); return datasetTableTaskLogMapper.selectByExampleWithBLOBs(example);
} }
} }
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
浏览文件 @ 08a193d4
...@@ -7,6 +7,7 @@ import io.dataease.commons.constants.ScheduleType; ...@@ -7,6 +7,7 @@ import io.dataease.commons.constants.ScheduleType;
import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.i18n.Translator; import io.dataease.i18n.Translator;
import io.dataease.service.ScheduleService; import io.dataease.service.ScheduleService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression; import org.quartz.CronExpression;
...@@ -61,7 +62,16 @@ public class DataSetTableTaskService { ...@@ -61,7 +62,16 @@ public class DataSetTableTaskService {
datasetTableTask.setCreateTime(System.currentTimeMillis()); datasetTableTask.setCreateTime(System.currentTimeMillis());
// SIMPLE 类型，提前占位 // SIMPLE 类型，提前占位
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
if (extractDataService.updateSyncStatus(dataSetTableService.get(datasetTableTask.getTableId()))) { if(datasetTableTask.getType().equalsIgnoreCase("add_scope")){
DatasetTableTaskLog request = new DatasetTableTaskLog();
request.setTableId(datasetTableTask.getTableId());
request.setStatus(JobStatus.Completed.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(request);
if (CollectionUtils.isEmpty(datasetTableTaskLogs)) {
throw new Exception(Translator.get("i18n_not_exec_add_sync"));
}
}
if (extractDataService.updateSyncStatusIsNone(dataSetTableService.get(datasetTableTask.getTableId()))) {
throw new Exception(Translator.get("i18n_sync_job_exists")); throw new Exception(Translator.get("i18n_sync_job_exists"));
}else { }else {
//write log //write log
......
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @ 08a193d4
...@@ -130,72 +130,7 @@ public class ExtractDataService { ...@@ -130,72 +130,7 @@ public class ExtractDataService {
"fi\n" + "fi\n" +
"rm -rf %s\n"; "rm -rf %s\n";
private String createDorisTablColumnSql(List<DatasetTableField> datasetTableFields) { public synchronized boolean updateSyncStatusIsNone(DatasetTable datasetTable ){
String Column_Fields = "dataease_uuid varchar(50), `";
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
switch (datasetTableField.getDeExtractType()) {
case 0:
if (datasetTableField.getSize() > 65533) {
Column_Fields = Column_Fields + "varchar(65533)" + ",`";
} else {
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
}
break;
case 1:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 2:
Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 3:
Column_Fields = Column_Fields + "DOUBLE" + ",`";
break;
case 4:
Column_Fields = Column_Fields + "TINYINT(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
default:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
}
}
Column_Fields = Column_Fields.substring(0, Column_Fields.length() - 2);
Column_Fields = "(" + Column_Fields + ")\n";
return Column_Fields;
}
private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql));
jdbcProvider.exec(datasourceRequest);
}
private void dropDorisTable(String dorisTableName) {
try {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
jdbcProvider.exec(datasourceRequest);
}catch (Exception ignore){}
}
private void replaceTable(String dorisTableName) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
;
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
}
public synchronized boolean updateSyncStatus(DatasetTable datasetTable ){
datasetTable.setSyncStatus(JobStatus.Underway.name()); datasetTable.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample(); DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTable.getId()); example.createCriteria().andIdEqualTo(datasetTable.getId());
...@@ -213,41 +148,45 @@ public class ExtractDataService { ...@@ -213,41 +148,45 @@ public class ExtractDataService {
} }
DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId);
boolean isCronJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())); boolean isCronJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString()));
if(updateSyncStatus(datasetTable) && isCronJob){ if(updateSyncStatusIsNone(datasetTable) && isCronJob){
LogUtil.info("Skip synchronization task for table : " + datasetTableId); LogUtil.info("Skip synchronization task for table : " + datasetTableId);
return; return;
} }
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
UpdateType updateType = UpdateType.valueOf(type); UpdateType updateType = UpdateType.valueOf(type);
Datasource datasource = new Datasource(); Datasource datasource = new Datasource();
try { if(context != null){
if(context != null){ datasetTable.setQrtzInstance(context.getFireInstanceId());
datasetTable.setQrtzInstance(context.getFireInstanceId()); datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
datasetTableMapper.updateByPrimaryKeySelective(datasetTable); }
if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) {
datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
} else {
datasource.setType(datasetTable.getType());
}
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
return -1;
} }
if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { if (o2.getColumnIndex() == null) {
datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); return 1;
} else {
datasource.setType(datasetTable.getType());
} }
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
String dorisTablColumnSql = createDorisTablColumnSql(datasetTableFields);
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build()); switch (updateType) {
datasetTableFields.sort((o1, o2) -> { case all_scope: // 全量更新
if (o1.getColumnIndex() == null) { try{
return -1;
}
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
String dorisTablColumnSql = createDorisTablColumnSql(datasetTableFields);
switch (updateType) {
// 全量更新
case all_scope:
if(datasource.getType().equalsIgnoreCase("excel")){ if(datasource.getType().equalsIgnoreCase("excel")){
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
}else { }
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
} }
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
...@@ -256,18 +195,30 @@ public class ExtractDataService { ...@@ -256,18 +195,30 @@ public class ExtractDataService {
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
extractData(datasetTable, "all_scope"); extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId)); replaceTable(DorisTableUtils.dorisName(datasetTableId));
datasetTableTaskLog.setStatus(JobStatus.Completed.name()); saveSucessLog(datasetTableTaskLog);
datasetTableTaskLog.setEndTime(System.currentTimeMillis()); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed);
dataSetTableTaskLogService.save(datasetTableTaskLog); }catch (Exception e){
break; saveErrorLog(datasetTableId, taskId, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
}finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
dataSetTableTaskService.update(datasetTableTask);
}
deleteFile("all_scope", datasetTableId);
}
break;
// 增量更新 case add_scope: // 增量更新
case add_scope: try {
if(datasource.getType().equalsIgnoreCase("excel")){ if(datasource.getType().equalsIgnoreCase("excel")){
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
extractData(datasetTable, "incremental_add"); extractData(datasetTable, "incremental_add");
saveSucessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed);
}else { }else {
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId); DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
...@@ -276,61 +227,146 @@ public class ExtractDataService { ...@@ -276,61 +227,146 @@ public class ExtractDataService {
DatasetTableTaskLog request = new DatasetTableTaskLog(); DatasetTableTaskLog request = new DatasetTableTaskLog();
request.setTableId(datasetTableId); request.setTableId(datasetTableId);
request.setStatus(JobStatus.Completed.name()); request.setStatus(JobStatus.Completed.name());
List<DataSetTaskLogDTO> dataSetTaskLogDTOS = dataSetTableTaskLogService.list(request); List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(request);
if (CollectionUtils.isEmpty(dataSetTaskLogDTOS)) { if (CollectionUtils.isEmpty(datasetTableTaskLogs)) {
return; return;
} }
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
// 增量添加 if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) { datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString() }
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())); .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource)); generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_add"); extractData(datasetTable, "incremental_add");
} }
// 增量删除 if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())) { String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())); .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql); generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource)); generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_delete"); extractData(datasetTable, "incremental_delete");
} }
saveSucessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed);
} }
datasetTableTaskLog.setStatus(JobStatus.Completed.name()); }catch (Exception e){
datasetTableTaskLog.setEndTime(System.currentTimeMillis()); saveErrorLog(datasetTableId, taskId, e);
dataSetTableTaskLogService.save(datasetTableTaskLog); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error);
break; }finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
dataSetTableTaskService.update(datasetTableTask);
}
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
}
break;
} }
datasetTable.setSyncStatus(JobStatus.Completed.name()); }
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTableId); private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed) {
datasetTableMapper.updateByExampleSelective(datasetTable, example); datasetTable.setSyncStatus(completed.name());
} catch (Exception e) { DatasetTableExample example = new DatasetTableExample();
e.printStackTrace(); example.createCriteria().andIdEqualTo(datasetTableId);
LogUtil.error("Extract data error: " + datasetTableId, e); datasetTableMapper.updateByExampleSelective(datasetTable, example);
}
private void saveSucessLog(DatasetTableTaskLog datasetTableTaskLog) {
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void saveErrorLog(String datasetTableId, String taskId, Exception e){
LogUtil.error("Extract data error: " + datasetTableId, e);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
if(StringUtils.isNotEmpty(taskId)){
datasetTableTaskLog.setTaskId(taskId);
}
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){
datasetTableTaskLog = datasetTableTaskLogs.get(0);
datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(ExceptionUtils.getStackTrace(e)); datasetTableTaskLog.setInfo(ExceptionUtils.getStackTrace(e));
datasetTableTaskLog.setEndTime(System.currentTimeMillis()); datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog); dataSetTableTaskLogService.save(datasetTableTaskLog);
}
datasetTable.setSyncStatus(JobStatus.Error.name()); }
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTableId);
datasetTableMapper.updateByExampleSelective(datasetTable, example);
if(updateType.name().equalsIgnoreCase("all_scope")){ private String createDorisTablColumnSql(List<DatasetTableField> datasetTableFields) {
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); String Column_Fields = "dataease_uuid varchar(50), `";
} for (DatasetTableField datasetTableField : datasetTableFields) {
} finally { Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { switch (datasetTableField.getDeExtractType()) {
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); case 0:
dataSetTableTaskService.update(datasetTableTask); if (datasetTableField.getSize() > 65533) {
Column_Fields = Column_Fields + "varchar(65533)" + ",`";
} else {
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
}
break;
case 1:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 2:
Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 3:
Column_Fields = Column_Fields + "DOUBLE" + ",`";
break;
case 4:
Column_Fields = Column_Fields + "TINYINT(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
default:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
} }
} }
Column_Fields = Column_Fields.substring(0, Column_Fields.length() - 2);
Column_Fields = "(" + Column_Fields + ")\n";
return Column_Fields;
}
private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql));
jdbcProvider.exec(datasourceRequest);
}
private void dropDorisTable(String dorisTableName) {
try {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
jdbcProvider.exec(datasourceRequest);
}catch (Exception ignore){}
}
private void replaceTable(String dorisTableName) throws Exception {
Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
;
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
} }
private DatasetTable getDatasetTable(String datasetTableId){ private DatasetTable getDatasetTable(String datasetTableId){
...@@ -724,6 +760,36 @@ public class ExtractDataService { ...@@ -724,6 +760,36 @@ public class ExtractDataService {
return userDefinedJavaClassStep; return userDefinedJavaClassStep;
} }
private void deleteFile(String type, String dataSetTableId){
String transName = null;
String jobName = null;
switch (type) {
case "all_scope":
transName = "trans_" + dataSetTableId;
jobName = "job_" + dataSetTableId;
break;
case "incremental_add":
transName = "trans_add_" + dataSetTableId;
jobName = "job_add_" + dataSetTableId;
break;
case "incremental_delete":
transName = "trans_delete_" + dataSetTableId;
jobName = "job_delete_" + dataSetTableId;
break;
default:
break;
}
try{
File file = new File(root_path + jobName + ".kjb");
FileUtils.forceDelete(file);
}catch (Exception e){}
try{
File file = new File(root_path + transName + ".ktr");
FileUtils.forceDelete(file);
}catch (Exception e){}
}
public boolean isKettleRunning() { public boolean isKettleRunning() {
try { try {
if (!InetAddress.getByName(carte).isReachable(1000)) { if (!InetAddress.getByName(carte).isReachable(1000)) {
......
backend/src/main/resources/i18n/messages_en_US.properties
浏览文件 @ 08a193d4
...@@ -252,3 +252,4 @@ i18n_id_or_pwd_error=Invalid ID or password ...@@ -252,3 +252,4 @@ i18n_id_or_pwd_error=Invalid ID or password
i18n_datasource_delete=Data source is delete i18n_datasource_delete=Data source is delete
i18n_dataset_delete=Data set is delete i18n_dataset_delete=Data set is delete
i18n_chart_delete=Chart is delete i18n_chart_delete=Chart is delete
i18n_not_exec_add_sync=There is no completed synchronization task. Incremental synchronization cannot be performed
\ No newline at end of file
backend/src/main/resources/i18n/messages_zh_CN.properties
浏览文件 @ 08a193d4
...@@ -252,3 +252,4 @@ i18n_id_or_pwd_error=无效的ID或密码 ...@@ -252,3 +252,4 @@ i18n_id_or_pwd_error=无效的ID或密码
i18n_datasource_delete=当前用到的数据源已被删除 i18n_datasource_delete=当前用到的数据源已被删除
i18n_dataset_delete=当前用到的数据集已被删除 i18n_dataset_delete=当前用到的数据集已被删除
i18n_chart_delete=当前用到的视图已被删除 i18n_chart_delete=当前用到的视图已被删除
i18n_not_exec_add_sync=没有已完成的同步任务，无法进行增量同步
backend/src/main/resources/i18n/messages_zh_TW.properties
浏览文件 @ 08a193d4
...@@ -254,3 +254,4 @@ i18n_id_or_pwd_error=無效的ID或密碼 ...@@ -254,3 +254,4 @@ i18n_id_or_pwd_error=無效的ID或密碼
i18n_datasource_delete=當前用到的數據源已被刪除 i18n_datasource_delete=當前用到的數據源已被刪除
i18n_dataset_delete=當前用到的數據集已被刪除 i18n_dataset_delete=當前用到的數據集已被刪除
i18n_chart_delete=當前用到的視圖已被刪除 i18n_chart_delete=當前用到的視圖已被刪除
i18n_not_exec_add_sync=沒有已經完成的同步任務，無法進行增量同步
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论