Skip to content

D
dataease
提交 214b6f86 authored 作者: taojinlong's avatar taojinlong
浏览文件
操作

解决冲突

上级 bdb4d839
显示空白字符变更
内嵌 并排
正在显示 包含 98 行增加 和 97 行删除
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @ 214b6f86
......@@ -6,7 +6,6 @@ import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasetTableTaskMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.base.mapper.ext.ExtChartViewMapper;
import io.dataease.base.mapper.ext.UtilMapper;
import io.dataease.commons.constants.*;
import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
......@@ -16,7 +15,6 @@ import io.dataease.datasource.provider.DatasourceProvider;
import io.dataease.datasource.provider.JdbcProvider;
import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.service.DatasourceService;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
......@@ -24,8 +22,8 @@ import io.dataease.provider.QueryProvider;
import io.dataease.service.message.DeMsgutil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
......@@ -92,12 +90,9 @@ public class ExtractDataService {
private DatasetTableMapper datasetTableMapper;
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;
@Resource
private DatasourceService datasourceService;
@Resource
private ExtChartViewMapper extChartViewMapper;
@Resource
private UtilMapper utilMapper;
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
......@@ -133,26 +128,25 @@ public class ExtractDataService {
"fi\n" +
"rm -rf %s\n";
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask){
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask) {
datasetTable.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull());
Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0;
if(existSyncTask){
if (existSyncTask) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())){
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())) {
return false;
}
return true;
}else {
} else {
datasetTableTask.setLastExecTime(System.currentTimeMillis());
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
datasetTableTask.setStatus(TaskStatus.Exec.name());
dataSetTableTaskService.update(datasetTableTask);
return false;
}
......@@ -195,12 +189,12 @@ public class ExtractDataService {
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
saveSucessLog(datasetTableTaskLog);
// sendWebMsg(datasetTable, null, true);
sendWebMsg(datasetTable, null, true);
deleteFile("all_scope", datasetTableId);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
// sendWebMsg(datasetTable, null, false);
sendWebMsg(datasetTable, null, false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
deleteFile("all_scope", datasetTableId);
......@@ -230,7 +224,7 @@ public class ExtractDataService {
}
//侵入式清除下属视图缓存
List<String> viewIds = extChartViewMapper.allViewIds(datasetTableId);
if (CollectionUtils.isNotEmpty(viewIds)){
if (CollectionUtils.isNotEmpty(viewIds)) {
viewIds.forEach(viewId -> {
CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId);
});
......@@ -239,26 +233,26 @@ public class ExtractDataService {
public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) {
DatasetTable datasetTable = getDatasetTable(datasetTableId);
if(datasetTable == null){
if (datasetTable == null) {
LogUtil.error("Can not find DatasetTable: " + datasetTableId);
return;
}
DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId);
if(datasetTableTask == null){
if (datasetTableTask == null) {
return;
}
if(datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name())|| datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Pending.name())){
if (datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name())) {
LogUtil.info("Skip synchronization task, task ID : " + datasetTableTask.getId());
return;
}
if(existSyncTask(datasetTable, datasetTableTask)){
if (existSyncTask(datasetTable, datasetTableTask)) {
LogUtil.info("Skip synchronization task for dataset, dataset ID : " + datasetTableId);
return;
}
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
UpdateType updateType = UpdateType.valueOf(type);
if(context != null){
if (context != null) {
datasetTable.setQrtzInstance(context.getFireInstanceId());
datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
}
......@@ -282,19 +276,19 @@ public class ExtractDataService {
switch (updateType) {
case all_scope: // 全量更新
try{
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
try {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
if(datasetTable.getType().equalsIgnoreCase("sql")){
if (datasetTable.getType().equalsIgnoreCase("sql")) {
generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource));
}else {
} else {
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
}
Long execTime = System.currentTimeMillis();
......@@ -302,24 +296,24 @@ public class ExtractDataService {
replaceTable(DorisTableUtils.dorisName(datasetTableId));
saveSucessLog(datasetTableTaskLog);
sendWebMsg(datasetTable, taskId,true);
sendWebMsg(datasetTable, datasetTableTask, true);
deleteFile("all_scope", datasetTableId);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
datasetTableTask.setLastExecStatus(JobStatus.Completed.name());
dataSetTableTaskService.update(datasetTableTask);
dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Completed);
}catch (Exception e){
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
datasetTableTask.setLastExecStatus(JobStatus.Error.name());
dataSetTableTaskService.update(datasetTableTask);
dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Error);
sendWebMsg(datasetTable, taskId,false);
sendWebMsg(datasetTable, datasetTableTask, false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
deleteFile("all_scope", datasetTableId);
}finally {
} finally {
}
break;
......@@ -336,10 +330,10 @@ public class ExtractDataService {
return;
}
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
Long execTime = System.currentTimeMillis();
......@@ -360,34 +354,29 @@ public class ExtractDataService {
}
saveSucessLog(datasetTableTaskLog);
sendWebMsg(datasetTable, taskId,true);
sendWebMsg(datasetTable, datasetTableTask, true);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Completed);
}catch (Exception e){
datasetTableTask.setLastExecStatus(JobStatus.Completed.name());
dataSetTableTaskService.update(datasetTableTask);
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
sendWebMsg(datasetTable, taskId,false);
sendWebMsg(datasetTable, datasetTableTask, false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Error);
datasetTableTask.setLastExecStatus(JobStatus.Error.name());
dataSetTableTaskService.update(datasetTableTask);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
}finally {
if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){
datasetTableTask.setStatus(TaskStatus.Stopped.name());
dataSetTableTaskService.update(datasetTableTask);
}
} finally {
}
break;
}
//侵入式清除下属视图缓存
List<String> viewIds = extChartViewMapper.allViewIds(datasetTableId);
if (CollectionUtils.isNotEmpty(viewIds)){
if (CollectionUtils.isNotEmpty(viewIds)) {
viewIds.forEach(viewId -> {
CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId);
});
......@@ -395,7 +384,8 @@ public class ExtractDataService {
}
private void sendWebMsg(DatasetTable datasetTable, String taskId, Boolean status) {
private void sendWebMsg(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Boolean status) {
String taskId = datasetTableTask.getId();
String msg = status ? "成功" : "失败";
Long typeId = status ? 5L : 6L;
String id = datasetTable.getId();
......@@ -403,18 +393,22 @@ public class ExtractDataService {
Set<Long> userIds = AuthUtils.userIdsByURD(authURD);
Gson gson = new Gson();
userIds.forEach(userId -> {
Map<String,Object> param = new HashMap<>();
Map<String, Object> param = new HashMap<>();
param.put("tableId", id);
if(StringUtils.isNotEmpty(taskId)){
if (StringUtils.isNotEmpty(taskId)) {
param.put("taskId", taskId);
}
DeMsgutil.sendMsg(userId, typeId, 1L, "数据集【"+datasetTable.getName()+"】同步"+msg, gson.toJson(param));
String content = "数据集【" + datasetTable.getName() + "】同步" + msg;
if (ObjectUtils.isNotEmpty(datasetTableTask) && ObjectUtils.isNotEmpty(datasetTableTask.getName())) {
content += " 任务名称【" + datasetTableTask.getName() + "】";
}
DeMsgutil.sendMsg(userId, typeId, 1L, content, gson.toJson(param));
});
}
private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) {
datasetTable.setSyncStatus(completed.name());
if(execTime != null){
if (execTime != null) {
datasetTable.setLastUpdateTime(execTime);
}
DatasetTableExample example = new DatasetTableExample();
......@@ -428,16 +422,16 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void saveErrorLog(String datasetTableId, String taskId, Exception e){
private void saveErrorLog(String datasetTableId, String taskId, Exception e) {
LogUtil.error("Extract data error: " + datasetTableId, e);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
if(StringUtils.isNotEmpty(taskId)){
if (StringUtils.isNotEmpty(taskId)) {
datasetTableTaskLog.setTaskId(taskId);
}
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog = datasetTableTaskLogs.get(0);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(e.getMessage());
......@@ -456,7 +450,7 @@ public class ExtractDataService {
if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
Column_Fields = Column_Fields + "varchar(65533)" + ",`";
} else {
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize()*3)) + ",`";
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize() * 3)) + ",`";
}
break;
case 1:
......@@ -499,7 +493,8 @@ public class ExtractDataService {
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
jdbcProvider.exec(datasourceRequest);
}catch (Exception ignore){}
} catch (Exception ignore) {
}
}
private void replaceTable(String dorisTableName) throws Exception {
......@@ -512,14 +507,15 @@ public class ExtractDataService {
jdbcProvider.exec(datasourceRequest);
}
private DatasetTable getDatasetTable(String datasetTableId){
for (int i=0;i<5;i++){
private DatasetTable getDatasetTable(String datasetTableId) {
for (int i = 0; i < 5; i++) {
DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
if(datasetTable == null){
if (datasetTable == null) {
try {
Thread.sleep(1000);
}catch (Exception ignore){}
}else {
} catch (Exception ignore) {
}
} else {
return datasetTable;
}
}
......@@ -532,11 +528,11 @@ public class ExtractDataService {
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Cron.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isEmpty(datasetTableTaskLogs)){
if (CollectionUtils.isEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
return datasetTableTaskLog;
}else {
} else {
return datasetTableTaskLogs.get(0);
}
}
......@@ -545,14 +541,15 @@ public class ExtractDataService {
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
for (int i=0;i<5;i++){
for (int i = 0; i < 5; i++) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
return datasetTableTaskLogs.get(0);
}
try {
Thread.sleep(1000);
}catch (Exception ignore){}
} catch (Exception ignore) {
}
}
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
......@@ -560,7 +557,6 @@ public class ExtractDataService {
}
private void extractData(DatasetTable datasetTable, String extractType) throws Exception {
datasourceService.validate(datasetTable.getDataSourceId());
KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class);
RepositoryDirectoryInterface repositoryDirectoryInterface = repository.loadRepositoryDirectoryTree();
JobMeta jobMeta = null;
......@@ -706,7 +702,7 @@ public class ExtractDataService {
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> dorisFileds = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> {
dorisFileds.add(DorisTableUtils.columnName(filed));
});
return String.join(",", dorisFileds);
......@@ -728,7 +724,7 @@ public class ExtractDataService {
case mysql:
MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
dataMeta.addExtraOption("MYSQL","characterEncoding", "UTF-8");
dataMeta.addExtraOption("MYSQL", "characterEncoding", "UTF-8");
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
......@@ -744,10 +740,10 @@ public class ExtractDataService {
break;
case oracle:
OracleConfigration oracleConfigration = new Gson().fromJson(datasource.getConfiguration(), OracleConfigration.class);
if(oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")){
if (oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")) {
String database = "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ORACLE_HOSTNAME)(PORT = ORACLE_PORT))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = ORACLE_SERVICE_NAME )))".replace("ORACLE_HOSTNAME", oracleConfigration.getHost()).replace("ORACLE_PORT", oracleConfigration.getPort().toString()).replace("ORACLE_SERVICE_NAME", oracleConfigration.getDataBase());
dataMeta = new DatabaseMeta("db", "ORACLE", "Native", "", database, "-1", oracleConfigration.getUsername(), oracleConfigration.getPassword());
}else {
} else {
dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword());
}
transMeta.addDatabase(dataMeta);
......@@ -806,12 +802,12 @@ public class ExtractDataService {
selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields);
}
if(extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")){
if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")) {
selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
}
if(!extractType.equalsIgnoreCase("all_scope")){
if (!extractType.equalsIgnoreCase("all_scope")) {
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
}
......@@ -834,23 +830,25 @@ public class ExtractDataService {
ExcelInputMeta excelInputMeta = new ExcelInputMeta();
if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
try{
try {
InputStream inputStream = new FileInputStream(filePath);
XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream);
XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0);
excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
}
if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL);
try{
try {
InputStream inputStream = new FileInputStream(filePath);
HSSFWorkbook workbook = new HSSFWorkbook(inputStream);
HSSFSheet sheet0 = workbook.getSheetAt(0);
excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
}catch (Exception e){e.printStackTrace();}
} catch (Exception e) {
e.printStackTrace();
}
}
excelInputMeta.setPassword("Encrypted");
excelInputMeta.setFileName(new String[]{filePath});
......@@ -860,10 +858,10 @@ public class ExtractDataService {
for (int i = 0; i < datasetTableFields.size(); i++) {
ExcelInputField field = new ExcelInputField();
field.setName(datasetTableFields.get(i).getOriginName());
if(datasetTableFields.get(i).getDeExtractType() == 1){
if (datasetTableFields.get(i).getDeExtractType() == 1) {
field.setType("String");
field.setFormat("yyyy-MM-dd HH:mm:ss");
}else {
} else {
field.setType("String");
}
fields[i] = field;
......@@ -904,14 +902,14 @@ public class ExtractDataService {
tmp_code = tmp_code.replace("handleWraps", handleWraps);
String Column_Fields = "";
if(datasourceType.equals(DatasourceTypes.excel) || datasourceType.equals(DatasourceTypes.oracle)){
if (datasourceType.equals(DatasourceTypes.excel) || datasourceType.equals(DatasourceTypes.oracle)) {
Column_Fields = String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()));
}else {
} else {
Column_Fields = String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()));
}
if(datasourceType.equals(DatasourceTypes.excel)){
if (datasourceType.equals(DatasourceTypes.excel)) {
tmp_code = tmp_code.replace("handleExcelIntColumn", handleExcelIntColumn).replace("Column_Fields", Column_Fields);
}else {
} else {
tmp_code = tmp_code.replace("handleExcelIntColumn", "").replace("Column_Fields", Column_Fields);
}
UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code);
......@@ -926,7 +924,7 @@ public class ExtractDataService {
return userDefinedJavaClassStep;
}
public void deleteFile(String type, String dataSetTableId){
public void deleteFile(String type, String dataSetTableId) {
String transName = null;
String jobName = null;
String fileName = null;
......@@ -950,18 +948,21 @@ public class ExtractDataService {
default:
break;
}
try{
try {
File file = new File(root_path + fileName + "." + extention);
FileUtils.forceDelete(file);
}catch (Exception e){}
try{
} catch (Exception e) {
}
try {
File file = new File(root_path + jobName + ".kjb");
FileUtils.forceDelete(file);
}catch (Exception e){}
try{
} catch (Exception e) {
}
try {
File file = new File(root_path + transName + ".ktr");
FileUtils.forceDelete(file);
}catch (Exception e){}
} catch (Exception e) {
}
}
public boolean isKettleRunning() {
......@@ -969,7 +970,7 @@ public class ExtractDataService {
if (!InetAddress.getByName(carte).isReachable(1000)) {
return false;
}
}catch (Exception e){
} catch (Exception e) {
return false;
}
......
backend/src/main/resources/db/migration/V10__web_msg.sql
浏览文件 @ 214b6f86
......@@ -56,9 +56,9 @@ BEGIN;
INSERT INTO `sys_msg_type` VALUES (1, 0, 'i18n_msg_type_panel_share', 'panel', 'to-msg-share');
INSERT INTO `sys_msg_type` VALUES (2, 1, 'i18n_msg_type_panel_share', 'panel', 'to-msg-share');
INSERT INTO `sys_msg_type` VALUES (3, 1, 'i18n_msg_type_panel_share_cacnel', 'panel', 'to-msg-share');
INSERT INTO `sys_msg_type` VALUES (4, 0, 'i18n_msg_type_dataset_sync', 'dataset', 'to-msg-dataset');
INSERT INTO `sys_msg_type` VALUES (5, 4, 'i18n_msg_type_dataset_sync_success', 'dataset', 'to-msg-dataset');
INSERT INTO `sys_msg_type` VALUES (6, 4, 'i18n_msg_type_dataset_sync_faild', 'dataset', 'to-msg-dataset');
INSERT INTO `sys_msg_type` VALUES (4, 0, 'i18n_msg_type_dataset_sync', 'sys-task-dataset', 'to-msg-dataset');
INSERT INTO `sys_msg_type` VALUES (5, 4, 'i18n_msg_type_dataset_sync_success', 'sys-task-dataset', 'to-msg-dataset');
INSERT INTO `sys_msg_type` VALUES (6, 4, 'i18n_msg_type_dataset_sync_faild', 'sys-task-dataset', 'to-msg-dataset');
COMMIT;
-- ----------------------------
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论