提交 87c5a440 authored 作者: junjie's avatar junjie

feat(数据集):excel解析date数据,默认存日期类型

上级 e05dd9f6
...@@ -26,6 +26,7 @@ import io.dataease.provider.QueryProvider; ...@@ -26,6 +26,7 @@ import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.hssf.usermodel.HSSFRow; import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.hssf.usermodel.HSSFWorkbook;
...@@ -45,6 +46,7 @@ import java.math.BigDecimal; ...@@ -45,6 +46,7 @@ import java.math.BigDecimal;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.text.NumberFormat; import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
...@@ -273,16 +275,16 @@ public class DataSetTableService { ...@@ -273,16 +275,16 @@ public class DataSetTableService {
map.put("status", "warnning"); map.put("status", "warnning");
map.put("msg", Translator.get("i18n_processing_data")); map.put("msg", Translator.get("i18n_processing_data"));
dataSetPreviewPage.setTotal(0); dataSetPreviewPage.setTotal(0);
}else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) { } else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId()); List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId());
map.put("status", "error"); map.put("status", "error");
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){ if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo()); map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo());
}else { } else {
map.put("msg", "Failed to extract data."); map.put("msg", "Failed to extract data.");
} }
dataSetPreviewPage.setTotal(0); dataSetPreviewPage.setTotal(0);
}else { } else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
...@@ -336,7 +338,7 @@ public class DataSetTableService { ...@@ -336,7 +338,7 @@ public class DataSetTableService {
}).collect(Collectors.toList()); }).collect(Collectors.toList());
} }
if(!map.containsKey("status")){ if (!map.containsKey("status")) {
map.put("status", "success"); map.put("status", "success");
} }
map.put("fields", fields); map.put("fields", fields);
...@@ -799,29 +801,38 @@ public class DataSetTableService { ...@@ -799,29 +801,38 @@ public class DataSetTableService {
return map; return map;
} }
private void inferFieldType(List<TableFiled> fields, List<String[]> data){ private void inferFieldType(List<TableFiled> fields, List<String[]> data) {
if(CollectionUtils.isEmpty(fields) || CollectionUtils.isEmpty(data)) { if (CollectionUtils.isEmpty(fields) || CollectionUtils.isEmpty(data)) {
return; return;
} }
String[] firstLine = data.get(0); String[] firstLine = data.get(0);
for (int i=0; i< fields.size()&& i < firstLine.length; i++) { for (int i = 0; i < fields.size() && i < firstLine.length; i++) {
TableFiled filed = fields.get(i); TableFiled filed = fields.get(i);
try{ try {
Integer.valueOf(firstLine[i]); Integer.valueOf(firstLine[i]);
filed.setFieldType("INT"); filed.setFieldType("INT");
continue; continue;
}catch (Exception ignore ){ } catch (Exception ignore) {
} }
try{ try {
Long.valueOf(firstLine[i]); Long.valueOf(firstLine[i]);
filed.setFieldType("LONG"); filed.setFieldType("LONG");
continue; continue;
}catch (Exception ignore ){} } catch (Exception ignore) {
try{ }
try {
Double.valueOf(firstLine[i]); Double.valueOf(firstLine[i]);
filed.setFieldType("DOUBLE"); filed.setFieldType("DOUBLE");
continue; continue;
}catch (Exception ignore ){} } catch (Exception ignore) {
}
try {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
simpleDateFormat.parse(firstLine[i]);
filed.setFieldType("TIME");
continue;
} catch (Exception ignore) {
}
} }
} }
...@@ -830,13 +841,19 @@ public class DataSetTableService { ...@@ -830,13 +841,19 @@ public class DataSetTableService {
if (cellTypeEnum.equals(CellType.STRING)) { if (cellTypeEnum.equals(CellType.STRING)) {
return cell.getStringCellValue(); return cell.getStringCellValue();
} else if (cellTypeEnum.equals(CellType.NUMERIC)) { } else if (cellTypeEnum.equals(CellType.NUMERIC)) {
if (HSSFDateUtil.isCellDateFormatted(cell)) {
Date date = cell.getDateCellValue();
//格式转换
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return sdf.format(date);
} else {
double d = cell.getNumericCellValue(); double d = cell.getNumericCellValue();
try { try {
Double value = new Double(d); Double value = new Double(d);
double eps = 1e-10; double eps = 1e-10;
if(value - Math.floor(value) < eps){ if (value - Math.floor(value) < eps) {
return value.longValue() + ""; return value.longValue() + "";
}else { } else {
NumberFormat nf = NumberFormat.getInstance(); NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(false); nf.setGroupingUsed(false);
return nf.format(value); return nf.format(value);
...@@ -845,6 +862,7 @@ public class DataSetTableService { ...@@ -845,6 +862,7 @@ public class DataSetTableService {
BigDecimal b = new BigDecimal(d); BigDecimal b = new BigDecimal(d);
return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + ""; return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + "";
} }
}
} else if (cellTypeEnum.equals(CellType.BOOLEAN)) { } else if (cellTypeEnum.equals(CellType.BOOLEAN)) {
return cell.getBooleanCellValue() ? "1" : "0"; return cell.getBooleanCellValue() ? "1" : "0";
} else { } else {
...@@ -885,7 +903,7 @@ public class DataSetTableService { ...@@ -885,7 +903,7 @@ public class DataSetTableService {
private UtilMapper utilMapper; private UtilMapper utilMapper;
@QuartzScheduled(cron = "0 0/3 * * * ?") @QuartzScheduled(cron = "0 0/3 * * * ?")
public void updateDatasetTableStatus(){ public void updateDatasetTableStatus() {
List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null);
List<String> activeQrtzInstances = qrtzSchedulerStates.stream().filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()).map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); List<String> activeQrtzInstances = qrtzSchedulerStates.stream().filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()).map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>(); List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
...@@ -893,12 +911,12 @@ public class DataSetTableService { ...@@ -893,12 +911,12 @@ public class DataSetTableService {
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name());
datasetTableMapper.selectByExample(example).forEach(datasetTable -> { datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if(StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))){ if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
jobStoppeddDatasetTables.add(datasetTable); jobStoppeddDatasetTables.add(datasetTable);
} }
}); });
if(CollectionUtils.isEmpty(jobStoppeddDatasetTables)){ if (CollectionUtils.isEmpty(jobStoppeddDatasetTables)) {
return; return;
} }
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论