Skip to content

D
dataease
提交 260d09a1 authored 作者: taojinlong's avatar taojinlong
浏览文件
操作

feat: 抽取 excel 数据

上级 ac938c28
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 156 行增加 和 10 行删除
backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
浏览文件 @ 260d09a1
......@@ -16,6 +16,8 @@ public class DatasetTableField implements Serializable {
private String name;
private String dataeaseName;
private String type;
private Integer size;
......
backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java
浏览文件 @ 260d09a1
......@@ -384,6 +384,76 @@ public class DatasetTableFieldExample {
return (Criteria) this;
}
public Criteria andDataeaseNameIsNull() {
addCriterion("dataease_name is null");
return (Criteria) this;
}
public Criteria andDataeaseNameIsNotNull() {
addCriterion("dataease_name is not null");
return (Criteria) this;
}
public Criteria andDataeaseNameEqualTo(String value) {
addCriterion("dataease_name =", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotEqualTo(String value) {
addCriterion("dataease_name <>", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameGreaterThan(String value) {
addCriterion("dataease_name >", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameGreaterThanOrEqualTo(String value) {
addCriterion("dataease_name >=", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLessThan(String value) {
addCriterion("dataease_name <", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLessThanOrEqualTo(String value) {
addCriterion("dataease_name <=", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLike(String value) {
addCriterion("dataease_name like", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotLike(String value) {
addCriterion("dataease_name not like", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameIn(List<String> values) {
addCriterion("dataease_name in", values, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotIn(List<String> values) {
addCriterion("dataease_name not in", values, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameBetween(String value1, String value2) {
addCriterion("dataease_name between", value1, value2, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotBetween(String value1, String value2) {
addCriterion("dataease_name not between", value1, value2, "dataeaseName");
return (Criteria) this;
}
public Criteria andTypeIsNull() {
addCriterion("`type` is null");
return (Criteria) this;
......
backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml
浏览文件 @ 260d09a1
......@@ -6,6 +6,7 @@
<result column="table_id" jdbcType="VARCHAR" property="tableId" />
<result column="origin_name" jdbcType="VARCHAR" property="originName" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="dataease_name" jdbcType="VARCHAR" property="dataeaseName" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="size" jdbcType="INTEGER" property="size" />
<result column="de_type" jdbcType="INTEGER" property="deType" />
......@@ -72,8 +73,8 @@
</where>
</sql>
<sql id="Base_Column_List">
id, table_id, origin_name, `name`, `type`, `size`, de_type, `checked`, column_index,
last_sync_time
id, table_id, origin_name, `name`, dataease_name, `type`, `size`, de_type, `checked`,
column_index, last_sync_time
</sql>
<select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableFieldExample" resultMap="BaseResultMap">
select
......@@ -107,13 +108,13 @@
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DatasetTableField">
insert into dataset_table_field (id, table_id, origin_name,
`name`, `type`, `size`, de_type,
`checked`, column_index, last_sync_time
)
`name`, dataease_name, `type`,
`size`, de_type, `checked`,
column_index, last_sync_time)
values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{originName,jdbcType=VARCHAR},
#{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER},
#{checked,jdbcType=BIT}, #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT}
)
#{name,jdbcType=VARCHAR}, #{dataeaseName,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR},
#{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, #{checked,jdbcType=BIT},
#{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT})
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableField">
insert into dataset_table_field
......@@ -130,6 +131,9 @@
<if test="name != null">
`name`,
</if>
<if test="dataeaseName != null">
dataease_name,
</if>
<if test="type != null">
`type`,
</if>
......@@ -162,6 +166,9 @@
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="dataeaseName != null">
#{dataeaseName,jdbcType=VARCHAR},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
......@@ -203,6 +210,9 @@
<if test="record.name != null">
`name` = #{record.name,jdbcType=VARCHAR},
</if>
<if test="record.dataeaseName != null">
dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
</if>
<if test="record.type != null">
`type` = #{record.type,jdbcType=VARCHAR},
</if>
......@@ -232,6 +242,7 @@
table_id = #{record.tableId,jdbcType=VARCHAR},
origin_name = #{record.originName,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
`size` = #{record.size,jdbcType=INTEGER},
de_type = #{record.deType,jdbcType=INTEGER},
......@@ -254,6 +265,9 @@
<if test="name != null">
`name` = #{name,jdbcType=VARCHAR},
</if>
<if test="dataeaseName != null">
dataease_name = #{dataeaseName,jdbcType=VARCHAR},
</if>
<if test="type != null">
`type` = #{type,jdbcType=VARCHAR},
</if>
......@@ -280,6 +294,7 @@
set table_id = #{tableId,jdbcType=VARCHAR},
origin_name = #{originName,jdbcType=VARCHAR},
`name` = #{name,jdbcType=VARCHAR},
dataease_name = #{dataeaseName,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
`size` = #{size,jdbcType=INTEGER},
de_type = #{deType,jdbcType=INTEGER},
......
backend/src/main/java/io/dataease/commons/utils/Md5Utils.java 0 → 100644
浏览文件 @ 260d09a1
package io.dataease.commons.utils;
import java.util.List;
import java.io.File;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class Md5Utils {
private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
private static final String UTF_8 = "UTF-8";
public static String md5(String src) {
return md5(src, UTF_8);
}
public static String md5(String src, String charset) {
try {
byte[] strTemp = charset == null || charset.equals("") ? src.getBytes() : src.getBytes(charset);
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(strTemp);
byte[] md = mdTemp.digest();
int j = md.length;
char[] str = new char[j * 2];
int k = 0;
for (byte byte0 : md) {
str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
str[k++] = HEX_DIGITS[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
throw new RuntimeException("MD5 encrypt error:", e);
}
}
}
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
浏览文件 @ 260d09a1
package io.dataease.datasource.constants;
public enum DatasourceTypes {
mysql, sqlServer
mysql, sqlServer, excel
}
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
浏览文件 @ 260d09a1
......@@ -8,6 +8,8 @@ import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.commons.utils.Md5Utils;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.dto.TableFiled;
......@@ -54,6 +56,10 @@ public class DataSetTableService {
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private CommonThreadPool commonThreadPool;
@Resource
private ExtractDataService extractDataService;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
@Value("${upload.file.path}")
private String path;
......@@ -79,6 +85,9 @@ public class DataSetTableService {
// 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
if (insert == 1) {
saveTableField(datasetTable);
commonThreadPool.addTask(()->{
extractDataService.extractData(datasetTable.getId(), null, "all_scope");
});
}
} else {
int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
......@@ -330,6 +339,11 @@ public class DataSetTableService {
datasetTableField.setTableId(datasetTable.getId());
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
if(StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")){
datasetTableField.setDataeaseName("C_" + Md5Utils.md5(filed.getFieldName()));
}else {
datasetTableField.setDataeaseName(filed.getFieldName());
}
datasetTableField.setType(filed.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
......@@ -560,6 +574,7 @@ public class DataSetTableService {
tableFiled.setFieldName(readCell(row.getCell(j)));
tableFiled.setRemarks(readCell(row.getCell(j)));
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
fields.add(tableFiled);
} else {
r[j] = readCell(row.getCell(j));
......
backend/src/main/resources/db/migration/V16__dataset_table_field.sql 0 → 100644
浏览文件 @ 260d09a1
ALTER TABLE `dataset_table_field` ADD COLUMN `dataease_name` varchar(255) NOT NULL COMMENT '字段名' AFTER `name`;
ALTER TABLE dataset_table_task_log CHANGE COLUMN `task_id` `task_id` VARCHAR(50) NULL COMMENT '任务ID' ;
\ No newline at end of file
backend/src/main/resources/generatorConfig.xml
浏览文件 @ 260d09a1
......@@ -67,7 +67,7 @@
<!-- <table tableName="datasource"/>-->
<!-- <table tableName="sys_dict"/>-->
<!-- <table tableName="sys_dict_item"/>-->
<table tableName="chart_view"/>
<table tableName="dataset_table_field"/>
<!-- <table tableName="panel_design"/>-->
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论