feat: 抽取 excel 数据

backend/src/main/java/io/dataease/base/domain/DatasetTableField.java

@@ -16,6 +16,8 @@ public class DatasetTableField implements Serializable {
 
     private String name;
 
+    private String dataeaseName;
+
     private String type;
 
     private Integer size;

backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java

@@ -384,6 +384,76 @@ public class DatasetTableFieldExample {
             return (Criteria) this;
         }
 
+        public Criteria andDataeaseNameIsNull() {
+            addCriterion("dataease_name is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameIsNotNull() {
+            addCriterion("dataease_name is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameEqualTo(String value) {
+            addCriterion("dataease_name =", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotEqualTo(String value) {
+            addCriterion("dataease_name <>", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameGreaterThan(String value) {
+            addCriterion("dataease_name >", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameGreaterThanOrEqualTo(String value) {
+            addCriterion("dataease_name >=", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLessThan(String value) {
+            addCriterion("dataease_name <", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLessThanOrEqualTo(String value) {
+            addCriterion("dataease_name <=", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLike(String value) {
+            addCriterion("dataease_name like", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotLike(String value) {
+            addCriterion("dataease_name not like", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameIn(List<String> values) {
+            addCriterion("dataease_name in", values, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotIn(List<String> values) {
+            addCriterion("dataease_name not in", values, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameBetween(String value1, String value2) {
+            addCriterion("dataease_name between", value1, value2, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotBetween(String value1, String value2) {
+            addCriterion("dataease_name not between", value1, value2, "dataeaseName");
+            return (Criteria) this;
+        }
+
         public Criteria andTypeIsNull() {
             addCriterion("`type` is null");
             return (Criteria) this;

backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml

@@ -6,6 +6,7 @@
     <result column="table_id" jdbcType="VARCHAR" property="tableId" />
     <result column="origin_name" jdbcType="VARCHAR" property="originName" />
     <result column="name" jdbcType="VARCHAR" property="name" />
+    <result column="dataease_name" jdbcType="VARCHAR" property="dataeaseName" />
     <result column="type" jdbcType="VARCHAR" property="type" />
     <result column="size" jdbcType="INTEGER" property="size" />
     <result column="de_type" jdbcType="INTEGER" property="deType" />
@@ -72,8 +73,8 @@
     </where>
   </sql>
   <sql id="Base_Column_List">
-    id, table_id, origin_name, `name`, `type`, `size`, de_type, `checked`, column_index, 
-    last_sync_time
+    id, table_id, origin_name, `name`, dataease_name, `type`, `size`, de_type, `checked`, 
+    column_index, last_sync_time
   </sql>
   <select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableFieldExample" resultMap="BaseResultMap">
     select
@@ -107,13 +108,13 @@
   </delete>
   <insert id="insert" parameterType="io.dataease.base.domain.DatasetTableField">
     insert into dataset_table_field (id, table_id, origin_name, 
-      `name`, `type`, `size`, de_type, 
-      `checked`, column_index, last_sync_time
-      )
+      `name`, dataease_name, `type`, 
+      `size`, de_type, `checked`, 
+      column_index, last_sync_time)
     values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{originName,jdbcType=VARCHAR}, 
-      #{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, 
-      #{checked,jdbcType=BIT}, #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT}
-      )
+      #{name,jdbcType=VARCHAR}, #{dataeaseName,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, 
+      #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, #{checked,jdbcType=BIT}, 
+      #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT})
   </insert>
   <insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableField">
     insert into dataset_table_field
@@ -130,6 +131,9 @@
       <if test="name != null">
         `name`,
       </if>
+      <if test="dataeaseName != null">
+        dataease_name,
+      </if>
       <if test="type != null">
         `type`,
       </if>
@@ -162,6 +166,9 @@
       <if test="name != null">
         #{name,jdbcType=VARCHAR},
       </if>
+      <if test="dataeaseName != null">
+        #{dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="type != null">
         #{type,jdbcType=VARCHAR},
       </if>
@@ -203,6 +210,9 @@
       <if test="record.name != null">
         `name` = #{record.name,jdbcType=VARCHAR},
       </if>
+      <if test="record.dataeaseName != null">
+        dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="record.type != null">
         `type` = #{record.type,jdbcType=VARCHAR},
       </if>
@@ -232,6 +242,7 @@
       table_id = #{record.tableId,jdbcType=VARCHAR},
       origin_name = #{record.originName,jdbcType=VARCHAR},
       `name` = #{record.name,jdbcType=VARCHAR},
+      dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
       `type` = #{record.type,jdbcType=VARCHAR},
       `size` = #{record.size,jdbcType=INTEGER},
       de_type = #{record.deType,jdbcType=INTEGER},
@@ -254,6 +265,9 @@
       <if test="name != null">
         `name` = #{name,jdbcType=VARCHAR},
       </if>
+      <if test="dataeaseName != null">
+        dataease_name = #{dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="type != null">
         `type` = #{type,jdbcType=VARCHAR},
       </if>
@@ -280,6 +294,7 @@
     set table_id = #{tableId,jdbcType=VARCHAR},
       origin_name = #{originName,jdbcType=VARCHAR},
       `name` = #{name,jdbcType=VARCHAR},
+      dataease_name = #{dataeaseName,jdbcType=VARCHAR},
       `type` = #{type,jdbcType=VARCHAR},
       `size` = #{size,jdbcType=INTEGER},
       de_type = #{deType,jdbcType=INTEGER},

backend/src/main/java/io/dataease/commons/utils/Md5Utils.java

+package io.dataease.commons.utils;
+
+import java.util.List;
+import java.io.File;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class Md5Utils {
+    private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+    private static final String UTF_8 = "UTF-8";
+
+    public static String md5(String src) {
+        return md5(src, UTF_8);
+    }
+
+    public static String md5(String src, String charset) {
+        try {
+            byte[] strTemp = charset == null || charset.equals("") ? src.getBytes() : src.getBytes(charset);
+            MessageDigest mdTemp = MessageDigest.getInstance("MD5");
+            mdTemp.update(strTemp);
+
+            byte[] md = mdTemp.digest();
+            int j = md.length;
+            char[] str = new char[j * 2];
+            int k = 0;
+
+            for (byte byte0 : md) {
+                str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
+                str[k++] = HEX_DIGITS[byte0 & 0xf];
+            }
+
+            return new String(str);
+        } catch (Exception e) {
+            throw new RuntimeException("MD5 encrypt error:", e);
+        }
+    }
+}

backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java

 package io.dataease.datasource.constants;
 
 public enum DatasourceTypes {
-    mysql, sqlServer
+    mysql, sqlServer, excel
 }

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -8,6 +8,8 @@ import io.dataease.base.mapper.DatasetTableMapper;
 import io.dataease.base.mapper.DatasourceMapper;
 import io.dataease.commons.utils.AuthUtils;
 import io.dataease.commons.utils.BeanUtils;
+import io.dataease.commons.utils.CommonThreadPool;
+import io.dataease.commons.utils.Md5Utils;
 import io.dataease.controller.request.dataset.DataSetTableRequest;
 import io.dataease.datasource.constants.DatasourceTypes;
 import io.dataease.datasource.dto.TableFiled;
@@ -54,6 +56,10 @@ public class DataSetTableService {
     @Resource
     private DataSetTableTaskService dataSetTableTaskService;
     @Resource
+    private CommonThreadPool commonThreadPool;
+    @Resource
+    private ExtractDataService extractDataService;
+    @Resource
     private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
     @Value("${upload.file.path}")
     private String path;
@@ -79,6 +85,9 @@ public class DataSetTableService {
             // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
             if (insert == 1) {
                 saveTableField(datasetTable);
+                commonThreadPool.addTask(()->{
+                    extractDataService.extractData(datasetTable.getId(), null, "all_scope");
+                });
             }
         } else {
             int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
@@ -330,6 +339,11 @@ public class DataSetTableService {
                 datasetTableField.setTableId(datasetTable.getId());
                 datasetTableField.setOriginName(filed.getFieldName());
                 datasetTableField.setName(filed.getRemarks());
+                if(StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")){
+                    datasetTableField.setDataeaseName("C_" + Md5Utils.md5(filed.getFieldName()));
+                }else {
+                    datasetTableField.setDataeaseName(filed.getFieldName());
+                }
                 datasetTableField.setType(filed.getFieldType());
                 if (ObjectUtils.isEmpty(ds)) {
                     datasetTableField.setDeType(transFieldType(filed.getFieldType()));
@@ -560,6 +574,7 @@ public class DataSetTableService {
                         tableFiled.setFieldName(readCell(row.getCell(j)));
                         tableFiled.setRemarks(readCell(row.getCell(j)));
                         tableFiled.setFieldType("TEXT");
+                        tableFiled.setFieldSize(1024);
                         fields.add(tableFiled);
                     } else {
                         r[j] = readCell(row.getCell(j));

backend/src/main/resources/db/migration/V16__dataset_table_field.sql

+ALTER TABLE `dataset_table_field` ADD COLUMN `dataease_name`  varchar(255) NOT NULL COMMENT '字段名'  AFTER `name`;
+ALTER TABLE dataset_table_task_log CHANGE COLUMN `task_id` `task_id` VARCHAR(50) NULL COMMENT '任务ID' ;
\ No newline at end of file

backend/src/main/resources/generatorConfig.xml

@@ -67,7 +67,7 @@
         <!--        <table tableName="datasource"/>-->
 <!--        <table tableName="sys_dict"/>-->
 <!--        <table tableName="sys_dict_item"/>-->
-        <table tableName="chart_view"/>
+        <table tableName="dataset_table_field"/>
 <!--        <table tableName="panel_design"/>-->