feat: ck 支持直连模式；es支持 6.*

backend/pom.xml

@@ -342,6 +342,12 @@
             <artifactId>spring-boot-starter-data-ldap</artifactId>
         </dependency>-->
 
+<!--        <dependency>-->
+<!--            <groupId>io.dataease.kettle</groupId>-->
+<!--            <artifactId>ck-plugin</artifactId>-->
+<!--            <version>1.0</version>-->
+<!--        </dependency>-->
+
         <dependency>
             <groupId>ru.yandex.clickhouse</groupId>
             <artifactId>clickhouse-jdbc</artifactId>

backend/src/main/java/io/dataease/base/domain/Datasource.java

@@ -21,8 +21,6 @@ public class Datasource implements Serializable {
 
     private String status;
 
-    private String computeType;
-
     private String configuration;
 
     private static final long serialVersionUID = 1L;

backend/src/main/java/io/dataease/base/domain/DatasourceExample.java

@@ -643,76 +643,6 @@ public class DatasourceExample {
             addCriterion("`status` not between", value1, value2, "status");
             return (Criteria) this;
         }
-
-        public Criteria andComputeTypeIsNull() {
-            addCriterion("compute_type is null");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeIsNotNull() {
-            addCriterion("compute_type is not null");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeEqualTo(String value) {
-            addCriterion("compute_type =", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeNotEqualTo(String value) {
-            addCriterion("compute_type <>", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeGreaterThan(String value) {
-            addCriterion("compute_type >", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeGreaterThanOrEqualTo(String value) {
-            addCriterion("compute_type >=", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeLessThan(String value) {
-            addCriterion("compute_type <", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeLessThanOrEqualTo(String value) {
-            addCriterion("compute_type <=", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeLike(String value) {
-            addCriterion("compute_type like", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeNotLike(String value) {
-            addCriterion("compute_type not like", value, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeIn(List<String> values) {
-            addCriterion("compute_type in", values, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeNotIn(List<String> values) {
-            addCriterion("compute_type not in", values, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeBetween(String value1, String value2) {
-            addCriterion("compute_type between", value1, value2, "computeType");
-            return (Criteria) this;
-        }
-
-        public Criteria andComputeTypeNotBetween(String value1, String value2) {
-            addCriterion("compute_type not between", value1, value2, "computeType");
-            return (Criteria) this;
-        }
     }
 
     public static class Criteria extends GeneratedCriteria {

backend/src/main/java/io/dataease/base/mapper/DatasourceMapper.xml

@@ -10,7 +10,6 @@
     <result column="update_time" jdbcType="BIGINT" property="updateTime" />
     <result column="create_by" jdbcType="VARCHAR" property="createBy" />
     <result column="status" jdbcType="VARCHAR" property="status" />
-    <result column="compute_type" jdbcType="VARCHAR" property="computeType" />
   </resultMap>
   <resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.base.domain.Datasource">
     <result column="configuration" jdbcType="LONGVARCHAR" property="configuration" />
@@ -74,7 +73,7 @@
     </where>
   </sql>
   <sql id="Base_Column_List">
-    id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`, compute_type
+    id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`
   </sql>
   <sql id="Blob_Column_List">
     configuration
@@ -130,12 +129,12 @@
   <insert id="insert" parameterType="io.dataease.base.domain.Datasource">
     insert into datasource (id, `name`, `desc`, 
       `type`, create_time, update_time, 
-      create_by, `status`, compute_type, 
-      configuration)
+      create_by, `status`, configuration
+      )
     values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR}, 
       #{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT}, 
-      #{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{computeType,jdbcType=VARCHAR}, 
-      #{configuration,jdbcType=LONGVARCHAR})
+      #{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR}
+      )
   </insert>
   <insert id="insertSelective" parameterType="io.dataease.base.domain.Datasource">
     insert into datasource
@@ -164,9 +163,6 @@
       <if test="status != null">
         `status`,
       </if>
-      <if test="computeType != null">
-        compute_type,
-      </if>
       <if test="configuration != null">
         configuration,
       </if>
@@ -196,9 +192,6 @@
       <if test="status != null">
         #{status,jdbcType=VARCHAR},
       </if>
-      <if test="computeType != null">
-        #{computeType,jdbcType=VARCHAR},
-      </if>
       <if test="configuration != null">
         #{configuration,jdbcType=LONGVARCHAR},
       </if>
@@ -237,9 +230,6 @@
       <if test="record.status != null">
         `status` = #{record.status,jdbcType=VARCHAR},
       </if>
-      <if test="record.computeType != null">
-        compute_type = #{record.computeType,jdbcType=VARCHAR},
-      </if>
       <if test="record.configuration != null">
         configuration = #{record.configuration,jdbcType=LONGVARCHAR},
       </if>
@@ -258,7 +248,6 @@
       update_time = #{record.updateTime,jdbcType=BIGINT},
       create_by = #{record.createBy,jdbcType=VARCHAR},
       `status` = #{record.status,jdbcType=VARCHAR},
-      compute_type = #{record.computeType,jdbcType=VARCHAR},
       configuration = #{record.configuration,jdbcType=LONGVARCHAR}
     <if test="_parameter != null">
       <include refid="Update_By_Example_Where_Clause" />
@@ -273,8 +262,7 @@
       create_time = #{record.createTime,jdbcType=BIGINT},
       update_time = #{record.updateTime,jdbcType=BIGINT},
       create_by = #{record.createBy,jdbcType=VARCHAR},
-      `status` = #{record.status,jdbcType=VARCHAR},
-      compute_type = #{record.computeType,jdbcType=VARCHAR}
+      `status` = #{record.status,jdbcType=VARCHAR}
     <if test="_parameter != null">
       <include refid="Update_By_Example_Where_Clause" />
     </if>
@@ -303,9 +291,6 @@
       <if test="status != null">
         `status` = #{status,jdbcType=VARCHAR},
       </if>
-      <if test="computeType != null">
-        compute_type = #{computeType,jdbcType=VARCHAR},
-      </if>
       <if test="configuration != null">
         configuration = #{configuration,jdbcType=LONGVARCHAR},
       </if>
@@ -321,7 +306,6 @@
       update_time = #{updateTime,jdbcType=BIGINT},
       create_by = #{createBy,jdbcType=VARCHAR},
       `status` = #{status,jdbcType=VARCHAR},
-      compute_type = #{computeType,jdbcType=VARCHAR},
       configuration = #{configuration,jdbcType=LONGVARCHAR}
     where id = #{id,jdbcType=VARCHAR}
   </update>
@@ -333,8 +317,7 @@
       create_time = #{createTime,jdbcType=BIGINT},
       update_time = #{updateTime,jdbcType=BIGINT},
       create_by = #{createBy,jdbcType=VARCHAR},
-      `status` = #{status,jdbcType=VARCHAR},
-      compute_type = #{computeType,jdbcType=VARCHAR}
+      `status` = #{status,jdbcType=VARCHAR}
     where id = #{id,jdbcType=VARCHAR}
   </update>
 </mapper>
\ No newline at end of file

backend/src/main/java/io/dataease/config/CommonConfig.java

@@ -4,7 +4,10 @@ import com.alibaba.fastjson.JSONObject;
 import com.fit2cloud.autoconfigure.QuartzAutoConfiguration;
 import io.dataease.base.domain.Datasource;
 import io.dataease.commons.utils.CommonThreadPool;
+import io.dataease.kettle.databaseplugin.ClickhouseDatabaseMeta;
 import org.pentaho.di.core.KettleEnvironment;
+import org.pentaho.di.core.plugins.DatabasePluginType;
+import org.pentaho.di.core.plugins.PluginRegistry;
 import org.pentaho.di.repository.filerep.KettleFileRepository;
 import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta;
 import org.springframework.boot.autoconfigure.AutoConfigureBefore;

backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java

@@ -7,7 +7,7 @@ public enum DatasourceTypes {
     sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
     doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
     oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
-    ch("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
+    ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
     es("es", "es", "", "\"", "\"", "\"", "\"");
 
     private String feature;

backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java

@@ -10,5 +10,7 @@ public class EsConfigDTO {
     private String url;
     private String username;
     private String password;
+    private String version;
+    private String uri;
     private String dataSourceType = "es";
 }

backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java

@@ -12,6 +12,7 @@ public class EsReponse {
     private String cursor;
     private Integer status;
     private Error error;
+    private String version;
 
     @Data
     public class Error{

backend/src/main/java/io/dataease/datasource/provider/EsProvider.java

 package io.dataease.datasource.provider;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
 import com.google.gson.Gson;
 import com.mchange.v2.c3p0.ComboPooledDataSource;
 import io.dataease.commons.utils.HttpClientConfig;
@@ -10,6 +12,7 @@ import io.dataease.datasource.dto.es.Requst;
 import io.dataease.datasource.dto.es.RequstWithCursor;
 import io.dataease.datasource.request.DatasourceRequest;
 import io.dataease.exception.DataEaseException;
+import io.dataease.i18n.Translator;
 import io.dataease.provider.es.EsQueryProvider;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang.StringUtils;
@@ -54,7 +57,7 @@ public class EsProvider extends DatasourceProvider {
             Requst requst = new Requst();
             requst.setQuery(dsr.getQuery());
             requst.setFetch_size(dsr.getFetchSize());
-            String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + "_sql?format=json" : esConfigDTO.getUrl() + "/" + "_sql?format=json";
+            String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + esConfigDTO.getUri() + "?format=json" : esConfigDTO.getUrl() + "/" + esConfigDTO.getUri() + "?format=json";
             String  response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
             EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
 
@@ -84,7 +87,7 @@ public class EsProvider extends DatasourceProvider {
     public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception {
         List<String[]> list = new LinkedList<>();
         try {
-            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
+            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
             list = fetchResult(response);
         } catch (Exception e) {
             DataEaseException.throwException(e);
@@ -110,7 +113,7 @@ public class EsProvider extends DatasourceProvider {
     public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
         List<TableFiled> tableFileds = new ArrayList<>();
         try {
-            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
+            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
             tableFileds = fetchResultField(response);
         } catch (Exception e) {
             DataEaseException.throwException(e);
@@ -140,7 +143,7 @@ public class EsProvider extends DatasourceProvider {
     public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
         Map<String, List> result = new HashMap<>();
         try {
-            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json");
+            String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
             result.put("dataList", fetchResult(response));
             result.put("fieldList", fetchResultField(response));
         } catch (Exception e) {
@@ -157,7 +160,7 @@ public class EsProvider extends DatasourceProvider {
     public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
         List<String> tables = new ArrayList<>();
         try {
-            String response = exexQuery(datasourceRequest, "show tables", "_sql?format=json");
+            String response = exexQuery(datasourceRequest, "show tables", "?format=json");
             tables = fetchTables(response);
         } catch (Exception e) {
             DataEaseException.throwException(e);
@@ -176,6 +179,9 @@ public class EsProvider extends DatasourceProvider {
            if(row.length == 3 && row[1].equalsIgnoreCase("TABLE") && row[2].equalsIgnoreCase("INDEX")){
                tables.add(row[0]);
            }
+            if(row.length == 2 && row[1].equalsIgnoreCase("BASE TABLE")){
+                tables.add(row[0]);
+            }
         }
         return tables;
     }
@@ -189,7 +195,7 @@ public class EsProvider extends DatasourceProvider {
     public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
         List<TableFiled> tableFileds = new ArrayList<>();
         try {
-            String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "_sql?format=json");
+            String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "?format=json");
             tableFileds = fetchResultField(response);
         } catch (Exception e) {
             DataEaseException.throwException(e);
@@ -199,16 +205,34 @@ public class EsProvider extends DatasourceProvider {
 
     @Override
     public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
+        EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
+        String response = exexGetQuery(datasourceRequest);
+        String version =  JSONObject.parseObject(response).getJSONObject("version").getString("number");
+        if(Integer.valueOf(version.substring(0,1)) < 6 ){
+            throw new Exception(Translator.get("i18n_es_limit"));
+        }
+        if(Integer.valueOf(version.substring(2,3)) < 3 ){
+            throw new Exception(Translator.get("i18n_es_limit"));
+        }
+        if(Integer.valueOf(version.substring(0,1)) == 6 ) {
+            esConfigDTO.setUri("_xpack/sql");
+        }
+        if(Integer.valueOf(version.substring(0,1)) == 7 ) {
+            esConfigDTO.setUri("_sql");
+        }
+        datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfigDTO));
         getTables(datasourceRequest);
     }
 
     private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){
         EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
-
+        uri = esConfigDTO.getUri()+uri;
         HttpClientConfig httpClientConfig = new HttpClientConfig();
+        if(StringUtils.isNotEmpty(esConfigDTO.getUsername()) && StringUtils.isNotEmpty(esConfigDTO.getPassword())){
             String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
             byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
             httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
+        }
 
         Requst requst = new Requst();
         requst.setQuery(sql);
@@ -218,4 +242,17 @@ public class EsProvider extends DatasourceProvider {
         return response;
     }
 
+    private String exexGetQuery(DatasourceRequest datasourceRequest){
+        EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
+        HttpClientConfig httpClientConfig = new HttpClientConfig();
+        if(StringUtils.isNotEmpty(esConfigDTO.getUsername()) && StringUtils.isNotEmpty(esConfigDTO.getPassword())){
+            String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
+            byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
+            httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
+        }
+
+        String  response = HttpClientUtil.get(esConfigDTO.getUrl(), httpClientConfig);
+        return response;
+    }
+
 }

backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java

@@ -7,6 +7,7 @@ import io.dataease.datasource.dto.*;
 import io.dataease.datasource.request.DatasourceRequest;
 import io.dataease.exception.DataEaseException;
 import io.dataease.i18n.Translator;
+import io.dataease.provider.QueryProvider;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 import java.beans.PropertyVetoException;
@@ -127,7 +128,7 @@ public class JdbcProvider extends DatasourceProvider {
             connection = getConnectionFromPool(datasourceRequest);
             Statement stat = connection.createStatement();
             rs = stat.executeQuery(datasourceRequest.getQuery());
-            return fetchResultField(rs);
+            return fetchResultField(rs, datasourceRequest);
         } catch (SQLException e) {
             DataEaseException.throwException(e);
         } catch (Exception e) {
@@ -152,7 +153,7 @@ public class JdbcProvider extends DatasourceProvider {
             Statement stat = connection.createStatement();
             rs = stat.executeQuery(datasourceRequest.getQuery());
             dataList = fetchResult(rs);
-            fieldList = fetchResultField(rs);
+            fieldList = fetchResultField(rs, datasourceRequest);
             result.put("dataList", dataList);
             result.put("fieldList", fieldList);
             return result;
@@ -168,10 +169,11 @@ public class JdbcProvider extends DatasourceProvider {
         return new HashMap<>();
     }
 
-    private List<TableFiled> fetchResultField(ResultSet rs) throws Exception {
+    private List<TableFiled> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
         List<TableFiled> fieldList = new ArrayList<>();
         ResultSetMetaData metaData = rs.getMetaData();
         int columnCount = metaData.getColumnCount();
+        QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
         for (int j = 0; j < columnCount; j++) {
             String f = metaData.getColumnName(j + 1);
             String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f;
@@ -180,7 +182,12 @@ public class JdbcProvider extends DatasourceProvider {
             field.setFieldName(l);
             field.setRemarks(l);
             field.setFieldType(t);
+
+            if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
+                field.setFieldSize(qp.transFieldSize(t));
+            }else {
                 field.setFieldSize(metaData.getColumnDisplaySize(j + 1));
+            }
             if(t.equalsIgnoreCase("LONG")){field.setFieldSize(65533);} //oracle LONG
             if(StringUtils.isNotEmpty(t) && t.toLowerCase().contains("date") && field.getFieldSize() < 50 ){
                 field.setFieldSize(50);
@@ -251,19 +258,19 @@ public class JdbcProvider extends DatasourceProvider {
             while (resultSet.next()) {
                 String tableName = resultSet.getString("TABLE_NAME");
                 String database = null;
-                if(datasourceRequest.getDatasource().getType().equalsIgnoreCase("ch")){
+                if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
                     database = resultSet.getString("TABLE_SCHEM");
                 }else {
                     database = resultSet.getString("TABLE_CAT");
                 }
                 if(database != null){
                     if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) {
-                        TableFiled tableFiled = getTableFiled(resultSet);
+                        TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
                         list.add(tableFiled);
                     }
                 }else {
                     if (tableName.equals(datasourceRequest.getTable())) {
-                        TableFiled tableFiled = getTableFiled(resultSet);
+                        TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
                         list.add(tableFiled);
                     }
                 }
@@ -281,7 +288,7 @@ public class JdbcProvider extends DatasourceProvider {
         return list;
     }
 
-    private TableFiled getTableFiled(ResultSet resultSet) throws SQLException {
+    private TableFiled getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
         TableFiled tableFiled = new TableFiled();
         String colName = resultSet.getString("COLUMN_NAME");
         tableFiled.setFieldName(colName);
@@ -290,13 +297,19 @@ public class JdbcProvider extends DatasourceProvider {
             remarks = colName;
         }
         tableFiled.setRemarks(remarks);
-        tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE")));
         String dbType = resultSet.getString("TYPE_NAME").toUpperCase();
         tableFiled.setFieldType(dbType);
         if(dbType.equalsIgnoreCase("LONG")){tableFiled.setFieldSize(65533);}
         if(StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50 ){
             tableFiled.setFieldSize(50);
         }
+
+        if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
+            QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
+            tableFiled.setFieldSize(qp.transFieldSize(dbType));
+        }else {
+            tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE")));
+        }
         return tableFiled;
     }
 
@@ -442,7 +455,7 @@ public class JdbcProvider extends DatasourceProvider {
                 driver = pgConfigration.getDriver();
                 jdbcurl = pgConfigration.getJdbc();
                 break;
-            case ch:
+            case ck:
                 CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
                 username = chConfigration.getUsername();
                 password = chConfigration.getPassword();
@@ -506,7 +519,7 @@ public class JdbcProvider extends DatasourceProvider {
                 dataSource.setJdbcUrl(pgConfigration.getJdbc());
                 jdbcDTO = pgConfigration;
                 break;
-            case ch:
+            case ck:
                 CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
                 dataSource.setUser(chConfigration.getUsername());
                 dataSource.setDriverClass(chConfigration.getDriver());
@@ -568,7 +581,7 @@ public class JdbcProvider extends DatasourceProvider {
                     throw new Exception(Translator.get("i18n_schema_is_empty"));
                 }
                 return "SELECT tablename FROM  pg_tables WHERE  tablename NOT LIKE 'pg%' AND tablename NOT LIKE 'sql_%' AND schemaname='SCHEMA' ;".replace("SCHEMA", pgConfigration.getSchema());
-            case ch:
+            case ck:
                 CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
                 return "SELECT name FROM system.tables where database='DATABASE';".replace("DATABASE", chConfigration.getDataBase());
             default:

backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java

@@ -52,8 +52,8 @@ public class ProviderFactory implements ApplicationContextAware {
                 return context.getBean("oracleQuery", QueryProvider.class);
             case es:
                 return context.getBean("esQuery", QueryProvider.class);
-            case ch:
-                return context.getBean("chQuery", QueryProvider.class);
+            case ck:
+                return context.getBean("ckQuery", QueryProvider.class);
             default:
                 return context.getBean("mysqlQuery", QueryProvider.class);
         }

backend/src/main/java/io/dataease/datasource/service/DatasourceService.java

@@ -65,9 +65,10 @@ public class DatasourceService {
         datasource.setUpdateTime(currentTimeMillis);
         datasource.setCreateTime(currentTimeMillis);
         datasource.setCreateBy(String.valueOf(AuthUtils.getUser().getUsername()));
+        checkAndUpdateDatasourceStatus(datasource);
         datasourceMapper.insertSelective(datasource);
         handleConnectionPool(datasource, "add");
-        checkAndUpdateDatasourceStatus(datasource);
+
         return datasource;
     }
 
@@ -129,9 +130,9 @@ public class DatasourceService {
         checkName(datasource);
         datasource.setCreateTime(null);
         datasource.setUpdateTime(System.currentTimeMillis());
+        checkAndUpdateDatasourceStatus(datasource);
         datasourceMapper.updateByPrimaryKeySelective(datasource);
         handleConnectionPool(datasource, "edit");
-        checkAndUpdateDatasourceStatus(datasource);
     }
 
     public ResultHolder validate(Datasource datasource) throws Exception {
@@ -256,10 +257,8 @@ public class DatasourceService {
             datasourceRequest.setDatasource(datasource);
             datasourceProvider.checkStatus(datasourceRequest);
             datasource.setStatus("Success");
-            datasourceMapper.updateByPrimaryKeySelective(datasource);
         } catch (Exception e) {
             datasource.setStatus("Error");
-            datasourceMapper.updateByPrimaryKeySelective(datasource);
         }
     }
 

backend/src/main/java/io/dataease/provider/QueryProvider.java

@@ -54,6 +54,9 @@ public abstract class QueryProvider {
 
     public abstract String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
 
+    public  Integer transFieldSize(String type){
+        return 50;
+    };
     /**
      * 单指标汇总
      *

backend/src/main/java/io/dataease/provider/ch/CHConstants.java → backend/src/main/java/io/dataease/provider/ck/CKConstants.java

-package io.dataease.provider.ch;
+package io.dataease.provider.ck;
 
 import io.dataease.provider.SQLConstants;
 
-import static io.dataease.datasource.constants.DatasourceTypes.ch;
+import static io.dataease.datasource.constants.DatasourceTypes.ck;
 
 /**
  * @Author gin
  * @Date 2021/7/8 7:22 下午
  */
-public class CHConstants extends SQLConstants {
-    public static final String KEYWORD_TABLE = ch.getKeywordPrefix() + "%s" + ch.getKeywordSuffix();
+public class CKConstants extends SQLConstants {
+    public static final String KEYWORD_TABLE = ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix();
 
-    public static final String KEYWORD_FIX = "%s." + ch.getKeywordPrefix() + "%s" + ch.getKeywordSuffix();
+    public static final String KEYWORD_FIX = "%s." + ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix();
 
     public static final String toInt32 = "toInt32(%s)";
 

backend/src/main/java/io/dataease/provider/ch/CHQueryProvider.java → backend/src/main/java/io/dataease/provider/ck/CKQueryProvider.java

-package io.dataease.provider.ch;
+package io.dataease.provider.ck;
 
 import io.dataease.base.domain.DatasetTableField;
 import io.dataease.base.domain.DatasetTableFieldExample;
@@ -32,8 +32,8 @@ import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
  * @Author gin
  * @Date 2021/5/17 2:43 下午
  */
-@Service("chQuery")
-public class CHQueryProvider extends QueryProvider {
+@Service("ckQuery")
+public class CKQueryProvider extends QueryProvider {
     @Resource
     private DatasetTableFieldMapper datasetTableFieldMapper;
 
@@ -80,6 +80,25 @@ public class CHQueryProvider extends QueryProvider {
         }
     }
 
+    @Override
+    public Integer transFieldSize(String field){
+        Integer type = transFieldType(field);
+        switch (type) {
+            case 0:
+                return 65533;
+            case 1:
+                return 60;
+            case 2:
+                return 0;
+            case 3:
+                return 0;
+            case 4:
+                return 0;
+            default:
+                return 65533;
+        }
+    }
+
     @Override
     public String createSQLPreview(String sql, String orderBy) {
         return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 0,1000";
@@ -88,7 +107,7 @@ public class CHQueryProvider extends QueryProvider {
     @Override
     public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds) {
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> xFields = new ArrayList<>();
@@ -100,9 +119,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(f.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
                 String fieldName = "";
@@ -110,29 +129,29 @@ public class CHQueryProvider extends QueryProvider {
                 if (f.getDeExtractType() == DeTypeConstants.DE_TIME) {
                     if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) {
                         if(f.getType().equalsIgnoreCase("DATE")){
-                            fieldName = String.format(CHConstants.toInt32, String.format(CHConstants.toDateTime, originField)) + "*1000";
+                            fieldName = String.format(CKConstants.toInt32, String.format(CKConstants.toDateTime, originField)) + "*1000";
                         }else {
-                            fieldName = String.format(CHConstants.toInt32, originField) + "*1000";
+                            fieldName = String.format(CKConstants.toInt32, originField) + "*1000";
                         }
                     } else {
                         fieldName = originField;
                     }
                 } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) {
                     if (f.getDeType() == DeTypeConstants.DE_INT) {
-                        fieldName = String.format(CHConstants.toInt64, originField);
+                        fieldName = String.format(CKConstants.toInt64, originField);
                     } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) {
-                        fieldName = String.format(CHConstants.toFloat64, originField);
+                        fieldName = String.format(CKConstants.toFloat64, originField);
                     } else if (f.getDeType() == DeTypeConstants.DE_TIME) {
-                        fieldName = String.format(CHConstants.toDateTime, originField);
+                        fieldName = String.format(CKConstants.toDateTime, originField);
                     } else {
                         fieldName = originField;
                     }
                 } else {
                     if (f.getDeType() == DeTypeConstants.DE_TIME) {
-                        String cast = String.format(CHConstants.toFloat64, originField);
-                        fieldName = String.format(CHConstants.toDateTime, cast);
+                        String cast = String.format(CKConstants.toFloat64, originField);
+                        fieldName = String.format(CKConstants.toDateTime, cast);
                     } else if (f.getDeType() == DeTypeConstants.DE_INT) {
-                        fieldName = String.format(CHConstants.toInt64, originField);
+                        fieldName = String.format(CKConstants.toInt64, originField);
                     } else {
                         fieldName = originField;
                     }
@@ -180,7 +199,7 @@ public class CHQueryProvider extends QueryProvider {
     @Override
     public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) {
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> xFields = new ArrayList<>();
@@ -194,9 +213,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(x.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
                 // 处理横轴字段
@@ -224,9 +243,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(y.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
                 // 处理纵轴字段
@@ -274,7 +293,7 @@ public class CHQueryProvider extends QueryProvider {
 
         ST st = stg.getInstanceOf("querySql");
         SQLObj tableSQL = SQLObj.builder()
-                .tableName(String.format(CHConstants.BRACKETS, sql))
+                .tableName(String.format(CKConstants.BRACKETS, sql))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
                 .build();
         if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@@ -286,7 +305,7 @@ public class CHQueryProvider extends QueryProvider {
     @Override
     public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) {
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> xFields = new ArrayList<>();
@@ -300,9 +319,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(x.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
                 // 处理横轴字段
@@ -347,7 +366,7 @@ public class CHQueryProvider extends QueryProvider {
         ST st = stg.getInstanceOf("previewSql");
         st.add("isGroup", false);
         SQLObj tableSQL = SQLObj.builder()
-                .tableName(String.format(CHConstants.BRACKETS, sql))
+                .tableName(String.format(CKConstants.BRACKETS, sql))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
                 .build();
         if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
@@ -369,7 +388,7 @@ public class CHQueryProvider extends QueryProvider {
     @Override
     public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds) {
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> xFields = new ArrayList<>();
@@ -386,9 +405,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(x.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
                 // 处理横轴字段
@@ -416,9 +435,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(y.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
                 // 处理纵轴字段
@@ -466,7 +485,7 @@ public class CHQueryProvider extends QueryProvider {
 
         ST st = stg.getInstanceOf("querySql");
         SQLObj tableSQL = SQLObj.builder()
-                .tableName(String.format(CHConstants.BRACKETS, sql))
+                .tableName(String.format(CKConstants.BRACKETS, sql))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
                 .build();
         if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@@ -483,7 +502,7 @@ public class CHQueryProvider extends QueryProvider {
     @Override
     public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds) {
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> xFields = new ArrayList<>();
@@ -497,9 +516,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(x.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
                 // 处理横轴字段
@@ -530,9 +549,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(y.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
                 // 处理纵轴字段
@@ -580,7 +599,7 @@ public class CHQueryProvider extends QueryProvider {
 
         ST st = stg.getInstanceOf("querySql");
         SQLObj tableSQL = SQLObj.builder()
-                .tableName(String.format(CHConstants.BRACKETS, sql))
+                .tableName(String.format(CKConstants.BRACKETS, sql))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
                 .build();
         if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@@ -603,7 +622,7 @@ public class CHQueryProvider extends QueryProvider {
     public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
         // 字段汇总 排序等
         SQLObj tableObj = SQLObj.builder()
-                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table))
+                .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
                 .build();
         List<SQLObj> yFields = new ArrayList<>();
@@ -617,9 +636,9 @@ public class CHQueryProvider extends QueryProvider {
                     // 解析origin name中有关联的字段生成sql表达式
                     originField = calcFieldRegex(y.getOriginName(), tableObj);
                 } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 } else {
-                    originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
+                    originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
                 }
                 String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
                 // 处理纵轴字段
@@ -662,7 +681,7 @@ public class CHQueryProvider extends QueryProvider {
 
         ST st = stg.getInstanceOf("querySql");
         SQLObj tableSQL = SQLObj.builder()
-                .tableName(String.format(CHConstants.BRACKETS, sql))
+                .tableName(String.format(CKConstants.BRACKETS, sql))
                 .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
                 .build();
         if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@@ -757,17 +776,17 @@ public class CHQueryProvider extends QueryProvider {
                 // 解析origin name中有关联的字段生成sql表达式
                 originName = calcFieldRegex(field.getOriginName(), tableObj);
             } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
-                originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
+                originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
             } else {
-                originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
+                originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
             }
             if (field.getDeType() == DeTypeConstants.DE_TIME) {
                 if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
-                    whereName = String.format(CHConstants.toDateTime, originName);
+                    whereName = String.format(CKConstants.toDateTime, originName);
                 }
                 if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) {
-                    String cast = String.format(CHConstants.toFloat64, originName);
-                    whereName = String.format(CHConstants.toDateTime, cast);
+                    String cast = String.format(CKConstants.toFloat64, originName);
+                    whereName = String.format(CKConstants.toDateTime, cast);
                 }
                 if (field.getDeExtractType() == 1) {
                     whereName = originName;
@@ -776,7 +795,7 @@ public class CHQueryProvider extends QueryProvider {
                 whereName = originName;
             }
             if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) {
-                whereValue = CHConstants.WHERE_VALUE_NULL;
+                whereValue = CKConstants.WHERE_VALUE_NULL;
             } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) {
                 whereTerm = String.format(whereTerm, originName);
             } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) {
@@ -784,7 +803,7 @@ public class CHQueryProvider extends QueryProvider {
             } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) {
                 whereValue = "'%" + value + "%'";
             } else {
-                whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, value);
+                whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, value);
             }
             list.add(SQLObj.builder()
                     .whereField(whereName)
@@ -814,18 +833,18 @@ public class CHQueryProvider extends QueryProvider {
                 // 解析origin name中有关联的字段生成sql表达式
                 originName = calcFieldRegex(field.getOriginName(), tableObj);
             } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
-                originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
+                originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
             } else {
-                originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
+                originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
             }
 
             if (field.getDeType() == DeTypeConstants.DE_TIME) {
                 if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
-                    whereName = String.format(CHConstants.toDateTime, originName);
+                    whereName = String.format(CKConstants.toDateTime, originName);
                 }
                 if (field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) {
-                    String cast = String.format(CHConstants.toFloat64, originName);
-                    whereName = String.format(CHConstants.toDateTime, cast);
+                    String cast = String.format(CKConstants.toFloat64, originName);
+                    whereName = String.format(CKConstants.toDateTime, cast);
                 }
                 if (field.getDeExtractType() == 1) {
                     whereName = originName;
@@ -844,12 +863,12 @@ public class CHQueryProvider extends QueryProvider {
                     SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                     String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
                     String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
-                    whereValue = String.format(CHConstants.WHERE_BETWEEN, startTime, endTime);
+                    whereValue = String.format(CKConstants.WHERE_BETWEEN, startTime, endTime);
                 } else {
-                    whereValue = String.format(CHConstants.WHERE_BETWEEN, value.get(0), value.get(1));
+                    whereValue = String.format(CKConstants.WHERE_BETWEEN, value.get(0), value.get(1));
                 }
             } else {
-                whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, value.get(0));
+                whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, value.get(0));
             }
             list.add(SQLObj.builder()
                     .whereField(whereName)
@@ -897,13 +916,13 @@ public class CHQueryProvider extends QueryProvider {
         if (x.getDeExtractType() == DeTypeConstants.DE_TIME) {
             if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) {
                 if(x.getType().equalsIgnoreCase("DATE")){
-                    fieldName = String.format(CHConstants.toInt32, String.format(CHConstants.toDateTime, originField)) + "*1000";
+                    fieldName = String.format(CKConstants.toInt32, String.format(CKConstants.toDateTime, originField)) + "*1000";
                 }else {
-                    fieldName = String.format(CHConstants.toInt32, originField) + "*1000";
+                    fieldName = String.format(CKConstants.toInt32, originField) + "*1000";
                 }
             } else if (x.getDeType() == DeTypeConstants.DE_TIME) {
                 String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
-                fieldName = String.format(CHConstants.formatDateTime, originField, format);
+                fieldName = String.format(CKConstants.formatDateTime, originField, format);
             } else {
                 fieldName = originField;
             }
@@ -911,9 +930,9 @@ public class CHQueryProvider extends QueryProvider {
             if (x.getDeType() == DeTypeConstants.DE_TIME) {
                 String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
                 if (x.getDeExtractType() == DeTypeConstants.DE_STRING) {
-                    fieldName = String.format(CHConstants.formatDateTime, String.format(CHConstants.toDateTime, originField), format);
+                    fieldName = String.format(CKConstants.formatDateTime, String.format(CKConstants.toDateTime, originField), format);
                 } else {
-                    fieldName = String.format(CHConstants.formatDateTime, String.format(CHConstants.toDateTime, String.format(CHConstants.toFloat64, originField)), format);
+                    fieldName = String.format(CKConstants.formatDateTime, String.format(CKConstants.toDateTime, String.format(CKConstants.toFloat64, originField)), format);
                 }
             } else {
                 fieldName = originField;
@@ -929,17 +948,17 @@ public class CHQueryProvider extends QueryProvider {
     private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
         String fieldName = "";
         if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) {
-            fieldName = CHConstants.AGG_COUNT;
+            fieldName = CKConstants.AGG_COUNT;
         } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
-            fieldName = String.format(CHConstants.AGG_FIELD, y.getSummary(), originField);
+            fieldName = String.format(CKConstants.AGG_FIELD, y.getSummary(), originField);
         } else {
             if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
-                String cast = y.getDeType() == 2? String.format(CHConstants.toInt64, originField) : String.format(CHConstants.toFloat64, originField);
-                String agg = String.format(CHConstants.AGG_FIELD, y.getSummary(), cast);
-                fieldName = String.format(CHConstants.toDecimal, agg);
+                String cast = y.getDeType() == 2? String.format(CKConstants.toInt64, originField) : String.format(CKConstants.toFloat64, originField);
+                String agg = String.format(CKConstants.AGG_FIELD, y.getSummary(), cast);
+                fieldName = String.format(CKConstants.toDecimal, agg);
             } else {
-                String cast = y.getDeType() == 2 ? String.format(CHConstants.toInt64, originField) : String.format(CHConstants.toFloat64, originField);
-                fieldName = String.format(CHConstants.AGG_FIELD, y.getSummary(), cast);
+                String cast = y.getDeType() == 2 ? String.format(CKConstants.toInt64, originField) : String.format(CKConstants.toFloat64, originField);
+                fieldName = String.format(CKConstants.AGG_FIELD, y.getSummary(), cast);
             }
         }
         return SQLObj.builder()
@@ -956,7 +975,7 @@ public class CHQueryProvider extends QueryProvider {
                 String whereValue = "";
                 // 原始类型不是时间，在de中被转成时间的字段做处理
                 if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
-                    whereValue = CHConstants.WHERE_VALUE_NULL;
+                    whereValue = CKConstants.WHERE_VALUE_NULL;
                 } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
                     whereTerm = String.format(whereTerm, originField);
                 } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
@@ -964,7 +983,7 @@ public class CHQueryProvider extends QueryProvider {
                 } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
                     whereValue = "'%" + f.getValue() + "%'";
                 } else {
-                    whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, f.getValue());
+                    whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, f.getValue());
                 }
                 list.add(SQLObj.builder()
                         .whereField(fieldAlias)
@@ -995,7 +1014,7 @@ public class CHQueryProvider extends QueryProvider {
         List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
         for (DatasetTableField ele : calcFields) {
             originField = originField.replaceAll("\\[" + ele.getId() + "]",
-                    String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
+                    String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
         }
         return originField;
     }

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -469,6 +469,7 @@ public class DataSetTableService {
                 QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
                 datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
                 map.put("sql", datasourceRequest.getQuery());
+                System.out.println(datasourceRequest.getQuery());
                 datasourceRequest.setPage(page);
                 datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
                 datasourceRequest.setPageSize(pageSize);
@@ -503,8 +504,12 @@ public class DataSetTableService {
                 QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
                 datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
                 map.put("sql", datasourceRequest.getQuery());
+                System.out.println(datasourceRequest.getQuery());
                 try {
                     data.addAll(jdbcProvider.getData(datasourceRequest));
+                    for(int i=0;i< data.size(); i++){
+                        System.out.println(data.get(i)[2]);
+                    }
                 } catch (Exception e) {
                     e.printStackTrace();
                     DEException.throwException(e.getMessage());
@@ -558,8 +563,12 @@ public class DataSetTableService {
                 QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
                 datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
                 map.put("sql", datasourceRequest.getQuery());
+                System.out.println(datasourceRequest.getQuery());
                 try {
                     data.addAll(jdbcProvider.getData(datasourceRequest));
+                    for(int i=0;i< data.size(); i++){
+                        System.out.println(data.get(i)[2]);
+                    }
                 } catch (Exception e) {
                     e.printStackTrace();
                     DEException.throwException(e.getMessage());
@@ -585,8 +594,12 @@ public class DataSetTableService {
             QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
             datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
             map.put("sql", datasourceRequest.getQuery());
+            System.out.println(datasourceRequest.getQuery());
             try {
                 data.addAll(jdbcProvider.getData(datasourceRequest));
+                for(int i=0;i< data.size(); i++){
+                    System.out.println(data.get(i)[2]);
+                }
             } catch (Exception e) {
                 e.printStackTrace();
                 DEException.throwException(e.getMessage());

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -31,6 +31,9 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.pentaho.di.cluster.SlaveServer;
 import org.pentaho.di.core.database.DatabaseMeta;
+import org.pentaho.di.core.plugins.DatabasePluginType;
+import org.pentaho.di.core.plugins.PluginInterface;
+import org.pentaho.di.core.plugins.PluginRegistry;
 import org.pentaho.di.core.row.ValueMetaInterface;
 import org.pentaho.di.core.util.HttpClientManager;
 import org.pentaho.di.job.Job;
@@ -312,6 +315,7 @@ public class ExtractDataService {
                     msg = true;
                     lastExecStatus = JobStatus.Completed;
                 } catch (Exception e) {
+                    e.printStackTrace();
                     saveErrorLog(datasetTableId, taskId, e);
                     msg = false;
                     lastExecStatus = JobStatus.Error;
@@ -456,7 +460,7 @@ public class ExtractDataService {
         for (DatasetTableField datasetTableField : datasetTableFields) {
             Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
             Integer size = datasetTableField.getSize() * 3;
-            if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
+            if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
                 size = 65533;
             }
             switch (datasetTableField.getDeExtractType()) {
@@ -468,7 +472,7 @@ public class ExtractDataService {
                     Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`";
                     break;
                 case 2:
-                    Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(size)) + ",`";
+                    Column_Fields = Column_Fields + "bigint" + ",`";
                     break;
                 case 3:
                     Column_Fields = Column_Fields + "DOUBLE" + ",`";
@@ -616,7 +620,8 @@ public class ExtractDataService {
             Thread.sleep(1000);
         }
         if (!transStatus.getStatusDescription().equals("Finished")) {
-            DataEaseException.throwException((transStatus.getLoggingString()));
+            DataEaseException.throwException(transStatus.getLoggingString());
+            return;
         }
 
         executing = true;
@@ -637,11 +642,6 @@ public class ExtractDataService {
         }
     }
 
-    private boolean isExitFile(String fileName) {
-        File file = new File(root_path + fileName);
-        return file.exists();
-    }
-
     private SlaveServer getSlaveServer() {
         SlaveServer remoteSlaveServer = new SlaveServer();
         remoteSlaveServer.setHostname(carte);// 设置远程IP
@@ -805,6 +805,16 @@ public class ExtractDataService {
                 inputStep = inputStep(transMeta, selectSQL);
                 udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle);
                 break;
+            case ck:
+                CHConfigration chConfigration = new Gson().fromJson(datasource.getConfiguration(), CHConfigration.class);
+                dataMeta = new DatabaseMeta("db", "ORACLE", "Native", chConfigration.getHost().trim(), chConfigration.getDataBase().trim(), chConfigration.getPort().toString(), chConfigration.getUsername(), chConfigration.getPassword());
+//                dataMeta.addExtraOption("MYSQL", "characterEncoding", "UTF-8");
+                dataMeta.setDatabaseType("Clickhouse");
+                transMeta.addDatabase(dataMeta);
+                selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
+                inputStep = inputStep(transMeta, selectSQL);
+                udjcStep = udjc(datasetTableFields, DatasourceTypes.ck);
+                break;
             case excel:
                 inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields);
                 udjcStep = udjc(datasetTableFields, DatasourceTypes.excel);

backend/src/main/resources/i18n/messages_en_US.properties

@@ -286,3 +286,4 @@ i18n_Stopped=END
 i18n_Exec=Running
 i18n_no_trigger=The current setting does not trigger task generation.
 i18n_dataset_field_delete=Union field deleted,please set again and redo.
+i18n_es_limit=Elasticsearch version cannot be less than 6.3
\ No newline at end of file

backend/src/main/resources/i18n/messages_zh_CN.properties

@@ -285,4 +285,4 @@ i18n_Stopped=执行结束
 i18n_Exec=运行中
 i18n_no_trigger=当前设置没有触发任务生成
 i18n_dataset_field_delete=该自定义数据集有关联字段被删除，请重新确认关联关系并重做该数据集
-
+i18n_es_limit=Elasticsearch 版本不能小于6.3

backend/src/main/resources/i18n/messages_zh_TW.properties

@@ -288,3 +288,4 @@ i18n_Stopped=執行結束
 i18n_Exec=運行中
 i18n_no_trigger=当前设置没有触发任务生成 當前設置沒有觸發任務生成.
 i18n_dataset_field_delete=該自定義數據集有關聯字段被刪除，請重新確認關聯關系並重做該數據集
+i18n_es_limit=Elasticsearch 版本不能小於6.3
\ No newline at end of file

frontend/src/views/dataset/add/AddDB.vue

@@ -28,8 +28,8 @@
         </el-form-item>
         <el-form-item class="form-item">
           <el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
-            <el-option :label="$t('dataset.direct_connect')" value="0" :disabled="selectedDatasource.computeType==='EXTRACT'"/>
-            <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.computeType==='DIRECT'" />
+            <el-option :label="$t('dataset.direct_connect')" value="0" />
+            <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck'" />
           </el-select>
         </el-form-item>
 

frontend/src/views/dataset/add/AddSQL.vue

@@ -32,8 +32,8 @@
           </el-form-item>
           <el-form-item v-if="!param.tableId" class="form-item">
             <el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
-              <el-option :label="$t('dataset.direct_connect')" value="0" :disabled="selectedDatasource.computeType==='EXTRACT'"/>
-              <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.computeType==='DIRECT'" />
+              <el-option :label="$t('dataset.direct_connect')" value="0" />
+              <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck'" />
             </el-select>
           </el-form-item>
 

frontend/src/views/system/datasource/form.vue

@@ -24,17 +24,6 @@
           </el-select>
         </el-form-item>
 
-        <el-form-item :label="$t('datasource.data_mode')" prop="type">
-          <el-select v-model="form.computeType" :placeholder="$t('datasource.please_choose_data_type')" class="select-width" :disabled="formType=='modify' || (formType==='add' && params && !!params.computeMode)">
-            <el-option
-              v-for="item in compute_mode"
-              :key="item.type"
-              :label="item.label"
-              :value="item.type"
-            />
-          </el-select>
-        </el-form-item>
-
         <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.host')" prop="configuration.host">
           <el-input v-model="form.configuration.host" autocomplete="off" />
         </el-form-item>
@@ -50,10 +39,10 @@
           <el-radio v-model="form.configuration.connectionType" label="serviceName">{{ $t('datasource.oracle_service_name') }}</el-radio>
         </el-form-item>
 
-        <el-form-item :label="$t('datasource.user_name')" prop="configuration.username">
+        <el-form-item :label="$t('datasource.user_name')"  >
           <el-input v-model="form.configuration.username" autocomplete="off" />
         </el-form-item>
-        <el-form-item :label="$t('datasource.password')" prop="configuration.password">
+        <el-form-item :label="$t('datasource.password')"  >
           <el-input v-model="form.configuration.password" autocomplete="off" show-password />
         </el-form-item>
         <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.port')" prop="configuration.port">
@@ -164,14 +153,9 @@ export default {
         { name: 'sqlServer', label: 'SQL Server', type: 'jdbc' },
         { name: 'pg', label: 'PostgreSQL', type: 'jdbc' },
         { name: 'es', label: 'Elasticsearch', type: 'es' },
-        { name: 'ch', label: 'ClickHouse', type: 'jdbc' }
+        { name: 'ck', label: 'ClickHouse', type: 'jdbc' }
         ],
       schemas: [],
-      compute_mode: [
-        {type: "DIRECT",  label: this.$t('datasource.direct')},
-        {type: "EXTRACT", label: this.$t('datasource.extract')},
-        {type: "ALL",     label: this.$t('datasource.all_compute_mode')}
-      ],
       canEdit: false,
       originConfiguration: {}
     }
@@ -222,6 +206,14 @@ export default {
       this.$refs.dsForm.resetFields()
     },
     save() {
+      if(this.form.type !== 'es' && !this.form.configuration.username){
+          this.$message.error(this.$t('datasource.please_input_user_name'))
+          return
+      }
+      if(this.form.type !== 'es' && !this.form.configuration.username){
+          this.$message.error(this.$t('datasource.please_input_password'))
+          return
+      }
       if (!this.form.configuration.schema && (this.form.type === 'oracle' || this.form.type === 'sqlServer')) {
         this.$message.error(this.$t('datasource.please_choose_schema'))
         return
@@ -312,15 +304,6 @@ export default {
           this.form.configuration.dataSourceType = this.allTypes[i].type
         }
       }
-      if(this.form.type === 'es'){
-        this.compute_mode = [{type: "DIRECT", label: this.$t('datasource.direct')}];
-      }else {
-        this.compute_mode = [
-          {type: "DIRECT", label: this.$t('datasource.direct')},
-          {type: "EXTRACT", label: this.$t('datasource.extract')},
-          {type: "ALL", label: this.$t('datasource.all_compute_mode')}
-        ];
-      }
     },
     backToList() {
       this.$emit('switch-component', { })