Skip to content

D
dataease
Unverified 提交 19107a74 authored 作者: taojinlong's avatar taojinlong 提交者： GitHub
浏览文件
操作

Merge pull request #1924 from dataease/pr@dev@impala

Merge branch 'dev' into pr@dev@impala
上级 c43ccee1 f4ee879c
显示空白字符变更
内嵌 并排
正在显示 包含 1225 行增加 和 69 行删除
backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
浏览文件 @ 19107a74
......@@ -4,6 +4,7 @@ public enum DatasourceTypes {
excel("excel", "excel", "", "", "", "", ""),
mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
hive("hive", "hive", "org.apache.hive.jdbc.HiveDriver", "`", "`", "'", "'"),
impala("impala", "impala", "org.apache.hive.jdbc.HiveDriver", "`", "`", "'", "'"),
mariadb("mariadb", "mariadb", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
......
backend/src/main/java/io/dataease/dto/datasource/ImpalaConfiguration.java 0 → 100644
浏览文件 @ 19107a74
package io.dataease.dto.datasource;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@Getter
@Setter
public class ImpalaConfiguration extends JdbcConfiguration {
private String driver = "com.cloudera.impala.jdbc.Driver";
private String extraParams = "";
public String getJdbc() {
if(StringUtils.isEmpty(extraParams.trim())){
return "jdbc:impala://HOSTNAME:PORT/DATABASE"
.replace("HOSTNAME", getHost().trim())
.replace("PORT", getPort().toString().trim())
.replace("DATABASE", getDataBase().trim());
}else {
return "jdbc:impala://HOSTNAME:PORT/DATABASE;EXTRA_PARAMS"
.replace("HOSTNAME", getHost().trim())
.replace("PORT", getPort().toString().trim())
.replace("DATABASE", getDataBase().trim())
.replace("EXTRA_PARAMS", getExtraParams().trim());
}
}
}
\ No newline at end of file
backend/src/main/java/io/dataease/provider/ProviderFactory.java
浏览文件 @ 19107a74
......@@ -54,6 +54,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("redshiftQuery", QueryProvider.class);
case hive:
return context.getBean("hiveQuery", QueryProvider.class);
case impala:
return context.getBean("impalaQuery", QueryProvider.class);
case db2:
return context.getBean("db2Query", QueryProvider.class);
case api:
......
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
浏览文件 @ 19107a74
......@@ -154,7 +154,7 @@ public class JdbcProvider extends DatasourceProvider {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
String database;
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())) {
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name()) || datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.impala.name())) {
database = resultSet.getString("TABLE_SCHEM");
} else {
database = resultSet.getString("TABLE_CAT");
......@@ -485,6 +485,14 @@ public class JdbcProvider extends DatasourceProvider {
driver = hiveConfiguration.getDriver();
jdbcurl = hiveConfiguration.getJdbc();
break;
case impala:
ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class);
System.out.println(new Gson().toJson(impalaConfiguration));
username = impalaConfiguration.getUsername();
password = impalaConfiguration.getPassword();
driver = impalaConfiguration.getDriver();
jdbcurl = impalaConfiguration.getJdbc();
break;
case db2:
Db2Configuration db2Configuration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), Db2Configuration.class);
username = db2Configuration.getUsername();
......@@ -586,6 +594,13 @@ public class JdbcProvider extends DatasourceProvider {
dataSource.setUrl(hiveConfiguration.getJdbc());
jdbcConfiguration = hiveConfiguration;
break;
case impala:
ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class);
dataSource.setPassword(impalaConfiguration.getPassword());
dataSource.setDriverClassName(impalaConfiguration.getDriver());
dataSource.setUrl(impalaConfiguration.getJdbc());
jdbcConfiguration = impalaConfiguration;
break;
case db2:
Db2Configuration db2Configuration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), Db2Configuration.class);
dataSource.setPassword(db2Configuration.getPassword());
......@@ -614,6 +629,7 @@ public class JdbcProvider extends DatasourceProvider {
case engine_doris:
case ds_doris:
case hive:
case impala:
return "show tables";
case sqlServer:
SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfiguration.class);
......
backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
浏览文件 @ 19107a74
......@@ -2,16 +2,16 @@ package io.dataease.provider.query.hive;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;
import static io.dataease.commons.constants.DatasourceTypes.hive;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class HiveConstants extends SQLConstants {
public static final String KEYWORD_TABLE = mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix();
public static final String KEYWORD_TABLE = hive.getKeywordPrefix() + "%s" + hive.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + hive.getKeywordPrefix() + "%s" + hive.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "unix_timestamp(%s)";
......
backend/src/main/java/io/dataease/provider/query/impala/ImpalaConstants.java 0 → 100644
浏览文件 @ 19107a74
package io.dataease.provider.query.impala;
import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;
public class ImpalaConstants extends SQLConstants {
public static final String KEYWORD_TABLE = mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "unix_timestamp(%s)";
public static final String DATE_FORMAT = "from_unixtime(UNIX_TIMESTAMP(%s), '%s')";
public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')";
public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')";
public static final String CAST = "CAST(%s AS %s)";
public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static final String DEFAULT_INT_FORMAT = "BIGINT";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
public static final String WHERE_VALUE_NULL = "(NULL,'')";
public static final String WHERE_VALUE_VALUE = "'%s'";
public static final String AGG_COUNT = "COUNT(*)";
public static final String AGG_FIELD = "%s(%s)";
public static final String WHERE_BETWEEN = "'%s' AND '%s'";
public static final String BRACKETS = "(%s)";
}
backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java 0 → 100644
浏览文件 @ 19107a74
package io.dataease.provider.query.impala;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.domain.Datasource;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.constants.DeTypeConstants;
import io.dataease.controller.request.chart.ChartExtFilterRequest;
import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
import io.dataease.provider.QueryProvider;
import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.STGroup;
import org.stringtemplate.v4.STGroupFile;
import javax.annotation.Resource;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("impalaQuery")
public class ImpalaQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
@Override
public Integer transFieldType(String field) {
field = field.toLowerCase();
switch (field) {
case "varchar":
case "string":
case "char":
return 0;// 文本
case "timestamp":
case "date":
return 1;// 时间
case "tinyint":
case "smallint":
case "int":
case "bigint":
return 2;// 整型
case "decimal":
case "float":
case "double":
return 3;// 浮点
case "boolean":
return 4;// 布尔
default:
return 0;
}
}
@Override
public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 1000";
}
@Override
public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
DatasetTableField f = fields.get(i);
String originField;
if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(f.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
String fieldName = "";
// 处理横轴字段
if (f.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (f.getDeType() == 2 || f.getDeType() == 3) {
fieldName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originField) + "*1000";
} else {
fieldName = originField;
}
} else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) {
if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT);
} else if (f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_FLOAT_FORMAT);
} else if (f.getDeType() == DeTypeConstants.DE_TIME) {
fieldName = String.format(ImpalaConstants.STR_TO_DATE, originField, ImpalaConstants.DEFAULT_DATE_FORMAT);
} else {
fieldName = originField;
}
} else {
if (f.getDeType() == DeTypeConstants.DE_TIME) {
String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000";
fieldName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT);
} else if (f.getDeType() == 2) {
fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT);
} else {
fieldName = originField;
}
}
xFields.add(SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build());
}
}
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("previewSql");
st_sql.add("isGroup", isGroup);
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
return st_sql.render();
}
@Override
public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null, fieldCustomFilter);
}
@Override
public String createQueryTableWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + realSize;
}
@Override
public String createQueryTableWithLimit(String table, List<DatasetTableField> fields, Integer limit, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + limit;
}
@Override
public String createQuerySqlWithLimit(String sql, List<DatasetTableField> fields, Integer limit, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + limit;
}
@Override
public String createQuerySQLWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + realSize;
}
@Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.add(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
// 处理仪表板字段过滤
String extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (extWheres != null) wheres.add(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<String> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(ImpalaConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return sqlLimit(st.render(), view);
}
@Override
public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
// 处理视图中字段过滤
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
// 处理仪表板字段过滤
String extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (extWheres != null) wheres.add(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("previewSql");
st_sql.add("isGroup", false);
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("previewSql");
st.add("isGroup", false);
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(ImpalaConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return sqlLimit(st.render(), view);
}
@Override
public String getSQLAsTmpTableInfo(String sql, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
return getSQLTableInfo("(" + sqlFix(sql) + ")", xAxis, fieldCustomFilter, extFilterRequestList, null, view);
}
@Override
public String getSQLAsTmp(String sql, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, null, view);
}
@Override
public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds, ChartViewWithBLOBs view) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
List<ChartViewFieldDTO> xList = new ArrayList<>();
xList.addAll(xAxis);
xList.addAll(extStack);
if (CollectionUtils.isNotEmpty(xList)) {
for (int i = 0; i < xList.size(); i++) {
ChartViewFieldDTO x = xList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.add(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
// 处理仪表板字段过滤
String extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (extWheres != null) wheres.add(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<String> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(ImpalaConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return sqlLimit(st.render(), view);
}
@Override
public String getSQLAsTmpStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, ChartViewWithBLOBs view) {
return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extStack, null, view);
}
@Override
public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds, ChartViewWithBLOBs view) {
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(xAxis)) {
for (int i = 0; i < xAxis.size(); i++) {
ChartViewFieldDTO x = xAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias));
// 处理横轴排序
if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
xOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(x.getSort())
.build());
}
}
}
List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
List<ChartViewFieldDTO> yList = new ArrayList<>();
yList.addAll(yAxis);
yList.addAll(extBubble);
if (CollectionUtils.isNotEmpty(yList)) {
for (int i = 0; i < yList.size(); i++) {
ChartViewFieldDTO y = yList.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.add(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
// 处理仪表板字段过滤
String extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(xFields);
fields.addAll(yFields);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (extWheres != null) wheres.add(extWheres);
List<SQLObj> groups = new ArrayList<>();
groups.addAll(xFields);
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(xOrders);
orders.addAll(yOrders);
List<String> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(ImpalaConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return sqlLimit(st.render(), view);
}
@Override
public String getSQLAsTmpScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, ChartViewWithBLOBs view) {
return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extBubble, null, view);
}
@Override
public String searchTable(String table) {
return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'";
}
@Override
public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
// 字段汇总 排序等
SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> yFields = new ArrayList<>();
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
if (CollectionUtils.isNotEmpty(yAxis)) {
for (int i = 0; i < yAxis.size(); i++) {
ChartViewFieldDTO y = yAxis.get(i);
String originField;
if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
yFields.add(getYFields(y, originField, fieldAlias));
// 处理纵轴过滤
yWheres.add(getYWheres(y, originField, fieldAlias));
// 处理纵轴排序
if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
yOrders.add(SQLObj.builder()
.orderField(originField)
.orderAlias(fieldAlias)
.orderDirection(y.getSort())
.build());
}
}
}
// 处理视图中字段过滤
String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
// 处理仪表板字段过滤
String extWheres = transExtFilterList(tableObj, extFilterRequestList);
// 构建sql所有参数
List<SQLObj> fields = new ArrayList<>();
fields.addAll(yFields);
List<String> wheres = new ArrayList<>();
if (customWheres != null) wheres.add(customWheres);
if (extWheres != null) wheres.add(extWheres);
List<SQLObj> groups = new ArrayList<>();
// 外层再次套sql
List<SQLObj> orders = new ArrayList<>();
orders.addAll(yOrders);
List<String> aggWheres = new ArrayList<>();
aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
ST st_sql = stg.getInstanceOf("querySql");
if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
String sql = st_sql.render();
ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(ImpalaConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
return sqlLimit(st.render(), view);
}
@Override
public String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, fieldCustomFilter, extFilterRequestList, view);
}
@Override
public String wrapSql(String sql) {
sql = sql.trim();
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
return tmpSql;
}
@Override
public String createRawQuerySQL(String table, List<DatasetTableField> fields, Datasource ds) {
String[] array = fields.stream().map(f -> {
StringBuilder stringBuilder = new StringBuilder();
if (f.getDeExtractType() == 4) { // 处理 tinyint
stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName());
} else {
stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
}
return stringBuilder.toString();
}).toArray(String[]::new);
return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table);
}
@Override
public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null);
}
@Override
public String convertTableToSql(String tableName, Datasource ds) {
return createSQLPreview("SELECT * FROM " + String.format(ImpalaConstants.KEYWORD_TABLE, tableName), null);
}
public String transMysqlFilterTerm(String term) {
switch (term) {
case "eq":
return " = ";
case "not_eq":
return " <> ";
case "lt":
return " < ";
case "le":
return " <= ";
case "gt":
return " > ";
case "ge":
return " >= ";
case "in":
return " IN ";
case "not in":
return " NOT IN ";
case "like":
return " LIKE ";
case "not like":
return " NOT LIKE ";
case "null":
return " IS NULL ";
case "not_null":
return " IS NOT NULL ";
case "empty":
return " = ";
case "not_empty":
return " <> ";
case "between":
return " BETWEEN ";
default:
return "";
}
}
public String transCustomFilterList(SQLObj tableObj, List<ChartFieldCustomFilterDTO> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<String> res = new ArrayList<>();
for (ChartFieldCustomFilterDTO request : requestList) {
List<SQLObj> list = new ArrayList<>();
DatasetTableField field = request.getField();
if (ObjectUtils.isEmpty(field)) {
continue;
}
String whereName = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == DeTypeConstants.DE_TIME) {
if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
whereName = String.format(ImpalaConstants.STR_TO_DATE, originName, ImpalaConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
String cast = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000";
whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == DeTypeConstants.DE_TIME) {
whereName = originName;
}
} else if (field.getDeType() == 2 || field.getDeType() == 3) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_FLOAT_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originName) + "*1000";
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
whereName = originName;
}
} else {
whereName = originName;
}
if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) {
if (CollectionUtils.isNotEmpty(request.getEnumCheckField())) {
res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))");
}
} else {
List<ChartCustomFilterItemDTO> filter = request.getFilter();
for (ChartCustomFilterItemDTO filterItemDTO : filter) {
String value = filterItemDTO.getValue();
String whereTerm = transMysqlFilterTerm(filterItemDTO.getTerm());
String whereValue = "";
if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {
whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, value);
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
List<String> strList = new ArrayList<>();
list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
if (CollectionUtils.isNotEmpty(list)) {
res.add("(" + String.join(" " + getLogic(request.getLogic()) + " ", strList) + ")");
}
}
}
return CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null;
}
public String transExtFilterList(SQLObj tableObj, List<ChartExtFilterRequest> requestList) {
if (CollectionUtils.isEmpty(requestList)) {
return null;
}
List<SQLObj> list = new ArrayList<>();
for (ChartExtFilterRequest request : requestList) {
List<String> value = request.getValue();
DatasetTableField field = request.getDatasetTableField();
if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) {
continue;
}
String whereName = "";
String whereTerm = transMysqlFilterTerm(request.getOperator());
String whereValue = "";
String originName;
if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
// 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
}
if (field.getDeType() == DeTypeConstants.DE_TIME) {
if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
whereName = String.format(ImpalaConstants.STR_TO_DATE, originName, ImpalaConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
String cast = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000";
whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT);
}
if (field.getDeExtractType() == DeTypeConstants.DE_TIME) {
whereName = originName;
}
} else if (field.getDeType() == 2 || field.getDeType() == 3) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_FLOAT_FORMAT);
}
if (field.getDeExtractType() == 1) {
whereName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originName) + "*1000";
}
if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
whereName = originName;
}
} else {
whereName = originName;
}
if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
whereValue = "'%" + value.get(0) + "%'";
} else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) {
if (request.getDatasetTableField().getDeType() == DeTypeConstants.DE_TIME) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
whereValue = String.format(ImpalaConstants.WHERE_BETWEEN, startTime, endTime);
} else {
whereValue = String.format(ImpalaConstants.WHERE_BETWEEN, value.get(0), value.get(1));
}
} else {
whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, value.get(0));
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereTermAndValue(whereTerm + whereValue)
.build());
}
List<String> strList = new ArrayList<>();
list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null;
}
private String sqlFix(String sql) {
if (sql.lastIndexOf(";") == (sql.length() - 1)) {
sql = sql.substring(0, sql.length() - 1);
}
return sql;
}
private String transDateFormat(String dateStyle, String datePattern) {
String split = "-";
if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {
split = "-";
} else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) {
split = "/";
} else {
split = "-";
}
if (StringUtils.isEmpty(dateStyle)) {
return "yyyy-MM-dd HH:mm:ss";
}
switch (dateStyle) {
case "y":
return "yyyy";
case "y_M":
return "yyyy" + split + "MM";
case "y_M_d":
return "yyyy" + split + "MM" + split + "dd";
case "H_m_s":
return "HH:mm:ss";
case "y_M_d_H_m":
return "yyyy" + split + "MM" + split + "dd" + " HH:mm";
case "y_M_d_H_m_s":
return "yyyy" + split + "MM" + split + "dd" + " HH:mm:ss";
default:
return "yyyy-MM-dd HH:mm:ss";
}
}
private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {
String fieldName = "";
if (x.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (x.getDeType() == 2 || x.getDeType() == 3) {
fieldName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originField) + "*1000";
} else if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
fieldName = String.format(ImpalaConstants.DATE_FORMAT, originField, format);
} else {
fieldName = originField;
}
} else {
if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
if (x.getDeExtractType() == DeTypeConstants.DE_STRING) {
fieldName = String.format(ImpalaConstants.DATE_FORMAT, originField, format);
} else {
String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000";
String from_unixtime = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT);
fieldName = String.format(ImpalaConstants.DATE_FORMAT, from_unixtime, format);
}
} else {
fieldName = originField;
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private List<SQLObj> getXWheres(ChartViewFieldDTO x, String originField, String fieldAlias) {
List<SQLObj> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) {
x.getFilter().forEach(f -> {
String whereName = "";
String whereTerm = transMysqlFilterTerm(f.getTerm());
String whereValue = "";
if (x.getDeType() == DeTypeConstants.DE_TIME && x.getDeExtractType() != 1) {
String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000";
whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT);
} else {
whereName = originField;
}
if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
whereValue = "'%" + f.getValue() + "%'";
} else {
whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, f.getValue());
}
list.add(SQLObj.builder()
.whereField(whereName)
.whereAlias(fieldAlias)
.whereTermAndValue(whereTerm + whereValue)
.build());
});
}
return list;
}
private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
String fieldName = "";
if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) {
fieldName = ImpalaConstants.AGG_COUNT;
} else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
fieldName = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), originField);
} else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
String cast = String.format(ImpalaConstants.CAST, originField, y.getDeType() == 2 ? ImpalaConstants.DEFAULT_INT_FORMAT : ImpalaConstants.DEFAULT_FLOAT_FORMAT);
String agg = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), cast);
fieldName = String.format(ImpalaConstants.CAST, agg, ImpalaConstants.DEFAULT_FLOAT_FORMAT);
} else {
String cast = String.format(ImpalaConstants.CAST, originField, y.getDeType() == 2 ? ImpalaConstants.DEFAULT_INT_FORMAT : ImpalaConstants.DEFAULT_FLOAT_FORMAT);
fieldName = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), cast);
}
}
return SQLObj.builder()
.fieldName(fieldName)
.fieldAlias(fieldAlias)
.build();
}
private String getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) {
List<SQLObj> list = new ArrayList<>();
if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) {
y.getFilter().forEach(f -> {
String whereTerm = transMysqlFilterTerm(f.getTerm());
String whereValue = "";
// 原始类型不是时间，在de中被转成时间的字段做处理
if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
whereValue = "";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
whereValue = "'%" + f.getValue() + "%'";
} else {
whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, f.getValue());
}
list.add(SQLObj.builder()
.whereField(fieldAlias)
.whereAlias(fieldAlias)
.whereTermAndValue(whereTerm + whereValue)
.build());
});
}
List<String> strList = new ArrayList<>();
list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + getLogic(y.getLogic()) + " ", strList) + ")" : null;
}
private String calcFieldRegex(String originField, SQLObj tableObj) {
originField = originField.replaceAll("[\\t\\n\\r]]", "");
// 正则提取[xxx]
String regex = "\\[(.*?)]";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(originField);
Set<String> ids = new HashSet<>();
while (matcher.find()) {
String id = matcher.group(1);
ids.add(id);
}
if (CollectionUtils.isEmpty(ids)) {
return originField;
}
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids));
List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
for (DatasetTableField ele : calcFields) {
originField = originField.replaceAll("\\[" + ele.getId() + "]",
String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
}
return originField;
}
private String sqlLimit(String sql, ChartViewWithBLOBs view) {
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
return sql + " LIMIT " + view.getResultCount();
} else {
return sql;
}
}
}
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
浏览文件 @ 19107a74
......@@ -292,25 +292,6 @@ public class ChartViewService {
}
// private void checkPermissions(List<? extends ChartViewFieldBaseDTO> chartViewFieldDTOS, List<DatasetTableField> fields, List<String> desensitizationList, Boolean alowDesensitization) throws Exception {
// String filedName = "";
// for (ChartViewFieldBaseDTO chartViewFieldDTO : chartViewFieldDTOS) {
// if (alowDesensitization) {
// if (!fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) {
// filedName = filedName + chartViewFieldDTO.getName() + " ,";
// }
// } else {
// if (desensitizationList.contains(chartViewFieldDTO.getDataeaseName()) || !fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) {
// filedName = filedName + chartViewFieldDTO.getName() + " ,";
// }
// }
// }
// filedName = filedName.endsWith(",") ? filedName.substring(0, filedName.length() - 1) : filedName;
// if (StringUtils.isNotEmpty(filedName)) {
// throw new Exception("以下字段没有权限: " + filedName);
// }
// }
public ChartViewDTO calcData(ChartViewDTO view, ChartExtRequest requestList, boolean cache) throws Exception {
if (ObjectUtils.isEmpty(view)) {
throw new RuntimeException(Translator.get("i18n_chart_delete"));
......
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
浏览文件 @ 19107a74
......@@ -101,6 +101,7 @@ public class DataSetTableService {
@Resource
private EngineService engineService;
private static boolean isUpdatingDatasetTableStatus = false;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
......@@ -2182,15 +2183,30 @@ public class DataSetTableService {
private UtilMapper utilMapper;
public void updateDatasetTableStatus() {
if(this.isUpdatingDatasetTableStatus){
return;
}else {
this.isUpdatingDatasetTableStatus = true;
}
try {
doUpdate();
}catch (Exception e){}
finally {
this.isUpdatingDatasetTableStatus = false;
}
}
private void doUpdate(){
List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null);
List<String> activeQrtzInstances = qrtzSchedulerStates.stream()
.filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime()
+ qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp())
.map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name());
datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(
datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
......@@ -2202,6 +2218,7 @@ public class DataSetTableService {
return;
}
//DatasetTable
DatasetTable record = new DatasetTable();
record.setSyncStatus(JobStatus.Error.name());
example.clear();
......@@ -2209,6 +2226,14 @@ public class DataSetTableService {
.andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
datasetTableMapper.updateByExampleSelective(record, example);
//Task
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();
criteria.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())).andStatusEqualTo(JobStatus.Underway.name());
List<DatasetTableTask> datasetTableTasks = dataSetTableTaskService.list(datasetTableTaskExample);
dataSetTableTaskService.updateTaskStatus(datasetTableTasks, JobStatus.Error);
//TaskLog
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo("Job stopped due to system error.");
......@@ -2216,19 +2241,14 @@ public class DataSetTableService {
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample();
datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name())
.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
List<String> taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream()
.map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList());
datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample);
dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error);
for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) {
extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId());
extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId());
extractDataService.deleteFile("incremental_delete", jobStoppeddDatasetTable.getId());
}
}
/*
* 判断数组中是否有重复的值
*/
......
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
浏览文件 @ 19107a74
......@@ -161,13 +161,13 @@ public class DataSetTableTaskService {
return datasetTableTaskMapper.selectByPrimaryKey(id);
}
public void updateTaskStatus(List<String> taskIds, JobStatus lastExecStatus) {
if (CollectionUtils.isEmpty(taskIds)) {
return;
public List<DatasetTableTask> list(DatasetTableTaskExample example) {
return datasetTableTaskMapper.selectByExample(example);
}
DatasetTableTaskExample example = new DatasetTableTaskExample();
example.createCriteria().andIdIn(taskIds);
List<DatasetTableTask> datasetTableTasks = datasetTableTaskMapper.selectByExample(example);
public void updateTaskStatus(List<DatasetTableTask> datasetTableTasks, JobStatus lastExecStatus) {
for (DatasetTableTask tableTask : datasetTableTasks) {
updateTaskStatus(tableTask, lastExecStatus);
}
......@@ -202,7 +202,7 @@ public class DataSetTableTaskService {
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
} else {
datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
// datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
datasetTableTask.setLastExecStatus(lastExecStatus.name());
if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) {
BaseGridRequest request = new BaseGridRequest();
......
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @ 19107a74
......@@ -140,7 +140,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
return !CollectionUtils.isNotEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
} else {
datasetTableTask.setLastExecTime(startTime);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
......@@ -159,7 +159,7 @@ public class ExtractDataService {
return;
}
UpdateType updateType = UpdateType.valueOf(type);
DatasetTableTaskLog datasetTableTaskLog;
if (datasetTableFields == null) {
datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
}
......@@ -174,10 +174,10 @@ public class ExtractDataService {
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
Long execTime = System.currentTimeMillis();
......@@ -222,7 +222,7 @@ public class ExtractDataService {
toDelete.forEach(datasetTableField -> dataSetTableFieldsService.delete(datasetTableField.getId()));
}
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
......@@ -233,7 +233,6 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
Long execTime = System.currentTimeMillis();
if (!engineService.isSimpleMode()) {
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
......@@ -245,7 +244,7 @@ public class ExtractDataService {
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
} finally {
deleteFile("incremental_add", datasetTableId);
......@@ -312,7 +311,7 @@ public class ExtractDataService {
msg = true;
lastExecStatus = JobStatus.Completed;
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
saveErrorLog(datasetTableTaskLog, e);
msg = false;
lastExecStatus = JobStatus.Error;
execTime = null;
......@@ -372,7 +371,7 @@ public class ExtractDataService {
msg = true;
lastExecStatus = JobStatus.Completed;
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
saveErrorLog(datasetTableTaskLog, e);
msg = false;
lastExecStatus = JobStatus.Error;
execTime = null;
......@@ -592,26 +591,14 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void saveErrorLog(String datasetTableId, String taskId, Exception e) {
LogUtil.error("Extract data error: " + datasetTableId, e);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
if (StringUtils.isNotEmpty(taskId)) {
datasetTableTaskLog.setTaskId(taskId);
}
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog = datasetTableTaskLogs.get(0);
private void saveErrorLog(DatasetTableTaskLog datasetTableTaskLog, Exception e) {
LogUtil.error("Extract data error: " + datasetTableTaskLog.getTaskId(), e);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(e.getMessage());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
}
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
......@@ -671,14 +658,9 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Cron.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
return datasetTableTaskLog;
} else {
return datasetTableTaskLogs.get(0);
}
}
private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId, Long startTime) {
......@@ -687,7 +669,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 5; i++) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
return datasetTableTaskLogs.get(0);
......
frontend/src/api/dataset/dataset.js
浏览文件 @ 19107a74
......@@ -213,5 +213,5 @@ export function checkCustomDs() {
loading: true
})
}
export const disabledSyncDs= ['es', 'ck', 'mongo', 'redshift', 'hive', 'impala']
export default { loadTable, getScene, addGroup, delGroup, addTable, delTable, groupTree, checkCustomDs }
frontend/src/views/dataset/add/AddDB.vue
浏览文件 @ 19107a74
......@@ -69,7 +69,7 @@
</template>
<script>
import {listDatasource, post, isKettleRunning} from '@/api/dataset/dataset'
import {listDatasource, post, isKettleRunning, disabledSyncDs} from '@/api/dataset/dataset'
import {engineMode} from "@/api/system/engine";
export default {
......@@ -94,7 +94,7 @@ export default {
selectedDatasource: {},
engineMode: 'local',
disabledSync: true,
disabledSyncDs: ['es', 'ck', 'mongo', 'redshift', 'hive']
disabledSyncDs: disabledSyncDs
}
},
watch: {
......
frontend/src/views/dataset/add/AddSQL.vue
浏览文件 @ 19107a74
......@@ -99,7 +99,7 @@
</template>
<script>
import {post, listDatasource, isKettleRunning} from '@/api/dataset/dataset'
import {post, listDatasource, isKettleRunning, disabledSyncDs} from '@/api/dataset/dataset'
import {codemirror} from 'vue-codemirror'
import {getTable} from '@/api/dataset/dataset'
// 核心样式
......@@ -160,7 +160,8 @@ export default {
kettleRunning: false,
selectedDatasource: {},
engineMode: 'local',
disabledSync: true
disabledSync: true,
disabledSyncDs: disabledSyncDs
}
},
computed: {
......
frontend/src/views/system/datasource/form.vue
浏览文件 @ 19107a74
......@@ -400,6 +400,7 @@ export default {
allTypes: [
{name: 'mysql', label: 'MySQL', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'},
{name: 'hive', label: 'Apache Hive', type: 'jdbc', extraParams: ''},
{name: 'impala', label: 'Apache Impala', type: 'jdbc', extraParams: 'auth=noSasl'},
{name: 'oracle', label: 'Oracle', type: 'jdbc'},
{name: 'sqlServer', label: 'SQL Server', type: 'jdbc', extraParams: ''},
{name: 'pg', label: 'PostgreSQL', type: 'jdbc', extraParams: ''},
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论