提交 558dfa7d authored 作者: wangjiahao's avatar wangjiahao

Merge remote-tracking branch 'origin/dev' into dev

# Conflicts: # backend/src/main/java/io/dataease/service/chart/ChartViewService.java
...@@ -100,6 +100,8 @@ public class ExcelXlsxReader extends DefaultHandler { ...@@ -100,6 +100,8 @@ public class ExcelXlsxReader extends DefaultHandler {
*/ */
private String formatString; private String formatString;
//定义前一个元素和当前元素的位置,用来计算其中空的单元格数量,如A6和A8等 //定义前一个元素和当前元素的位置,用来计算其中空的单元格数量,如A6和A8等
private String preRef = null, ref = null; private String preRef = null, ref = null;
......
...@@ -172,7 +172,7 @@ public class DataSetTableController { ...@@ -172,7 +172,7 @@ public class DataSetTableController {
@ApiOperation("检测doris") @ApiOperation("检测doris")
@PostMapping("checkDorisTableIsExists/{id}") @PostMapping("checkDorisTableIsExists/{id}")
public Boolean checkDorisTableIsExists(@PathVariable String id) throws Exception { public Boolean checkDorisTableIsExists(@PathVariable String id) throws Exception {
return dataSetTableService.checkDorisTableIsExists(id); return dataSetTableService.checkEngineTableIsExists(id);
} }
@ApiOperation("搜索") @ApiOperation("搜索")
......
...@@ -39,9 +39,9 @@ public class DDLProviderImpl extends DDLProvider { ...@@ -39,9 +39,9 @@ public class DDLProviderImpl extends DDLProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) { for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings)))) values.append("('").append(Md5Utils.md5(String.join(",", Arrays.asList(strings))))
.append("," ).append(String.join(",", Arrays.asList(strings))) .append("','" ).append(String.join("','", Arrays.asList(strings)))
.append("),"); .append("'),");
} }
return insertSql + values.substring(0, values.length() - 1); return insertSql + values.substring(0, values.length() - 1);
} }
......
...@@ -395,6 +395,7 @@ public class JdbcProvider extends DatasourceProvider { ...@@ -395,6 +395,7 @@ public class JdbcProvider extends DatasourceProvider {
dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId()); dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId());
if (dataSource != null) { if (dataSource != null) {
dataSource.close(); dataSource.close();
jdbcConnection.remove(datasourceRequest.getDatasource().getId());
} }
break; break;
default: default:
...@@ -426,6 +427,7 @@ public class JdbcProvider extends DatasourceProvider { ...@@ -426,6 +427,7 @@ public class JdbcProvider extends DatasourceProvider {
case mysql: case mysql:
case mariadb: case mariadb:
case engine_doris: case engine_doris:
case engine_mysql:
case ds_doris: case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
username = mysqlConfiguration.getUsername(); username = mysqlConfiguration.getUsername();
...@@ -529,6 +531,7 @@ public class JdbcProvider extends DatasourceProvider { ...@@ -529,6 +531,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) { switch (datasourceType) {
case mysql: case mysql:
case mariadb: case mariadb:
case engine_mysql:
case engine_doris: case engine_doris:
case ds_doris: case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
...@@ -604,6 +607,7 @@ public class JdbcProvider extends DatasourceProvider { ...@@ -604,6 +607,7 @@ public class JdbcProvider extends DatasourceProvider {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) { switch (datasourceType) {
case mysql: case mysql:
case engine_mysql:
case mariadb: case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase()); return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
...@@ -657,6 +661,7 @@ public class JdbcProvider extends DatasourceProvider { ...@@ -657,6 +661,7 @@ public class JdbcProvider extends DatasourceProvider {
case mysql: case mysql:
case mariadb: case mariadb:
case engine_doris: case engine_doris:
case engine_mysql:
case ds_doris: case ds_doris:
case ck: case ck:
return null; return null;
......
...@@ -25,7 +25,7 @@ public class MysqlConstants extends SQLConstants { ...@@ -25,7 +25,7 @@ public class MysqlConstants extends SQLConstants {
public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S";
public static final String DEFAULT_INT_FORMAT = "BIGINT"; public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)";
public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
......
package io.dataease.provider.engine.mysql; package io.dataease.provider.engine.mysql;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.commons.utils.TableUtils; import io.dataease.commons.utils.TableUtils;
import io.dataease.provider.DDLProviderImpl; import io.dataease.provider.DDLProviderImpl;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.List;
/** /**
* @Author gin * @Author gin
* @Date 2021/5/17 4:27 下午 * @Date 2021/5/17 4:27 下午
*/ */
@Service("mysqlEngineDDL") @Service("mysqlEngineDDL")
public class MysqlDDLProvider extends DDLProviderImpl { public class MysqlDDLProvider extends DDLProviderImpl {
private static final String creatTableSql =
"CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields;" ;
@Override @Override
public String createView(String name, String viewSQL) { public String createView(String name, String viewSQL) {
return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
...@@ -27,9 +36,60 @@ public class MysqlDDLProvider extends DDLProviderImpl { ...@@ -27,9 +36,60 @@ public class MysqlDDLProvider extends DDLProviderImpl {
@Override @Override
public String replaceTable(String name){ public String replaceTable(String name){
String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; " String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE"
.replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name)); .replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name); String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name);
return replaceTableSql + ";" + dropTableSql; return replaceTableSql + ";" + dropTableSql;
} }
@Override
public String createTableSql(String tableName, List<DatasetTableField> datasetTableFields) {
String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
}
private String createDorisTableColumnSql(final List<DatasetTableField> datasetTableFields) {
StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
Integer size = datasetTableField.getSize() * 4;
switch (datasetTableField.getDeExtractType()) {
case 0:
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
case 1:
size = size < 50? 50 : size;
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
case 2:
Column_Fields.append("varchar(100)").append(",`");
break;
case 3:
Column_Fields.append("varchar(100)").append(",`");
break;
case 4:
Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
break;
default:
if (size < 65533) {
Column_Fields.append("varchar(length)".replace("length", String.valueOf(datasetTableField.getSize()))).append(",`");
}else {
Column_Fields.append("longtext").append(",`");
}
break;
}
}
Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 1)).append("PRIMARY KEY(dataease_uuid)");
Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
return Column_Fields.toString();
}
} }
...@@ -551,7 +551,7 @@ public class DataSetTableService { ...@@ -551,7 +551,7 @@ public class DataSetTableService {
} }
} else { } else {
// check doris table // check doris table
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync")); throw new RuntimeException(Translator.get("i18n_data_not_sync"));
} }
Datasource ds = engineService.getDeEngine(); Datasource ds = engineService.getDeEngine();
...@@ -620,7 +620,7 @@ public class DataSetTableService { ...@@ -620,7 +620,7 @@ public class DataSetTableService {
} }
} else { } else {
// check doris table // check doris table
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync")); throw new RuntimeException(Translator.get("i18n_data_not_sync"));
} }
Datasource ds = engineService.getDeEngine(); Datasource ds = engineService.getDeEngine();
...@@ -648,7 +648,7 @@ public class DataSetTableService { ...@@ -648,7 +648,7 @@ public class DataSetTableService {
} }
} }
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { if (!checkEngineTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync")); throw new RuntimeException(Translator.get("i18n_data_not_sync"));
} }
...@@ -2167,12 +2167,12 @@ public class DataSetTableService { ...@@ -2167,12 +2167,12 @@ public class DataSetTableService {
return filePath; return filePath;
} }
public Boolean checkDorisTableIsExists(String id) throws Exception { public Boolean checkEngineTableIsExists(String id) throws Exception {
Datasource dorisDatasource = engineService.getDeEngine(); Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource); datasourceRequest.setDatasource(engine);
QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(engine.getType());
datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id))); datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id)));
List<String[]> data = jdbcProvider.getData(datasourceRequest); List<String[]> data = jdbcProvider.getData(datasourceRequest);
return CollectionUtils.isNotEmpty(data); return CollectionUtils.isNotEmpty(data);
......
...@@ -445,6 +445,7 @@ public class ExtractDataService { ...@@ -445,6 +445,7 @@ public class ExtractDataService {
List<String[]> dataList = result.get("dataList"); List<String[]> dataList = result.get("dataList");
if (engineService.isSimpleMode()) { if (engineService.isSimpleMode()) {
extractDataForSimpleMode(extractType, datasetTable.getId(), dataList); extractDataForSimpleMode(extractType, datasetTable.getId(), dataList);
return;
} }
Datasource engine = engineService.getDeEngine(); Datasource engine = engineService.getDeEngine();
...@@ -712,7 +713,9 @@ public class ExtractDataService { ...@@ -712,7 +713,9 @@ public class ExtractDataService {
for (ExcelSheetData sheet : excelXlsxReader.totalSheets) { for (ExcelSheetData sheet : excelXlsxReader.totalSheets) {
if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) { if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) {
for (List<String> dataItem : sheet.getData()) { for (List<String> dataItem : sheet.getData()) {
data.add(dataItem.toArray(new String[dataItem.size()])); if(dataItem.size()>0){
data.add(dataItem.toArray(new String[dataItem.size()]));
}
} }
} }
} }
......
...@@ -323,7 +323,6 @@ public class DatasourceService { ...@@ -323,7 +323,6 @@ public class DatasourceService {
List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample()); List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample());
datasources.forEach(datasource -> { datasources.forEach(datasource -> {
commonThreadPool.addTask(()->{ commonThreadPool.addTask(()->{
System.out.println(System.currentTimeMillis());
try { try {
handleConnectionPool(datasource, "add"); handleConnectionPool(datasource, "add");
} catch (Exception e) { } catch (Exception e) {
......
...@@ -31,7 +31,7 @@ public class EngineService { ...@@ -31,7 +31,7 @@ public class EngineService {
private DeEngineMapper deEngineMapper; private DeEngineMapper deEngineMapper;
@Resource @Resource
private DatasourceService datasource; private DatasourceService datasource;
static private Datasource ds = new Datasource(); static private Datasource ds = null;
public Boolean isLocalMode(){ public Boolean isLocalMode(){
...@@ -59,6 +59,9 @@ public class EngineService { ...@@ -59,6 +59,9 @@ public class EngineService {
} }
public ResultHolder validate(DatasourceDTO datasource) throws Exception { public ResultHolder validate(DatasourceDTO datasource) throws Exception {
if(StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())){
throw new Exception("未完整设置数据引擎");
}
try { try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
...@@ -78,13 +81,22 @@ public class EngineService { ...@@ -78,13 +81,22 @@ public class EngineService {
deEngineMapper.updateByPrimaryKeyWithBLOBs(engine); deEngineMapper.updateByPrimaryKeyWithBLOBs(engine);
} }
datasource.handleConnectionPool(this.ds, "delete"); datasource.handleConnectionPool(this.ds, "delete");
BeanUtils.copyBean(this.ds, engine); setDs(engine);
datasource.handleConnectionPool(this.ds, "add"); datasource.handleConnectionPool(this.ds, "add");
return ResultHolder.success(engine); return ResultHolder.success(engine);
} }
private void setDs(DeEngine engine){
if(this.ds == null){
this.ds = new Datasource();
BeanUtils.copyBean(this.ds, engine);
}else {
BeanUtils.copyBean(this.ds, engine);
}
}
public Datasource getDeEngine() throws Exception{ public Datasource getDeEngine() throws Exception{
if (this.ds != null || StringUtils.isNotEmpty(ds.getType())) { if (this.ds != null) {
return this.ds; return this.ds;
} }
if(isLocalMode()){ if(isLocalMode()){
...@@ -97,21 +109,23 @@ public class EngineService { ...@@ -97,21 +109,23 @@ public class EngineService {
jsonObject.put("port", env.getProperty("doris.port", "9030")); jsonObject.put("port", env.getProperty("doris.port", "9030"));
jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030")); jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
Datasource datasource = new Datasource(); DeEngine engine = new DeEngine();
datasource.setId("doris"); engine.setId("doris");
datasource.setName("doris"); engine.setName("doris");
datasource.setDesc("doris"); engine.setDesc("doris");
datasource.setType("engine_doris"); engine.setType("engine_doris");
datasource.setConfiguration(jsonObject.toJSONString()); engine.setConfiguration(jsonObject.toJSONString());
this.ds = datasource; setDs(engine);
} }else {
if(isSimpleMode()){
List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample()); List<DeEngine> deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample());
if(CollectionUtils.isEmpty(deEngines)){ if(CollectionUtils.isEmpty(deEngines)){
throw new Exception("未设置数据引擎"); throw new Exception("未设置数据引擎");
} }
BeanUtils.copyBean(this.ds, deEngines.get(0)); setDs(deEngines.get(0));
} }
// if(isSimpleMode()){
//
// }
//TODO cluster mode //TODO cluster mode
return this.ds; return this.ds;
......
...@@ -2,13 +2,13 @@ ...@@ -2,13 +2,13 @@
<div ref="tableContainer" :style="bg_class" style="padding: 8px;width: 100%;height: 100%;overflow: hidden;"> <div ref="tableContainer" :style="bg_class" style="padding: 8px;width: 100%;height: 100%;overflow: hidden;">
<p v-show="title_show" ref="title" :style="title_class">{{ chart.title }}</p> <p v-show="title_show" ref="title" :style="title_class">{{ chart.title }}</p>
<div <div
v-if="chart.data && chart.data.datas && chart.data.datas.length > 0" v-if="chart.data && chart.data.series && chart.data.series.length > 0"
id="label-content" id="label-content"
:style="content_class" :style="content_class"
> >
<span :style="label_class"> <span :style="label_class">
<p :style="label_content_class"> <p v-if="chart.data.series[0].data && chart.data.series[0].data.length > 0" :style="label_content_class">
{{ chart.data.datas[0].name }} {{ chart.data.series[0].data[0] }}
</p> </p>
</span> </span>
<!-- 字段名暂时隐藏--> <!-- 字段名暂时隐藏-->
......
...@@ -101,7 +101,7 @@ export default { ...@@ -101,7 +101,7 @@ export default {
return { return {
form: form:
{ {
type: 'mysql', type: 'engine_mysql',
configuration: { configuration: {
host: '', host: '',
dataBase: '', dataBase: '',
...@@ -129,30 +129,31 @@ export default { ...@@ -129,30 +129,31 @@ export default {
disabledSave: false, disabledSave: false,
loading: false, loading: false,
rules: { rules: {
host: [ 'configuration.host': [
{ {
required: true, required: true,
message: this.$t('system_parameter_setting.host'), message: this.$t('datasource.please_input_host'),
trigger: ['change', 'blur'] trigger: ['change', 'blur']
} }
], ],
port: [ 'configuration.port': [
{ {
required: true, required: true,
message: this.$t('system_parameter_setting.port'), message: this.$t('datasource.please_input_port'),
trigger: ['change', 'blur'] trigger: ['change', 'blur']
} }
], ],
account: [ 'configuration.dataBase': [
{ {
required: true, required: true,
message: this.$t('system_parameter_setting.account'), message: this.$t('datasource.please_input_data_base'),
trigger: ['change', 'blur'] trigger: ['change', 'blur']
}] }
]
}, },
allTypes: [ allTypes: [
{ {
name: 'mysql', name: 'engine_mysql',
label: 'MySQL', label: 'MySQL',
type: 'jdbc', type: 'jdbc',
extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true' extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论