Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
D
dataease
项目
项目
详情
活动
周期分析
仓库
仓库
文件
提交
分支
标签
贡献者
图表
比较
统计图
议题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程
统计图
Wiki
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
zhu
dataease
Commits
f9bb4b31
提交
f9bb4b31
authored
5月 24, 2021
作者:
taojinlong
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix: 抽取excel数据格式问题;
上级
f64122b7
隐藏空白字符变更
内嵌
并排
正在显示
10 个修改的文件
包含
84 行增加
和
57 行删除
+84
-57
DatasourceService.java
...ava/io/dataease/datasource/service/DatasourceService.java
+26
-1
AppStartReadHBaseListener.java
.../java/io/dataease/listener/AppStartReadHBaseListener.java
+0
-47
DataSourceInitStartListener.java
...ava/io/dataease/listener/DataSourceInitStartListener.java
+25
-0
DataSetGroupService.java
...java/io/dataease/service/dataset/DataSetGroupService.java
+2
-0
DataSetTableService.java
...java/io/dataease/service/dataset/DataSetTableService.java
+2
-0
ExtractDataService.java
.../java/io/dataease/service/dataset/ExtractDataService.java
+25
-8
Toolbar.vue
frontend/src/components/canvas/components/Toolbar.vue
+1
-1
en.js
frontend/src/lang/en.js
+1
-0
tw.js
frontend/src/lang/tw.js
+1
-0
zh.js
frontend/src/lang/zh.js
+1
-0
没有找到文件。
backend/src/main/java/io/dataease/datasource/service/DatasourceService.java
浏览文件 @
f9bb4b31
...
...
@@ -7,6 +7,8 @@ import io.dataease.base.mapper.ext.ExtDataSourceMapper;
import
io.dataease.base.mapper.ext.query.GridExample
;
import
io.dataease.commons.exception.DEException
;
import
io.dataease.commons.utils.AuthUtils
;
import
io.dataease.commons.utils.CommonThreadPool
;
import
io.dataease.commons.utils.LogUtil
;
import
io.dataease.controller.request.DatasourceUnionRequest
;
import
io.dataease.controller.sys.base.BaseGridRequest
;
import
io.dataease.controller.sys.base.ConditionEntity
;
...
...
@@ -40,6 +42,8 @@ public class DatasourceService {
private
DatasetTableMapper
datasetTableMapper
;
@Resource
private
DataSetGroupService
dataSetGroupService
;
@Resource
private
CommonThreadPool
commonThreadPool
;
public
Datasource
addDatasource
(
Datasource
datasource
)
{
DatasourceExample
example
=
new
DatasourceExample
();
...
...
@@ -114,7 +118,6 @@ public class DatasourceService {
DataTableInfoDTO
dataTableInfoDTO
=
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
);
if
(
StringUtils
.
equals
(
name
,
dataTableInfoDTO
.
getTable
()))
{
dbTableDTO
.
setEnableCheck
(
false
);
List
<
DatasetGroup
>
parents
=
dataSetGroupService
.
getParents
(
datasetTable
.
getSceneId
());
StringBuilder
stringBuilder
=
new
StringBuilder
();
parents
.
forEach
(
ele
->
stringBuilder
.
append
(
ele
.
getName
()).
append
(
"/"
));
...
...
@@ -131,4 +134,26 @@ public class DatasourceService {
public
Datasource
get
(
String
id
)
{
return
datasourceMapper
.
selectByPrimaryKey
(
id
);
}
public
void
initAllDataSourceConnectionPool
(){
List
<
Datasource
>
datasources
=
datasourceMapper
.
selectByExampleWithBLOBs
(
new
DatasourceExample
());
datasources
.
forEach
(
datasource
->
{
try
{
commonThreadPool
.
addTask
(()
->{
try
{
System
.
out
.
println
(
new
Gson
().
toJson
(
datasource
));
DatasourceProvider
datasourceProvider
=
ProviderFactory
.
getProvider
(
datasource
.
getType
());
DatasourceRequest
datasourceRequest
=
new
DatasourceRequest
();
datasourceRequest
.
setDatasource
(
datasource
);
datasourceProvider
.
initDataSource
(
datasourceRequest
);
LogUtil
.
error
(
"Succsss to init datasource connection pool: "
+
datasource
.
getName
());
}
catch
(
Exception
e
){
LogUtil
.
error
(
"Failed to init datasource connection pool: "
+
datasource
.
getName
(),
e
);
}
});
}
catch
(
Exception
e
){
e
.
printStackTrace
();
}
});
}
}
backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java
deleted
100644 → 0
浏览文件 @
f64122b7
//package io.dataease.listener;
//
//import io.dataease.base.mapper.DatasetTableMapper;
//import io.dataease.commons.utils.CommonThreadPool;
//import io.dataease.service.dataset.DataSetTableFieldsService;
//import org.springframework.boot.context.event.ApplicationReadyEvent;
//import org.springframework.context.ApplicationListener;
//import org.springframework.core.annotation.Order;
//import org.springframework.core.env.Environment;
//import org.springframework.stereotype.Component;
//
//import javax.annotation.Resource;
//
//@Component
//@Order(value = 2)
//public class AppStartReadHBaseListener implements ApplicationListener<ApplicationReadyEvent> {
// @Resource
// private CommonThreadPool commonThreadPool;
//// @Resource
//// private SparkCalc sparkCalc;
// @Resource
// private Environment env; // 保存了配置文件的信息
//
// @Resource
// private DatasetTableMapper datasetTableMapper;
// @Resource
// private DataSetTableFieldsService dataSetTableFieldsService;
//
// @Override
// public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
//// System.out.println("================= Read HBase start =================");
//// // 项目启动,从数据集中找到定时抽取的表,从HBase中读取放入缓存
//// DatasetTableExample datasetTableExample = new DatasetTableExample();
//// datasetTableExample.createCriteria().andModeEqualTo(1);
//// List<DatasetTable> datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample);
//// for (DatasetTable table : datasetTables) {
////// commonThreadPool.addTask(() -> {
//// try {
//// List<DatasetTableField> fields = dataSetTableFieldsService.getFieldsByTableId(table.getId());
//// sparkCalc.getHBaseDataAndCache(table.getId(), fields);
//// } catch (Exception e) {
//// e.printStackTrace();
//// }
////// });
//// }
// }
//}
backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
0 → 100644
浏览文件 @
f9bb4b31
package
io
.
dataease
.
listener
;
import
io.dataease.base.domain.DatasetTableTask
;
import
io.dataease.datasource.service.DatasourceService
;
import
io.dataease.service.ScheduleService
;
import
io.dataease.service.dataset.DataSetTableTaskService
;
import
org.springframework.boot.context.event.ApplicationReadyEvent
;
import
org.springframework.context.ApplicationListener
;
import
org.springframework.core.annotation.Order
;
import
org.springframework.stereotype.Component
;
import
javax.annotation.Resource
;
import
java.util.List
;
@Component
@Order
(
value
=
1
)
public
class
DataSourceInitStartListener
implements
ApplicationListener
<
ApplicationReadyEvent
>
{
@Resource
private
DatasourceService
datasourceService
;
@Override
public
void
onApplicationEvent
(
ApplicationReadyEvent
applicationReadyEvent
)
{
datasourceService
.
initAllDataSourceConnectionPool
();
}
}
backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java
浏览文件 @
f9bb4b31
...
...
@@ -14,6 +14,7 @@ import io.dataease.dto.dataset.DataSetTableDTO;
import
io.dataease.i18n.Translator
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.springframework.context.annotation.Lazy
;
import
org.springframework.stereotype.Service
;
import
javax.annotation.Resource
;
...
...
@@ -32,6 +33,7 @@ public class DataSetGroupService {
@Resource
private
DatasetGroupMapper
datasetGroupMapper
;
@Resource
@Lazy
private
DataSetTableService
dataSetTableService
;
@Resource
private
ExtDataSetGroupMapper
extDataSetGroupMapper
;
...
...
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
浏览文件 @
f9bb4b31
...
...
@@ -35,6 +35,7 @@ import org.apache.poi.xssf.usermodel.XSSFSheet;
import
org.apache.poi.xssf.usermodel.XSSFWorkbook
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.stereotype.Service
;
import
org.springframework.transaction.annotation.Transactional
;
import
org.springframework.web.multipart.MultipartFile
;
import
javax.annotation.Resource
;
...
...
@@ -50,6 +51,7 @@ import java.util.stream.Collectors;
* @Date 2021/2/23 2:54 下午
*/
@Service
@Transactional
(
rollbackFor
=
Exception
.
class
)
public
class
DataSetTableService
{
@Resource
private
DatasetTableMapper
datasetTableMapper
;
...
...
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @
f9bb4b31
...
...
@@ -63,6 +63,7 @@ import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef;
import
org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta
;
import
org.pentaho.di.www.SlaveServerJobStatus
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.context.annotation.Lazy
;
import
org.springframework.stereotype.Service
;
import
javax.annotation.Resource
;
...
...
@@ -82,6 +83,7 @@ import java.util.stream.Collectors;
public
class
ExtractDataService
{
@Resource
@Lazy
private
DataSetTableService
dataSetTableService
;
@Resource
private
DataSetTableFieldsService
dataSetTableFieldsService
;
...
...
@@ -91,7 +93,6 @@ public class ExtractDataService {
private
DataSetTableTaskService
dataSetTableTaskService
;
@Resource
private
DatasourceMapper
datasourceMapper
;
private
static
ExecutorService
pool
=
Executors
.
newScheduledThreadPool
(
50
);
//设置连接池
private
static
String
lastUpdateTime
=
"${__last_update_time__}"
;
private
static
String
currentUpdateTime
=
"${__current_update_time__}"
;
...
...
@@ -113,7 +114,6 @@ public class ExtractDataService {
"UNIQUE KEY(dataease_uuid)\n"
+
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n"
+
"PROPERTIES(\"replication_num\" = \"1\");"
;
private
static
String
shellScript
=
"curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load\n"
+
"rm -rf %s\n"
;
...
...
@@ -448,6 +448,7 @@ public class ExtractDataService {
selectSQL
=
qp
.
createQuerySQL
(
tableName
,
datasetTableFields
);
}
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
udjcStep
=
udjc
(
datasetTableFields
,
false
);
break
;
case
sqlServer:
SqlServerConfigration
sqlServerConfigration
=
new
Gson
().
fromJson
(
datasource
.
getConfiguration
(),
SqlServerConfigration
.
class
);
...
...
@@ -459,10 +460,12 @@ public class ExtractDataService {
selectSQL
=
qp
.
createQuerySQL
(
tableName
,
datasetTableFields
);
}
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
udjcStep
=
udjc
(
datasetTableFields
,
false
);
break
;
case
excel:
String
filePath
=
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
).
getData
();
inputStep
=
excelInputStep
(
filePath
,
datasetTableFields
);
udjcStep
=
udjc
(
datasetTableFields
,
true
);
default
:
break
;
}
...
...
@@ -487,7 +490,7 @@ public class ExtractDataService {
break
;
}
udjcStep
=
udjc
(
datasetTableFields
);
outputStep
=
outputStep
(
dorisOutputTable
);
hi1
=
new
TransHopMeta
(
inputStep
,
udjcStep
);
hi2
=
new
TransHopMeta
(
udjcStep
,
outputStep
);
...
...
@@ -569,11 +572,11 @@ public class ExtractDataService {
return
outputStep
;
}
private
StepMeta
udjc
(
List
<
DatasetTableField
>
datasetTableFields
)
{
String
needToChangeolumnType
=
""
;
private
StepMeta
udjc
(
List
<
DatasetTableField
>
datasetTableFields
,
boolean
isExcel
)
{
String
needToChange
C
olumnType
=
""
;
for
(
DatasetTableField
datasetTableField
:
datasetTableFields
)
{
if
(
datasetTableField
.
getDeExtractType
()
!=
null
&&
datasetTableField
.
getDeExtractType
()
==
4
)
{
needToChange
olumnType
=
needToChange
olumnType
+
alterColumnTypeCode
.
replace
(
"FILED"
,
datasetTableField
.
getOriginName
());
needToChange
ColumnType
=
needToChangeC
olumnType
+
alterColumnTypeCode
.
replace
(
"FILED"
,
datasetTableField
.
getOriginName
());
}
}
...
...
@@ -583,8 +586,13 @@ public class ExtractDataService {
fields
.
add
(
fieldInfo
);
userDefinedJavaClassMeta
.
setFieldInfo
(
fields
);
List
<
UserDefinedJavaClassDef
>
definitions
=
new
ArrayList
<
UserDefinedJavaClassDef
>();
UserDefinedJavaClassDef
userDefinedJavaClassDef
=
new
UserDefinedJavaClassDef
(
UserDefinedJavaClassDef
.
ClassType
.
TRANSFORM_CLASS
,
"Processor"
,
code
.
replace
(
"alterColumnTypeCode"
,
needToChangeolumnType
).
replace
(
"Column_Fields"
,
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getOriginName
).
collect
(
Collectors
.
toList
()))));
String
tmp_code
=
code
.
replace
(
"alterColumnTypeCode"
,
needToChangeColumnType
).
replace
(
"Column_Fields"
,
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getOriginName
).
collect
(
Collectors
.
toList
())));
if
(
isExcel
){
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
handleExcelIntColumn
);
}
else
{
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
""
);
}
UserDefinedJavaClassDef
userDefinedJavaClassDef
=
new
UserDefinedJavaClassDef
(
UserDefinedJavaClassDef
.
ClassType
.
TRANSFORM_CLASS
,
"Processor"
,
tmp_code
);
userDefinedJavaClassDef
.
setActive
(
true
);
definitions
.
add
(
userDefinedJavaClassDef
);
...
...
@@ -629,6 +637,14 @@ public class ExtractDataService {
" }\n"
+
" }\n"
;
private
static
String
handleExcelIntColumn
=
" \t\tif(tmp != null && tmp.endsWith(\".0\")){\n"
+
" try {\n"
+
" Integer.valueOf(tmp.substring(0, tmp.length()-2));\n"
+
" get(Fields.Out, filed).setValue(r, tmp.substring(0, tmp.length()-2));\n"
+
" get(Fields.Out, filed).getValueMeta().setType(2);\n"
+
" }catch (Exception e){}\n"
+
" }"
;
private
static
String
code
=
"import org.pentaho.di.core.row.ValueMetaInterface;\n"
+
"import java.util.List;\n"
+
"import java.io.File;\n"
+
...
...
@@ -659,6 +675,7 @@ public class ExtractDataService {
" for (String filed : fileds) {\n"
+
" String tmp = get(Fields.In, filed).getString(r);\n"
+
"alterColumnTypeCode \n"
+
"handleExcelIntColumn \n"
+
" str = str + tmp;\n"
+
" }\n"
+
"\n"
+
...
...
frontend/src/components/canvas/components/Toolbar.vue
浏览文件 @
f9bb4b31
...
...
@@ -160,7 +160,7 @@ export default {
handleFileChange
(
e
)
{
const
file
=
e
.
target
.
files
[
0
]
if
(
!
file
.
type
.
includes
(
'image'
))
{
toast
(
'只能插入图片'
)
toast
(
this
.
$t
(
'panel.picture_limit'
)
)
return
}
...
...
frontend/src/lang/en.js
浏览文件 @
f9bb4b31
...
...
@@ -815,6 +815,7 @@ export default {
input_limit_0_50
:
'0-50 chars'
},
panel
:
{
picture_limit
:
'Only pictures can be inserted'
,
drag_here
:
'Please drag the left field here'
,
copy_link_passwd
:
'Copy link and password'
,
copy_link
:
'Copy link'
,
...
...
frontend/src/lang/tw.js
浏览文件 @
f9bb4b31
...
...
@@ -814,6 +814,7 @@ export default {
input_limit_0_50
:
'0-50字符'
},
panel
:
{
picture_limit
:
'只能插入圖片'
,
drag_here
:
'請將左側字段拖至此處'
,
copy_link_passwd
:
'複製鏈接及密碼'
,
copy_link
:
'複製鏈接'
,
...
...
frontend/src/lang/zh.js
浏览文件 @
f9bb4b31
...
...
@@ -816,6 +816,7 @@ export default {
input_limit_0_50
:
'0-50字符'
},
panel
:
{
picture_limit
:
'只能插入图片'
,
drag_here
:
'请将左侧字段拖至此处'
,
copy_link_passwd
:
'复制链接及密码'
,
copy_link
:
'复制链接'
,
...
...
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论