Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
D
dataease
项目
项目
详情
活动
周期分析
仓库
仓库
文件
提交
分支
标签
贡献者
图表
比较
统计图
议题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程
统计图
Wiki
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
zhu
dataease
Commits
f8164dbf
Unverified
提交
f8164dbf
authored
7月 23, 2021
作者:
fit2cloud-chenyw
提交者:
GitHub
7月 23, 2021
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #305 from dataease/pr@v1.1@fix_数据集消息flyway
fix: 1.数据集同步消息跳转路由flyway;2.消息内容增加任务名称
上级
45a4e90c
5e07c423
隐藏空白字符变更
内嵌
并排
正在显示
2 个修改的文件
包含
99 行增加
和
86 行删除
+99
-86
ExtractDataService.java
.../java/io/dataease/service/dataset/ExtractDataService.java
+96
-83
V10__web_msg.sql
backend/src/main/resources/db/migration/V10__web_msg.sql
+3
-3
没有找到文件。
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @
f8164dbf
...
@@ -22,8 +22,8 @@ import io.dataease.provider.QueryProvider;
...
@@ -22,8 +22,8 @@ import io.dataease.provider.QueryProvider;
import
io.dataease.service.message.DeMsgutil
;
import
io.dataease.service.message.DeMsgutil
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.io.FileUtils
;
import
org.apache.commons.io.FileUtils
;
import
org.apache.commons.lang3.ObjectUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.exception.ExceptionUtils
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.impl.client.CloseableHttpClient
;
import
org.apache.http.impl.client.CloseableHttpClient
;
...
@@ -128,23 +128,23 @@ public class ExtractDataService {
...
@@ -128,23 +128,23 @@ public class ExtractDataService {
"fi\n"
+
"fi\n"
+
"rm -rf %s\n"
;
"rm -rf %s\n"
;
public
synchronized
boolean
existSyncTask
(
DatasetTable
datasetTable
,
DatasetTableTask
datasetTableTask
)
{
public
synchronized
boolean
existSyncTask
(
DatasetTable
datasetTable
,
DatasetTableTask
datasetTableTask
)
{
datasetTable
.
setSyncStatus
(
JobStatus
.
Underway
.
name
());
datasetTable
.
setSyncStatus
(
JobStatus
.
Underway
.
name
());
DatasetTableExample
example
=
new
DatasetTableExample
();
DatasetTableExample
example
=
new
DatasetTableExample
();
example
.
createCriteria
().
andIdEqualTo
(
datasetTable
.
getId
()).
andSyncStatusNotEqualTo
(
JobStatus
.
Underway
.
name
());
example
.
createCriteria
().
andIdEqualTo
(
datasetTable
.
getId
()).
andSyncStatusNotEqualTo
(
JobStatus
.
Underway
.
name
());
example
.
or
(
example
.
createCriteria
().
andIdEqualTo
(
datasetTable
.
getId
()).
andSyncStatusIsNull
());
example
.
or
(
example
.
createCriteria
().
andIdEqualTo
(
datasetTable
.
getId
()).
andSyncStatusIsNull
());
Boolean
existSyncTask
=
datasetTableMapper
.
updateByExampleSelective
(
datasetTable
,
example
)
==
0
;
Boolean
existSyncTask
=
datasetTableMapper
.
updateByExampleSelective
(
datasetTable
,
example
)
==
0
;
if
(
existSyncTask
)
{
if
(
existSyncTask
)
{
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
datasetTableTaskLog
.
setTaskId
(
datasetTableTask
.
getId
());
datasetTableTaskLog
.
setTaskId
(
datasetTableTask
.
getId
());
datasetTableTaskLog
.
setTableId
(
datasetTable
.
getId
());
datasetTableTaskLog
.
setTableId
(
datasetTable
.
getId
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
)
&&
datasetTableTaskLogs
.
get
(
0
).
getTriggerType
().
equalsIgnoreCase
(
TriggerType
.
Custom
.
name
()))
{
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
)
&&
datasetTableTaskLogs
.
get
(
0
).
getTriggerType
().
equalsIgnoreCase
(
TriggerType
.
Custom
.
name
()))
{
return
false
;
return
false
;
}
}
return
true
;
return
true
;
}
else
{
}
else
{
datasetTableTask
.
setLastExecTime
(
System
.
currentTimeMillis
());
datasetTableTask
.
setLastExecTime
(
System
.
currentTimeMillis
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Underway
.
name
());
dataSetTableTaskService
.
update
(
datasetTableTask
);
dataSetTableTaskService
.
update
(
datasetTableTask
);
...
@@ -224,7 +224,7 @@ public class ExtractDataService {
...
@@ -224,7 +224,7 @@ public class ExtractDataService {
}
}
//侵入式清除下属视图缓存
//侵入式清除下属视图缓存
List
<
String
>
viewIds
=
extChartViewMapper
.
allViewIds
(
datasetTableId
);
List
<
String
>
viewIds
=
extChartViewMapper
.
allViewIds
(
datasetTableId
);
if
(
CollectionUtils
.
isNotEmpty
(
viewIds
)){
if
(
CollectionUtils
.
isNotEmpty
(
viewIds
))
{
viewIds
.
forEach
(
viewId
->
{
viewIds
.
forEach
(
viewId
->
{
CacheUtils
.
remove
(
JdbcConstants
.
VIEW_CACHE_KEY
,
viewId
);
CacheUtils
.
remove
(
JdbcConstants
.
VIEW_CACHE_KEY
,
viewId
);
});
});
...
@@ -233,26 +233,26 @@ public class ExtractDataService {
...
@@ -233,26 +233,26 @@ public class ExtractDataService {
public
void
extractData
(
String
datasetTableId
,
String
taskId
,
String
type
,
JobExecutionContext
context
)
{
public
void
extractData
(
String
datasetTableId
,
String
taskId
,
String
type
,
JobExecutionContext
context
)
{
DatasetTable
datasetTable
=
getDatasetTable
(
datasetTableId
);
DatasetTable
datasetTable
=
getDatasetTable
(
datasetTableId
);
if
(
datasetTable
==
null
)
{
if
(
datasetTable
==
null
)
{
LogUtil
.
error
(
"Can not find DatasetTable: "
+
datasetTableId
);
LogUtil
.
error
(
"Can not find DatasetTable: "
+
datasetTableId
);
return
;
return
;
}
}
DatasetTableTask
datasetTableTask
=
datasetTableTaskMapper
.
selectByPrimaryKey
(
taskId
);
DatasetTableTask
datasetTableTask
=
datasetTableTaskMapper
.
selectByPrimaryKey
(
taskId
);
if
(
datasetTableTask
==
null
)
{
if
(
datasetTableTask
==
null
)
{
return
;
return
;
}
}
if
(
datasetTableTask
.
getStatus
().
equalsIgnoreCase
(
TaskStatus
.
Stopped
.
name
()))
{
if
(
datasetTableTask
.
getStatus
().
equalsIgnoreCase
(
TaskStatus
.
Stopped
.
name
()))
{
LogUtil
.
info
(
"Skip synchronization task, task ID : "
+
datasetTableTask
.
getId
());
LogUtil
.
info
(
"Skip synchronization task, task ID : "
+
datasetTableTask
.
getId
());
return
;
return
;
}
}
if
(
existSyncTask
(
datasetTable
,
datasetTableTask
))
{
if
(
existSyncTask
(
datasetTable
,
datasetTableTask
))
{
LogUtil
.
info
(
"Skip synchronization task for dataset, dataset ID : "
+
datasetTableId
);
LogUtil
.
info
(
"Skip synchronization task for dataset, dataset ID : "
+
datasetTableId
);
return
;
return
;
}
}
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
UpdateType
updateType
=
UpdateType
.
valueOf
(
type
);
UpdateType
updateType
=
UpdateType
.
valueOf
(
type
);
if
(
context
!=
null
)
{
if
(
context
!=
null
)
{
datasetTable
.
setQrtzInstance
(
context
.
getFireInstanceId
());
datasetTable
.
setQrtzInstance
(
context
.
getFireInstanceId
());
datasetTableMapper
.
updateByPrimaryKeySelective
(
datasetTable
);
datasetTableMapper
.
updateByPrimaryKeySelective
(
datasetTable
);
}
}
...
@@ -276,19 +276,19 @@ public class ExtractDataService {
...
@@ -276,19 +276,19 @@ public class ExtractDataService {
switch
(
updateType
)
{
switch
(
updateType
)
{
case
all_scope:
// 全量更新
case
all_scope:
// 全量更新
try
{
try
{
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
CRON
.
toString
()))
{
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
CRON
.
toString
()))
{
datasetTableTaskLog
=
writeDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
datasetTableTaskLog
=
writeDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
}
}
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
SIMPLE
.
toString
()))
{
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
SIMPLE
.
toString
()))
{
datasetTableTaskLog
=
getDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
datasetTableTaskLog
=
getDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
}
}
createDorisTable
(
DorisTableUtils
.
dorisName
(
datasetTableId
),
dorisTablColumnSql
);
createDorisTable
(
DorisTableUtils
.
dorisName
(
datasetTableId
),
dorisTablColumnSql
);
createDorisTable
(
DorisTableUtils
.
dorisTmpName
(
DorisTableUtils
.
dorisName
(
datasetTableId
)),
dorisTablColumnSql
);
createDorisTable
(
DorisTableUtils
.
dorisTmpName
(
DorisTableUtils
.
dorisName
(
datasetTableId
)),
dorisTablColumnSql
);
generateTransFile
(
"all_scope"
,
datasetTable
,
datasource
,
datasetTableFields
,
null
);
generateTransFile
(
"all_scope"
,
datasetTable
,
datasource
,
datasetTableFields
,
null
);
if
(
datasetTable
.
getType
().
equalsIgnoreCase
(
"sql"
))
{
if
(
datasetTable
.
getType
().
equalsIgnoreCase
(
"sql"
))
{
generateJobFile
(
"all_scope"
,
datasetTable
,
fetchSqlField
(
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
).
getSql
(),
datasource
));
generateJobFile
(
"all_scope"
,
datasetTable
,
fetchSqlField
(
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
).
getSql
(),
datasource
));
}
else
{
}
else
{
generateJobFile
(
"all_scope"
,
datasetTable
,
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getDataeaseName
).
collect
(
Collectors
.
toList
())));
generateJobFile
(
"all_scope"
,
datasetTable
,
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getDataeaseName
).
collect
(
Collectors
.
toList
())));
}
}
Long
execTime
=
System
.
currentTimeMillis
();
Long
execTime
=
System
.
currentTimeMillis
();
...
@@ -296,7 +296,7 @@ public class ExtractDataService {
...
@@ -296,7 +296,7 @@ public class ExtractDataService {
replaceTable
(
DorisTableUtils
.
dorisName
(
datasetTableId
));
replaceTable
(
DorisTableUtils
.
dorisName
(
datasetTableId
));
saveSucessLog
(
datasetTableTaskLog
);
saveSucessLog
(
datasetTableTaskLog
);
sendWebMsg
(
datasetTable
,
taskId
,
true
);
sendWebMsg
(
datasetTable
,
datasetTableTask
,
true
);
deleteFile
(
"all_scope"
,
datasetTableId
);
deleteFile
(
"all_scope"
,
datasetTableId
);
...
@@ -304,16 +304,16 @@ public class ExtractDataService {
...
@@ -304,16 +304,16 @@ public class ExtractDataService {
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Completed
.
name
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Completed
.
name
());
dataSetTableTaskService
.
update
(
datasetTableTask
);
dataSetTableTaskService
.
update
(
datasetTableTask
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
saveErrorLog
(
datasetTableId
,
taskId
,
e
);
saveErrorLog
(
datasetTableId
,
taskId
,
e
);
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Error
.
name
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Error
.
name
());
dataSetTableTaskService
.
update
(
datasetTableTask
);
dataSetTableTaskService
.
update
(
datasetTableTask
);
sendWebMsg
(
datasetTable
,
taskId
,
false
);
sendWebMsg
(
datasetTable
,
datasetTableTask
,
false
);
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Error
,
null
);
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Error
,
null
);
dropDorisTable
(
DorisTableUtils
.
dorisTmpName
(
DorisTableUtils
.
dorisName
(
datasetTableId
)));
dropDorisTable
(
DorisTableUtils
.
dorisTmpName
(
DorisTableUtils
.
dorisName
(
datasetTableId
)));
deleteFile
(
"all_scope"
,
datasetTableId
);
deleteFile
(
"all_scope"
,
datasetTableId
);
}
finally
{
}
finally
{
}
}
break
;
break
;
...
@@ -330,10 +330,10 @@ public class ExtractDataService {
...
@@ -330,10 +330,10 @@ public class ExtractDataService {
return
;
return
;
}
}
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
CRON
.
toString
()))
{
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
CRON
.
toString
()))
{
datasetTableTaskLog
=
writeDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
datasetTableTaskLog
=
writeDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
}
}
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
SIMPLE
.
toString
()))
{
if
(
datasetTableTask
!=
null
&&
datasetTableTask
.
getRate
().
equalsIgnoreCase
(
ScheduleType
.
SIMPLE
.
toString
()))
{
datasetTableTaskLog
=
getDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
datasetTableTaskLog
=
getDatasetTableTaskLog
(
datasetTableTaskLog
,
datasetTableId
,
taskId
);
}
}
Long
execTime
=
System
.
currentTimeMillis
();
Long
execTime
=
System
.
currentTimeMillis
();
...
@@ -354,7 +354,7 @@ public class ExtractDataService {
...
@@ -354,7 +354,7 @@ public class ExtractDataService {
}
}
saveSucessLog
(
datasetTableTaskLog
);
saveSucessLog
(
datasetTableTaskLog
);
sendWebMsg
(
datasetTable
,
taskId
,
true
);
sendWebMsg
(
datasetTable
,
datasetTableTask
,
true
);
deleteFile
(
"incremental_add"
,
datasetTableId
);
deleteFile
(
"incremental_add"
,
datasetTableId
);
deleteFile
(
"incremental_delete"
,
datasetTableId
);
deleteFile
(
"incremental_delete"
,
datasetTableId
);
...
@@ -362,21 +362,21 @@ public class ExtractDataService {
...
@@ -362,21 +362,21 @@ public class ExtractDataService {
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Completed
,
execTime
);
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Completed
,
execTime
);
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Completed
.
name
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Completed
.
name
());
dataSetTableTaskService
.
update
(
datasetTableTask
);
dataSetTableTaskService
.
update
(
datasetTableTask
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
saveErrorLog
(
datasetTableId
,
taskId
,
e
);
saveErrorLog
(
datasetTableId
,
taskId
,
e
);
sendWebMsg
(
datasetTable
,
taskId
,
false
);
sendWebMsg
(
datasetTable
,
datasetTableTask
,
false
);
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Error
,
null
);
updateTableStatus
(
datasetTableId
,
datasetTable
,
JobStatus
.
Error
,
null
);
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Error
.
name
());
datasetTableTask
.
setLastExecStatus
(
JobStatus
.
Error
.
name
());
dataSetTableTaskService
.
update
(
datasetTableTask
);
dataSetTableTaskService
.
update
(
datasetTableTask
);
deleteFile
(
"incremental_add"
,
datasetTableId
);
deleteFile
(
"incremental_add"
,
datasetTableId
);
deleteFile
(
"incremental_delete"
,
datasetTableId
);
deleteFile
(
"incremental_delete"
,
datasetTableId
);
}
finally
{
}
finally
{
}
}
break
;
break
;
}
}
//侵入式清除下属视图缓存
//侵入式清除下属视图缓存
List
<
String
>
viewIds
=
extChartViewMapper
.
allViewIds
(
datasetTableId
);
List
<
String
>
viewIds
=
extChartViewMapper
.
allViewIds
(
datasetTableId
);
if
(
CollectionUtils
.
isNotEmpty
(
viewIds
)){
if
(
CollectionUtils
.
isNotEmpty
(
viewIds
))
{
viewIds
.
forEach
(
viewId
->
{
viewIds
.
forEach
(
viewId
->
{
CacheUtils
.
remove
(
JdbcConstants
.
VIEW_CACHE_KEY
,
viewId
);
CacheUtils
.
remove
(
JdbcConstants
.
VIEW_CACHE_KEY
,
viewId
);
});
});
...
@@ -384,7 +384,8 @@ public class ExtractDataService {
...
@@ -384,7 +384,8 @@ public class ExtractDataService {
}
}
private
void
sendWebMsg
(
DatasetTable
datasetTable
,
String
taskId
,
Boolean
status
)
{
private
void
sendWebMsg
(
DatasetTable
datasetTable
,
DatasetTableTask
datasetTableTask
,
Boolean
status
)
{
String
taskId
=
datasetTableTask
.
getId
();
String
msg
=
status
?
"成功"
:
"失败"
;
String
msg
=
status
?
"成功"
:
"失败"
;
Long
typeId
=
status
?
5L
:
6L
;
Long
typeId
=
status
?
5L
:
6L
;
String
id
=
datasetTable
.
getId
();
String
id
=
datasetTable
.
getId
();
...
@@ -392,18 +393,22 @@ public class ExtractDataService {
...
@@ -392,18 +393,22 @@ public class ExtractDataService {
Set
<
Long
>
userIds
=
AuthUtils
.
userIdsByURD
(
authURD
);
Set
<
Long
>
userIds
=
AuthUtils
.
userIdsByURD
(
authURD
);
Gson
gson
=
new
Gson
();
Gson
gson
=
new
Gson
();
userIds
.
forEach
(
userId
->
{
userIds
.
forEach
(
userId
->
{
Map
<
String
,
Object
>
param
=
new
HashMap
<>();
Map
<
String
,
Object
>
param
=
new
HashMap
<>();
param
.
put
(
"tableId"
,
id
);
param
.
put
(
"tableId"
,
id
);
if
(
StringUtils
.
isNotEmpty
(
taskId
)){
if
(
StringUtils
.
isNotEmpty
(
taskId
))
{
param
.
put
(
"taskId"
,
taskId
);
param
.
put
(
"taskId"
,
taskId
);
}
}
DeMsgutil
.
sendMsg
(
userId
,
typeId
,
1L
,
"数据集【"
+
datasetTable
.
getName
()+
"】同步"
+
msg
,
gson
.
toJson
(
param
));
String
content
=
"数据集【"
+
datasetTable
.
getName
()
+
"】同步"
+
msg
;
if
(
ObjectUtils
.
isNotEmpty
(
datasetTableTask
)
&&
ObjectUtils
.
isNotEmpty
(
datasetTableTask
.
getName
()))
{
content
+=
" 任务名称【"
+
datasetTableTask
.
getName
()
+
"】"
;
}
DeMsgutil
.
sendMsg
(
userId
,
typeId
,
1L
,
content
,
gson
.
toJson
(
param
));
});
});
}
}
private
void
updateTableStatus
(
String
datasetTableId
,
DatasetTable
datasetTable
,
JobStatus
completed
,
Long
execTime
)
{
private
void
updateTableStatus
(
String
datasetTableId
,
DatasetTable
datasetTable
,
JobStatus
completed
,
Long
execTime
)
{
datasetTable
.
setSyncStatus
(
completed
.
name
());
datasetTable
.
setSyncStatus
(
completed
.
name
());
if
(
execTime
!=
null
)
{
if
(
execTime
!=
null
)
{
datasetTable
.
setLastUpdateTime
(
execTime
);
datasetTable
.
setLastUpdateTime
(
execTime
);
}
}
DatasetTableExample
example
=
new
DatasetTableExample
();
DatasetTableExample
example
=
new
DatasetTableExample
();
...
@@ -417,16 +422,16 @@ public class ExtractDataService {
...
@@ -417,16 +422,16 @@ public class ExtractDataService {
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
}
}
private
void
saveErrorLog
(
String
datasetTableId
,
String
taskId
,
Exception
e
){
private
void
saveErrorLog
(
String
datasetTableId
,
String
taskId
,
Exception
e
)
{
LogUtil
.
error
(
"Extract data error: "
+
datasetTableId
,
e
);
LogUtil
.
error
(
"Extract data error: "
+
datasetTableId
,
e
);
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
DatasetTableTaskLog
datasetTableTaskLog
=
new
DatasetTableTaskLog
();
datasetTableTaskLog
.
setTableId
(
datasetTableId
);
datasetTableTaskLog
.
setTableId
(
datasetTableId
);
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
if
(
StringUtils
.
isNotEmpty
(
taskId
))
{
if
(
StringUtils
.
isNotEmpty
(
taskId
))
{
datasetTableTaskLog
.
setTaskId
(
taskId
);
datasetTableTaskLog
.
setTaskId
(
taskId
);
}
}
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
))
{
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
))
{
datasetTableTaskLog
=
datasetTableTaskLogs
.
get
(
0
);
datasetTableTaskLog
=
datasetTableTaskLogs
.
get
(
0
);
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Error
.
name
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Error
.
name
());
datasetTableTaskLog
.
setInfo
(
e
.
getMessage
());
datasetTableTaskLog
.
setInfo
(
e
.
getMessage
());
...
@@ -445,7 +450,7 @@ public class ExtractDataService {
...
@@ -445,7 +450,7 @@ public class ExtractDataService {
if
(
datasetTableField
.
getSize
()
>
65533
||
datasetTableField
.
getSize
()
*
3
>
65533
)
{
if
(
datasetTableField
.
getSize
()
>
65533
||
datasetTableField
.
getSize
()
*
3
>
65533
)
{
Column_Fields
=
Column_Fields
+
"varchar(65533)"
+
",`"
;
Column_Fields
=
Column_Fields
+
"varchar(65533)"
+
",`"
;
}
else
{
}
else
{
Column_Fields
=
Column_Fields
+
"varchar(lenth)"
.
replace
(
"lenth"
,
String
.
valueOf
(
datasetTableField
.
getSize
()
*
3
))
+
",`"
;
Column_Fields
=
Column_Fields
+
"varchar(lenth)"
.
replace
(
"lenth"
,
String
.
valueOf
(
datasetTableField
.
getSize
()
*
3
))
+
",`"
;
}
}
break
;
break
;
case
1
:
case
1
:
...
@@ -488,7 +493,8 @@ public class ExtractDataService {
...
@@ -488,7 +493,8 @@ public class ExtractDataService {
datasourceRequest
.
setDatasource
(
dorisDatasource
);
datasourceRequest
.
setDatasource
(
dorisDatasource
);
datasourceRequest
.
setQuery
(
dropTableSql
.
replace
(
"TABLE_NAME"
,
dorisTableName
));
datasourceRequest
.
setQuery
(
dropTableSql
.
replace
(
"TABLE_NAME"
,
dorisTableName
));
jdbcProvider
.
exec
(
datasourceRequest
);
jdbcProvider
.
exec
(
datasourceRequest
);
}
catch
(
Exception
ignore
){}
}
catch
(
Exception
ignore
)
{
}
}
}
private
void
replaceTable
(
String
dorisTableName
)
throws
Exception
{
private
void
replaceTable
(
String
dorisTableName
)
throws
Exception
{
...
@@ -501,14 +507,15 @@ public class ExtractDataService {
...
@@ -501,14 +507,15 @@ public class ExtractDataService {
jdbcProvider
.
exec
(
datasourceRequest
);
jdbcProvider
.
exec
(
datasourceRequest
);
}
}
private
DatasetTable
getDatasetTable
(
String
datasetTableId
){
private
DatasetTable
getDatasetTable
(
String
datasetTableId
)
{
for
(
int
i
=
0
;
i
<
5
;
i
++)
{
for
(
int
i
=
0
;
i
<
5
;
i
++)
{
DatasetTable
datasetTable
=
dataSetTableService
.
get
(
datasetTableId
);
DatasetTable
datasetTable
=
dataSetTableService
.
get
(
datasetTableId
);
if
(
datasetTable
==
null
)
{
if
(
datasetTable
==
null
)
{
try
{
try
{
Thread
.
sleep
(
1000
);
Thread
.
sleep
(
1000
);
}
catch
(
Exception
ignore
){}
}
catch
(
Exception
ignore
)
{
}
else
{
}
}
else
{
return
datasetTable
;
return
datasetTable
;
}
}
}
}
...
@@ -521,11 +528,11 @@ public class ExtractDataService {
...
@@ -521,11 +528,11 @@ public class ExtractDataService {
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTaskLog
.
setTriggerType
(
TriggerType
.
Cron
.
name
());
datasetTableTaskLog
.
setTriggerType
(
TriggerType
.
Cron
.
name
());
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
if
(
CollectionUtils
.
isEmpty
(
datasetTableTaskLogs
))
{
if
(
CollectionUtils
.
isEmpty
(
datasetTableTaskLogs
))
{
datasetTableTaskLog
.
setStartTime
(
System
.
currentTimeMillis
());
datasetTableTaskLog
.
setStartTime
(
System
.
currentTimeMillis
());
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
return
datasetTableTaskLog
;
return
datasetTableTaskLog
;
}
else
{
}
else
{
return
datasetTableTaskLogs
.
get
(
0
);
return
datasetTableTaskLogs
.
get
(
0
);
}
}
}
}
...
@@ -534,14 +541,15 @@ public class ExtractDataService {
...
@@ -534,14 +541,15 @@ public class ExtractDataService {
datasetTableTaskLog
.
setTableId
(
datasetTableId
);
datasetTableTaskLog
.
setTableId
(
datasetTableId
);
datasetTableTaskLog
.
setTaskId
(
taskId
);
datasetTableTaskLog
.
setTaskId
(
taskId
);
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
datasetTableTaskLog
.
setStatus
(
JobStatus
.
Underway
.
name
());
for
(
int
i
=
0
;
i
<
5
;
i
++)
{
for
(
int
i
=
0
;
i
<
5
;
i
++)
{
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
List
<
DatasetTableTaskLog
>
datasetTableTaskLogs
=
dataSetTableTaskLogService
.
select
(
datasetTableTaskLog
);
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
))
{
if
(
CollectionUtils
.
isNotEmpty
(
datasetTableTaskLogs
))
{
return
datasetTableTaskLogs
.
get
(
0
);
return
datasetTableTaskLogs
.
get
(
0
);
}
}
try
{
try
{
Thread
.
sleep
(
1000
);
Thread
.
sleep
(
1000
);
}
catch
(
Exception
ignore
){}
}
catch
(
Exception
ignore
)
{
}
}
}
datasetTableTaskLog
.
setStartTime
(
System
.
currentTimeMillis
());
datasetTableTaskLog
.
setStartTime
(
System
.
currentTimeMillis
());
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
dataSetTableTaskLogService
.
save
(
datasetTableTaskLog
);
...
@@ -694,7 +702,7 @@ public class ExtractDataService {
...
@@ -694,7 +702,7 @@ public class ExtractDataService {
datasourceRequest
.
setDatasource
(
ds
);
datasourceRequest
.
setDatasource
(
ds
);
datasourceRequest
.
setQuery
(
qp
.
wrapSql
(
sql
));
datasourceRequest
.
setQuery
(
qp
.
wrapSql
(
sql
));
List
<
String
>
dorisFileds
=
new
ArrayList
<>();
List
<
String
>
dorisFileds
=
new
ArrayList
<>();
datasourceProvider
.
fetchResultField
(
datasourceRequest
).
stream
().
map
(
TableFiled:
:
getFieldName
).
forEach
(
filed
->{
datasourceProvider
.
fetchResultField
(
datasourceRequest
).
stream
().
map
(
TableFiled:
:
getFieldName
).
forEach
(
filed
->
{
dorisFileds
.
add
(
DorisTableUtils
.
columnName
(
filed
));
dorisFileds
.
add
(
DorisTableUtils
.
columnName
(
filed
));
});
});
return
String
.
join
(
","
,
dorisFileds
);
return
String
.
join
(
","
,
dorisFileds
);
...
@@ -716,7 +724,7 @@ public class ExtractDataService {
...
@@ -716,7 +724,7 @@ public class ExtractDataService {
case
mysql:
case
mysql:
MysqlConfigration
mysqlConfigration
=
new
Gson
().
fromJson
(
datasource
.
getConfiguration
(),
MysqlConfigration
.
class
);
MysqlConfigration
mysqlConfigration
=
new
Gson
().
fromJson
(
datasource
.
getConfiguration
(),
MysqlConfigration
.
class
);
dataMeta
=
new
DatabaseMeta
(
"db"
,
"MYSQL"
,
"Native"
,
mysqlConfigration
.
getHost
(),
mysqlConfigration
.
getDataBase
(),
mysqlConfigration
.
getPort
().
toString
(),
mysqlConfigration
.
getUsername
(),
mysqlConfigration
.
getPassword
());
dataMeta
=
new
DatabaseMeta
(
"db"
,
"MYSQL"
,
"Native"
,
mysqlConfigration
.
getHost
(),
mysqlConfigration
.
getDataBase
(),
mysqlConfigration
.
getPort
().
toString
(),
mysqlConfigration
.
getUsername
(),
mysqlConfigration
.
getPassword
());
dataMeta
.
addExtraOption
(
"MYSQL"
,
"characterEncoding"
,
"UTF-8"
);
dataMeta
.
addExtraOption
(
"MYSQL"
,
"characterEncoding"
,
"UTF-8"
);
transMeta
.
addDatabase
(
dataMeta
);
transMeta
.
addDatabase
(
dataMeta
);
selectSQL
=
getSelectSQL
(
extractType
,
datasetTable
,
datasource
,
datasetTableFields
,
selectSQL
);
selectSQL
=
getSelectSQL
(
extractType
,
datasetTable
,
datasource
,
datasetTableFields
,
selectSQL
);
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
...
@@ -732,13 +740,13 @@ public class ExtractDataService {
...
@@ -732,13 +740,13 @@ public class ExtractDataService {
break
;
break
;
case
oracle:
case
oracle:
OracleConfigration
oracleConfigration
=
new
Gson
().
fromJson
(
datasource
.
getConfiguration
(),
OracleConfigration
.
class
);
OracleConfigration
oracleConfigration
=
new
Gson
().
fromJson
(
datasource
.
getConfiguration
(),
OracleConfigration
.
class
);
if
(
oracleConfigration
.
getConnectionType
().
equalsIgnoreCase
(
"serviceName"
))
{
if
(
oracleConfigration
.
getConnectionType
().
equalsIgnoreCase
(
"serviceName"
))
{
String
database
=
"(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ORACLE_HOSTNAME)(PORT = ORACLE_PORT))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = ORACLE_SERVICE_NAME )))"
.
replace
(
"ORACLE_HOSTNAME"
,
oracleConfigration
.
getHost
()).
replace
(
"ORACLE_PORT"
,
oracleConfigration
.
getPort
().
toString
()).
replace
(
"ORACLE_SERVICE_NAME"
,
oracleConfigration
.
getDataBase
());
String
database
=
"(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ORACLE_HOSTNAME)(PORT = ORACLE_PORT))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = ORACLE_SERVICE_NAME )))"
.
replace
(
"ORACLE_HOSTNAME"
,
oracleConfigration
.
getHost
()).
replace
(
"ORACLE_PORT"
,
oracleConfigration
.
getPort
().
toString
()).
replace
(
"ORACLE_SERVICE_NAME"
,
oracleConfigration
.
getDataBase
());
dataMeta
=
new
DatabaseMeta
(
"db"
,
"ORACLE"
,
"Native"
,
""
,
database
,
"-1"
,
oracleConfigration
.
getUsername
(),
oracleConfigration
.
getPassword
());
dataMeta
=
new
DatabaseMeta
(
"db"
,
"ORACLE"
,
"Native"
,
""
,
database
,
"-1"
,
oracleConfigration
.
getUsername
(),
oracleConfigration
.
getPassword
());
}
else
{
}
else
{
dataMeta
=
new
DatabaseMeta
(
"db"
,
"ORACLE"
,
"Native"
,
oracleConfigration
.
getHost
(),
oracleConfigration
.
getDataBase
(),
oracleConfigration
.
getPort
().
toString
(),
oracleConfigration
.
getUsername
(),
oracleConfigration
.
getPassword
());
dataMeta
=
new
DatabaseMeta
(
"db"
,
"ORACLE"
,
"Native"
,
oracleConfigration
.
getHost
(),
oracleConfigration
.
getDataBase
(),
oracleConfigration
.
getPort
().
toString
(),
oracleConfigration
.
getUsername
(),
oracleConfigration
.
getPassword
());
}
}
transMeta
.
addDatabase
(
dataMeta
);
transMeta
.
addDatabase
(
dataMeta
);
selectSQL
=
getSelectSQL
(
extractType
,
datasetTable
,
datasource
,
datasetTableFields
,
selectSQL
);
selectSQL
=
getSelectSQL
(
extractType
,
datasetTable
,
datasource
,
datasetTableFields
,
selectSQL
);
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
inputStep
=
inputStep
(
transMeta
,
selectSQL
);
...
@@ -794,12 +802,12 @@ public class ExtractDataService {
...
@@ -794,12 +802,12 @@ public class ExtractDataService {
selectSQL
=
qp
.
createRawQuerySQL
(
tableName
,
datasetTableFields
);
selectSQL
=
qp
.
createRawQuerySQL
(
tableName
,
datasetTableFields
);
}
}
if
(
extractType
.
equalsIgnoreCase
(
"all_scope"
)
&&
datasetTable
.
getType
().
equalsIgnoreCase
(
"sql"
))
{
if
(
extractType
.
equalsIgnoreCase
(
"all_scope"
)
&&
datasetTable
.
getType
().
equalsIgnoreCase
(
"sql"
))
{
selectSQL
=
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
).
getSql
();
selectSQL
=
new
Gson
().
fromJson
(
datasetTable
.
getInfo
(),
DataTableInfoDTO
.
class
).
getSql
();
QueryProvider
qp
=
ProviderFactory
.
getQueryProvider
(
datasource
.
getType
());
QueryProvider
qp
=
ProviderFactory
.
getQueryProvider
(
datasource
.
getType
());
selectSQL
=
qp
.
createRawQuerySQLAsTmp
(
selectSQL
,
datasetTableFields
);
selectSQL
=
qp
.
createRawQuerySQLAsTmp
(
selectSQL
,
datasetTableFields
);
}
}
if
(!
extractType
.
equalsIgnoreCase
(
"all_scope"
))
{
if
(!
extractType
.
equalsIgnoreCase
(
"all_scope"
))
{
QueryProvider
qp
=
ProviderFactory
.
getQueryProvider
(
datasource
.
getType
());
QueryProvider
qp
=
ProviderFactory
.
getQueryProvider
(
datasource
.
getType
());
selectSQL
=
qp
.
createRawQuerySQLAsTmp
(
selectSQL
,
datasetTableFields
);
selectSQL
=
qp
.
createRawQuerySQLAsTmp
(
selectSQL
,
datasetTableFields
);
}
}
...
@@ -822,23 +830,25 @@ public class ExtractDataService {
...
@@ -822,23 +830,25 @@ public class ExtractDataService {
ExcelInputMeta
excelInputMeta
=
new
ExcelInputMeta
();
ExcelInputMeta
excelInputMeta
=
new
ExcelInputMeta
();
if
(
StringUtils
.
equalsIgnoreCase
(
suffix
,
"xlsx"
))
{
if
(
StringUtils
.
equalsIgnoreCase
(
suffix
,
"xlsx"
))
{
excelInputMeta
.
setSpreadSheetType
(
SpreadSheetType
.
SAX_POI
);
excelInputMeta
.
setSpreadSheetType
(
SpreadSheetType
.
SAX_POI
);
try
{
try
{
InputStream
inputStream
=
new
FileInputStream
(
filePath
);
InputStream
inputStream
=
new
FileInputStream
(
filePath
);
XSSFWorkbook
xssfWorkbook
=
new
XSSFWorkbook
(
inputStream
);
XSSFWorkbook
xssfWorkbook
=
new
XSSFWorkbook
(
inputStream
);
XSSFSheet
sheet0
=
xssfWorkbook
.
getSheetAt
(
0
);
XSSFSheet
sheet0
=
xssfWorkbook
.
getSheetAt
(
0
);
excelInputMeta
.
setSheetName
(
new
String
[]{
sheet0
.
getSheetName
()});
excelInputMeta
.
setSheetName
(
new
String
[]{
sheet0
.
getSheetName
()});
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
}
}
if
(
StringUtils
.
equalsIgnoreCase
(
suffix
,
"xls"
))
{
if
(
StringUtils
.
equalsIgnoreCase
(
suffix
,
"xls"
))
{
excelInputMeta
.
setSpreadSheetType
(
SpreadSheetType
.
JXL
);
excelInputMeta
.
setSpreadSheetType
(
SpreadSheetType
.
JXL
);
try
{
try
{
InputStream
inputStream
=
new
FileInputStream
(
filePath
);
InputStream
inputStream
=
new
FileInputStream
(
filePath
);
HSSFWorkbook
workbook
=
new
HSSFWorkbook
(
inputStream
);
HSSFWorkbook
workbook
=
new
HSSFWorkbook
(
inputStream
);
HSSFSheet
sheet0
=
workbook
.
getSheetAt
(
0
);
HSSFSheet
sheet0
=
workbook
.
getSheetAt
(
0
);
excelInputMeta
.
setSheetName
(
new
String
[]{
sheet0
.
getSheetName
()});
excelInputMeta
.
setSheetName
(
new
String
[]{
sheet0
.
getSheetName
()});
}
catch
(
Exception
e
){
e
.
printStackTrace
();}
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
}
}
excelInputMeta
.
setPassword
(
"Encrypted"
);
excelInputMeta
.
setPassword
(
"Encrypted"
);
excelInputMeta
.
setFileName
(
new
String
[]{
filePath
});
excelInputMeta
.
setFileName
(
new
String
[]{
filePath
});
...
@@ -848,10 +858,10 @@ public class ExtractDataService {
...
@@ -848,10 +858,10 @@ public class ExtractDataService {
for
(
int
i
=
0
;
i
<
datasetTableFields
.
size
();
i
++)
{
for
(
int
i
=
0
;
i
<
datasetTableFields
.
size
();
i
++)
{
ExcelInputField
field
=
new
ExcelInputField
();
ExcelInputField
field
=
new
ExcelInputField
();
field
.
setName
(
datasetTableFields
.
get
(
i
).
getOriginName
());
field
.
setName
(
datasetTableFields
.
get
(
i
).
getOriginName
());
if
(
datasetTableFields
.
get
(
i
).
getDeExtractType
()
==
1
)
{
if
(
datasetTableFields
.
get
(
i
).
getDeExtractType
()
==
1
)
{
field
.
setType
(
"String"
);
field
.
setType
(
"String"
);
field
.
setFormat
(
"yyyy-MM-dd HH:mm:ss"
);
field
.
setFormat
(
"yyyy-MM-dd HH:mm:ss"
);
}
else
{
}
else
{
field
.
setType
(
"String"
);
field
.
setType
(
"String"
);
}
}
fields
[
i
]
=
field
;
fields
[
i
]
=
field
;
...
@@ -892,14 +902,14 @@ public class ExtractDataService {
...
@@ -892,14 +902,14 @@ public class ExtractDataService {
tmp_code
=
tmp_code
.
replace
(
"handleWraps"
,
handleWraps
);
tmp_code
=
tmp_code
.
replace
(
"handleWraps"
,
handleWraps
);
String
Column_Fields
=
""
;
String
Column_Fields
=
""
;
if
(
datasourceType
.
equals
(
DatasourceTypes
.
excel
)
||
datasourceType
.
equals
(
DatasourceTypes
.
oracle
))
{
if
(
datasourceType
.
equals
(
DatasourceTypes
.
excel
)
||
datasourceType
.
equals
(
DatasourceTypes
.
oracle
))
{
Column_Fields
=
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getOriginName
).
collect
(
Collectors
.
toList
()));
Column_Fields
=
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getOriginName
).
collect
(
Collectors
.
toList
()));
}
else
{
}
else
{
Column_Fields
=
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getDataeaseName
).
collect
(
Collectors
.
toList
()));
Column_Fields
=
String
.
join
(
","
,
datasetTableFields
.
stream
().
map
(
DatasetTableField:
:
getDataeaseName
).
collect
(
Collectors
.
toList
()));
}
}
if
(
datasourceType
.
equals
(
DatasourceTypes
.
excel
))
{
if
(
datasourceType
.
equals
(
DatasourceTypes
.
excel
))
{
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
handleExcelIntColumn
).
replace
(
"Column_Fields"
,
Column_Fields
);
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
handleExcelIntColumn
).
replace
(
"Column_Fields"
,
Column_Fields
);
}
else
{
}
else
{
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
""
).
replace
(
"Column_Fields"
,
Column_Fields
);
tmp_code
=
tmp_code
.
replace
(
"handleExcelIntColumn"
,
""
).
replace
(
"Column_Fields"
,
Column_Fields
);
}
}
UserDefinedJavaClassDef
userDefinedJavaClassDef
=
new
UserDefinedJavaClassDef
(
UserDefinedJavaClassDef
.
ClassType
.
TRANSFORM_CLASS
,
"Processor"
,
tmp_code
);
UserDefinedJavaClassDef
userDefinedJavaClassDef
=
new
UserDefinedJavaClassDef
(
UserDefinedJavaClassDef
.
ClassType
.
TRANSFORM_CLASS
,
"Processor"
,
tmp_code
);
...
@@ -914,7 +924,7 @@ public class ExtractDataService {
...
@@ -914,7 +924,7 @@ public class ExtractDataService {
return
userDefinedJavaClassStep
;
return
userDefinedJavaClassStep
;
}
}
public
void
deleteFile
(
String
type
,
String
dataSetTableId
){
public
void
deleteFile
(
String
type
,
String
dataSetTableId
)
{
String
transName
=
null
;
String
transName
=
null
;
String
jobName
=
null
;
String
jobName
=
null
;
String
fileName
=
null
;
String
fileName
=
null
;
...
@@ -929,7 +939,7 @@ public class ExtractDataService {
...
@@ -929,7 +939,7 @@ public class ExtractDataService {
transName
=
"trans_add_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
transName
=
"trans_add_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
jobName
=
"job_add_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
jobName
=
"job_add_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
fileName
=
DorisTableUtils
.
dorisAddName
(
dataSetTableId
);
fileName
=
DorisTableUtils
.
dorisAddName
(
dataSetTableId
);
break
;
break
;
case
"incremental_delete"
:
case
"incremental_delete"
:
transName
=
"trans_delete_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
transName
=
"trans_delete_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
jobName
=
"job_delete_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
jobName
=
"job_delete_"
+
DorisTableUtils
.
dorisName
(
dataSetTableId
);
...
@@ -938,28 +948,31 @@ public class ExtractDataService {
...
@@ -938,28 +948,31 @@ public class ExtractDataService {
default
:
default
:
break
;
break
;
}
}
try
{
try
{
File
file
=
new
File
(
root_path
+
fileName
+
"."
+
extention
);
File
file
=
new
File
(
root_path
+
fileName
+
"."
+
extention
);
FileUtils
.
forceDelete
(
file
);
FileUtils
.
forceDelete
(
file
);
}
catch
(
Exception
e
){}
}
catch
(
Exception
e
)
{
try
{
}
try
{
File
file
=
new
File
(
root_path
+
jobName
+
".kjb"
);
File
file
=
new
File
(
root_path
+
jobName
+
".kjb"
);
FileUtils
.
forceDelete
(
file
);
FileUtils
.
forceDelete
(
file
);
}
catch
(
Exception
e
){}
}
catch
(
Exception
e
)
{
try
{
}
try
{
File
file
=
new
File
(
root_path
+
transName
+
".ktr"
);
File
file
=
new
File
(
root_path
+
transName
+
".ktr"
);
FileUtils
.
forceDelete
(
file
);
FileUtils
.
forceDelete
(
file
);
}
catch
(
Exception
e
){}
}
catch
(
Exception
e
)
{
}
}
}
public
boolean
isKettleRunning
()
{
public
boolean
isKettleRunning
()
{
try
{
try
{
if
(!
InetAddress
.
getByName
(
carte
).
isReachable
(
1000
))
{
if
(!
InetAddress
.
getByName
(
carte
).
isReachable
(
1000
))
{
return
false
;
return
false
;
}
}
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
return
false
;
return
false
;
}
}
HttpGet
getMethod
=
new
HttpGet
(
"http://"
+
carte
+
":"
+
port
);
HttpGet
getMethod
=
new
HttpGet
(
"http://"
+
carte
+
":"
+
port
);
HttpClientManager
.
HttpClientBuilderFacade
clientBuilder
=
HttpClientManager
.
getInstance
().
createBuilder
();
HttpClientManager
.
HttpClientBuilderFacade
clientBuilder
=
HttpClientManager
.
getInstance
().
createBuilder
();
...
...
backend/src/main/resources/db/migration/V10__web_msg.sql
浏览文件 @
f8164dbf
...
@@ -56,9 +56,9 @@ BEGIN;
...
@@ -56,9 +56,9 @@ BEGIN;
INSERT
INTO
`sys_msg_type`
VALUES
(
1
,
0
,
'i18n_msg_type_panel_share'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
1
,
0
,
'i18n_msg_type_panel_share'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
2
,
1
,
'i18n_msg_type_panel_share'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
2
,
1
,
'i18n_msg_type_panel_share'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
3
,
1
,
'i18n_msg_type_panel_share_cacnel'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
3
,
1
,
'i18n_msg_type_panel_share_cacnel'
,
'panel'
,
'to-msg-share'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
4
,
0
,
'i18n_msg_type_dataset_sync'
,
'dataset'
,
'to-msg-dataset'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
4
,
0
,
'i18n_msg_type_dataset_sync'
,
'
sys-task-
dataset'
,
'to-msg-dataset'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
5
,
4
,
'i18n_msg_type_dataset_sync_success'
,
'dataset'
,
'to-msg-dataset'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
5
,
4
,
'i18n_msg_type_dataset_sync_success'
,
'
sys-task-
dataset'
,
'to-msg-dataset'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
6
,
4
,
'i18n_msg_type_dataset_sync_faild'
,
'dataset'
,
'to-msg-dataset'
);
INSERT
INTO
`sys_msg_type`
VALUES
(
6
,
4
,
'i18n_msg_type_dataset_sync_faild'
,
'
sys-task-
dataset'
,
'to-msg-dataset'
);
COMMIT
;
COMMIT
;
-- ----------------------------
-- ----------------------------
...
...
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论