Skip to content

D
dataease
提交 3af3e579 authored 作者: taojinlong's avatar taojinlong
浏览文件
操作

feat: 任务管理

上级 3707ca5a
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 416 行增加 和 35 行删除
backend/src/main/java/io/dataease/commons/constants/TaskStatus.java 0 → 100644
浏览文件 @ 3af3e579
package io.dataease.commons.constants;
public enum TaskStatus {
Underway, Stopped
}
backend/src/main/java/io/dataease/commons/constants/TriggerType.java 0 → 100644
浏览文件 @ 3af3e579
package io.dataease.commons.constants;
public enum TriggerType {
Cron, Custom
}
backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java 0 → 100644
浏览文件 @ 3af3e579
package io.dataease.dto.dataset;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.base.domain.DatasetTableTaskLog;
import lombok.Getter;
import lombok.Setter;
/**
* @Author gin
* @Date 2021/3/9 3:19 下午
*/
@Getter
@Setter
public class DataSetTaskDTO extends DatasetTableTask {
private String datasetName;
private Long nextExecTime;
private String taskStatus;
}
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
浏览文件 @ 3af3e579
......@@ -57,7 +57,7 @@ public class DataSetTableTaskService {
public DatasetTableTask save(DataSetTaskRequest dataSetTaskRequest) throws Exception {
checkName(dataSetTaskRequest);
DatasetTableTask datasetTableTask = dataSetTaskRequest.getDatasetTableTask();
if(!datasetTableTask.getType().equalsIgnoreCase("all_scope")){
if(!datasetTableTask.getType().equalsIgnoreCase("add_scope")){
dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig());
}
// check
......@@ -82,14 +82,24 @@ public class DataSetTableTaskService {
datasetTableTask.setId(UUID.randomUUID().toString());
datasetTableTask.setCreateTime(System.currentTimeMillis());
datasetTableTask.setStatus(TaskStatus.Underway.name());
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { // SIMPLE 类型，提前占位
execNow(datasetTableTask);
}
datasetTableTaskMapper.insert(datasetTableTask);
} else {
datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask);
}
scheduleService.addSchedule(datasetTableTask);
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString()) && datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Underway.name())) { // SIMPLE 类型，提前占位
execNow(datasetTableTask);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
datasetTableTask.setLastExecTime(System.currentTimeMillis());
update(datasetTableTask);
}
if(!datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){
scheduleService.addSchedule(datasetTableTask);
}else {
if(datasetTableTask.getStatus().equalsIgnoreCase(JobStatus.Underway.name())){
scheduleService.addSchedule(datasetTableTask);
}
}
return datasetTableTask;
}
......@@ -100,17 +110,17 @@ public class DataSetTableTaskService {
DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync"));
}
}
if (extractDataService.updateSyncStatusIsNone(dataSetTableService.get(datasetTableTask.getTableId()))) {
if (extractDataService.existSyncTask(dataSetTableService.get(datasetTableTask.getTableId()), null)) {
DataEaseException.throwException(Translator.get("i18n_sync_job_exists"));
} else { //write log
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableTask.getTableId());
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
//write log
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableTask.getTableId());
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
public void delete(String id) {
......@@ -195,14 +205,16 @@ public class DataSetTableTaskService {
}
public void execTask(DatasetTableTask datasetTableTask) throws Exception{
execNow(datasetTableTask);
// datasetTableTask.setStatus(TaskStatus.Underway.name());
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
datasetTableTask.setLastExecTime(System.currentTimeMillis());
update(datasetTableTask);
if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())){
scheduleService.fireNow(datasetTableTask);
}
if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){
execNow(datasetTableTask);
datasetTableTask.setStatus(TaskStatus.Underway.name());
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
update(datasetTableTask);
scheduleService.addSchedule(datasetTableTask);
}
......
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @ 3af3e579
......@@ -131,24 +131,42 @@ public class ExtractDataService {
"fi\n" +
"rm -rf %s\n";
public synchronized boolean updateSyncStatusIsNone(DatasetTable datasetTable ){
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask){
datasetTable.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull());
return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0;
Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0;
if(existSyncTask){
if(datasetTableTask != null){
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if(CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())){
return false;
}
}
return true;
}else {
return false;
}
}
public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) {
DatasetTable datasetTable = getDatasetTable(datasetTableId);
if(datasetTable == null){
LogUtil.error("Can not find DatasetTable: " + datasetTableId);
return;
}
DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId);
boolean isCronJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString()));
if(updateSyncStatusIsNone(datasetTable) && isCronJob){
LogUtil.info("Skip synchronization task for table : " + datasetTableId);
if(datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name()) && !datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){
LogUtil.info("Skip synchronization task, task ID : " + datasetTableTask.getId());
return;
}
if(existSyncTask(datasetTable, datasetTableTask)){
LogUtil.info("Skip synchronization task for dataset, dataset ID : " + datasetTableId);
return;
}
datasetTableTask.setLastExecTime(System.currentTimeMillis());
......@@ -210,21 +228,25 @@ public class ExtractDataService {
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
// datasetTableTask.setStatus(TaskStatus.Stopped.name());
// }
datasetTableTask.setLastExecStatus(JobStatus.Completed.name());
dataSetTableTaskService.update(datasetTableTask);
}catch (Exception e){
saveErrorLog(datasetTableId, taskId, e);
datasetTableTask.setLastExecStatus(JobStatus.Error.name());
// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
// datasetTableTask.setStatus(TaskStatus.Stopped.name());
// }
dataSetTableTaskService.update(datasetTableTask);
sendWebMsg(datasetTable, taskId,false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
deleteFile("all_scope", datasetTableId);
}finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
dataSetTableTaskService.update(datasetTableTask);
}
}
break;
......@@ -283,6 +305,9 @@ public class ExtractDataService {
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
datasetTableTask.setLastExecStatus(JobStatus.Completed.name());
// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
// datasetTableTask.setStatus(TaskStatus.Stopped.name());
// }
dataSetTableTaskService.update(datasetTableTask);
}
}catch (Exception e){
......@@ -290,14 +315,13 @@ public class ExtractDataService {
sendWebMsg(datasetTable, taskId,false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
datasetTableTask.setLastExecStatus(JobStatus.Error.name());
// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
// datasetTableTask.setStatus(TaskStatus.Stopped.name());
// }
dataSetTableTaskService.update(datasetTableTask);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
}finally {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
dataSetTableTaskService.update(datasetTableTask);
}
}
break;
}
......
backend/src/main/resources/db/migration/V11__system_task.sql 0 → 100644
浏览文件 @ 3af3e579
INSERT INTO `sys_menu` VALUES (57, 1, 3, 1, '任务管理', 'sys-task', 'system/task/index', 2000, 'task', 'system-task', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL);
INSERT INTO `sys_menu` VALUES (58, 57, 0, 1, '数据集任务', 'sys-task-dataset', 'system/task/dataset', 1, 'task', 'dataset', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL);
ALTER TABLE `dataset_table_task`
ADD COLUMN `last_exec_time` BIGINT(13) NULL DEFAULT NULL COMMENT '上次执行时间' AFTER `create_time`,
ADD COLUMN `status` VARCHAR(50) NULL DEFAULT NULL COMMENT '任务状态' AFTER `last_exec_time`,
ADD COLUMN `last_exec_status` VARCHAR(50) NULL DEFAULT NULL COMMENT '上次执行结果' AFTER `last_exec_time`;
update dataset_table_task set status='Underway' where rate='CRON';
update dataset_table_task set status='Stopped' where rate='SIMPLE_COMPLETE';
ALTER TABLE `dataset_table_task_log` ADD COLUMN `trigger_type` VARCHAR(45) NULL AFTER `create_time`;
frontend/src/icons/svg/task.svg 0 → 100644
浏览文件 @ 3af3e579
<svg t="1625478118793" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1202" width="200" height="200"><path d="M585.142857 804.571429h365.714286v-73.142858H585.142857v73.142858z m-219.428571-292.571429h585.142857v-73.142857H365.714286v73.142857z m365.714285-292.571429h219.428572V146.285714h-219.428572v73.142857z m292.571429 475.428572v146.285714c0 20.004571-16.566857 36.571429-36.571429 36.571429H36.571429c-20.004571 0-36.571429-16.566857-36.571429-36.571429v-146.285714c0-20.004571 16.566857-36.571429 36.571429-36.571429h950.857142c20.004571 0 36.571429 16.566857 36.571429 36.571429z m0-292.571429v146.285715c0 20.004571-16.566857 36.571429-36.571429 36.571428H36.571429c-20.004571 0-36.571429-16.566857-36.571429-36.571428v-146.285715c0-20.004571 16.566857-36.571429 36.571429-36.571428h950.857142c20.004571 0 36.571429 16.566857 36.571429 36.571428z m0-292.571428v146.285714c0 20.004571-16.566857 36.571429-36.571429 36.571429H36.571429c-20.004571 0-36.571429-16.566857-36.571429-36.571429V109.714286c0-20.004571 16.566857-36.571429 36.571429-36.571429h950.857142c20.004571 0 36.571429 16.566857 36.571429 36.571429z" fill="" p-id="1203"></path></svg>
frontend/src/lang/en.js
浏览文件 @ 3af3e579
......@@ -913,7 +913,24 @@ export default {
m2: ' To',
char_can_not_more_50: 'Dataset name can not more 50',
task_add_title: 'Add Task',
task_edit_title: 'Edit Task'
task_edit_title: 'Edit Task',
task: {
list: 'Task list',
record: 'Execution record',
create: 'New task',
name: 'Task name',
last_exec_time: 'Last execution time',
next_exec_time: 'Next execution time',
last_exec_status: 'Last execution result',
task_status: 'Task status',
dataset: 'Data set',
search_by_name: 'Search by name',
underway: 'Running',
stopped: 'Stopped',
exec: 'Execute',
confirm_exec: 'Manual trigger execution？',
change_success: 'State switch successful'
}
},
datasource: {
datasource: 'Data Source',
......
frontend/src/lang/tw.js
浏览文件 @ 3af3e579
......@@ -913,7 +913,24 @@ export default {
m2: ' 移動到',
char_can_not_more_50: '數據集名稱不能超過50個字符',
task_add_title: '添加任務',
task_edit_title: '編輯任務'
task_edit_title: '編輯任務',
task: {
list: '任務列表',
record: '執行紀錄',
create: '新建任務',
name: '任務名稱',
last_exec_time: '上次執行時間',
next_exec_time: '下次執行時間',
last_exec_status: '上次執行結果',
task_status: '任務狀態',
dataset: '數據集',
search_by_name: '根據名稱搜索',
underway: '運行中',
stopped: '停止',
exec: '執行',
confirm_exec: '手動觸發執行？',
change_success: '狀態切換成功'
}
},
datasource: {
datasource: '數據源',
......
frontend/src/lang/zh.js
浏览文件 @ 3af3e579
......@@ -928,7 +928,8 @@ export default {
underway: '运行中',
stopped: '停止',
exec: '执行',
confirm_exec: '手动触发执行？'
confirm_exec: '手动触发执行？',
change_success: '状态切换成功'
}
},
datasource: {
......
frontend/src/views/system/task/TaskRecord.vue 0 → 100644
浏览文件 @ 3af3e579
<template>
<el-col>
<el-row style="margin-top: 10px;">
<complex-table :data="data" :columns="columns" local-key="datasetTaskRecord" :search-config="searchConfig" :pagination-config="paginationConfig" @select="select" @search="search" @sort-change="sortChange">
<el-table-column prop="name" :label="$t('dataset.task_name')"/>
<el-table-column prop="startTime" :label="$t('dataset.start_time')">
<template slot-scope="scope">
<span>{{ scope.row.startTime | timestampFormatDate }}</span>
</template>
</el-table-column>
<el-table-column prop="endTime" :label="$t('dataset.end_time')">
<template slot-scope="scope">
<span>{{ scope.row.endTime | timestampFormatDate }}</span>
</template>
</el-table-column>
<el-table-column prop="status" :label="$t('dataset.status')">
<template slot-scope="scope">
<span v-if="scope.row.status === 'Completed'" style="color: green">{{ $t('dataset.completed') }}</span>
<span v-if="scope.row.status === 'Underway'" style="color: blue">
<i class="el-icon-loading" />
{{ $t('dataset.underway') }}
</span>
<span v-if="scope.row.status === 'Error'" style="color: red">
<el-link type="danger" style="font-size: 12px" @click="showErrorMassage(scope.row.info)">{{ $t('dataset.error') }}</el-link>
</span>
</template>
</el-table-column>
</complex-table>
</el-row>
</el-col>
</template>
<script>
import LayoutContent from '@/components/business/LayoutContent'
import ComplexTable from '@/components/business/complex-table'
import { formatCondition, formatQuickCondition, addOrder, formatOrders } from '@/utils/index'
import '@riophae/vue-treeselect/dist/vue-treeselect.css'
import { post} from '@/api/dataset/dataset'
import cron from '@/components/cron/cron'
import TableSelector from '@/views/chart/view/TableSelector'
export default {
name: 'TaskRecord',
components: { ComplexTable, LayoutContent, cron, TableSelector},
data() {
return {
header: '',
columns: [],
buttons: [
{
label: this.$t('commons.edit'), icon: 'el-icon-edit', type: 'primary', click: this.edit,
show: this.checkPermission(['user:edit'])
}
],
searchConfig: {
useQuickSearch: true,
useComplexSearch: true,
quickPlaceholder: this.$t('dataset.task.search_by_name'),
components: [
{ field: 'dataset_table_task.name', label: this.$t('dataset.task.name'), component: 'DeComplexInput' },
{ field: 'dataset_table_task_log.status', label: this.$t('commons.status'), component: 'FuComplexSelect', options: [{ label: this.$t('dataset.completed'), value: 'Completed' }, { label: this.$t('dataset.underway'), value: 'Underway' }, { label: this.$t('dataset.error'), value: 'Error' }], multiple: false}
]
},
paginationConfig: {
currentPage: 1,
pageSize: 10,
total: 0
},
data: [],
dialogVisible: false,
editPasswordVisible: false,
form: {
roles: [{
id: ''
}]
},
checkPasswordForm: {},
ruleForm: {},
defaultForm: { id: null, username: null, nickName: null, gender: '男', email: null, enabled: 1, deptId: null, phone: null },
depts: null,
roles: [],
roleDatas: [],
userRoles: [],
formType: 'add',
orderConditions: [],
last_condition: null,
}
},
created() {
this.search()
this.timer = setInterval(() => {
this.search()
}, 5000)
},
computed: {
},
beforeDestroy() {
clearInterval(this.timer)
},
methods: {
sortChange({ column, prop, order }) {
this.orderConditions = []
if (!order) {
this.search(this.last_condition)
return
}
if (prop === 'dept') {
prop = 'u.deptId'
}
if (prop === 'status') {
prop = 'u.enabled'
}
this.orderConditions = []
addOrder({ field: prop, value: order }, this.orderConditions)
this.search(this.last_condition)
},
select(selection) {
},
search(condition) {
this.last_condition = condition
console.log(condition)
condition = formatQuickCondition(condition, 'dataset_table_task.name')
const temp = formatCondition(condition)
const param = temp || {}
param['orders'] = formatOrders(this.orderConditions)
post('/dataset/taskLog/list/' + this.paginationConfig.currentPage + '/' + this.paginationConfig.pageSize, param).then(response => {
this.data = response.data.listObject
this.paginationConfig.total = response.data.itemCount
})
},
deleteTask(task) {
this.$confirm(this.$t('dataset.confirm_delete'), this.$t('dataset.tips'), {
confirmButtonText: this.$t('dataset.confirm'),
cancelButtonText: this.$t('dataset.cancel'),
type: 'warning'
}).then(() => {
post('/dataset/task/delete/' + task.id, null).then(response => {
this.$message({
message: this.$t('dataset.delete_success'),
type: 'success',
showClose: true
})
this.search()
})
}).catch(() => {
})
}
}
}
</script>
<style scoped>
.el-divider--horizontal {
margin: 12px 0;
}
.el-radio{
margin-right: 10px;
}
.el-radio>>>.el-radio__label{
font-size: 12px;
}
.dialog-css >>> .el-dialog__header {
padding: 20px 20px 0;
}
.dialog-css >>> .el-dialog__body {
padding: 10px 20px 20px;
}
.el-form-item {
margin-bottom: 10px;
}
span{
font-size: 12px;
}
</style>
frontend/src/views/system/task/dataset.vue 0 → 100644
浏览文件 @ 3af3e579
<template>
<layout-content v-loading="$store.getters.loadingMap[$store.getters.currentPath]">
<el-row style="height: 100%;overflow-y: hidden;width: 100%;">
<el-tabs v-model="tabActive">
<el-tab-pane :label="$t('dataset.task.list')" name="DatasetTaskList">
<dataset-task-list />
</el-tab-pane>
<el-tab-pane :label="$t('dataset.task.record')" name="TaskRecord">
<task-record />
</el-tab-pane>
</el-tabs>
</el-row>
</layout-content>
</template>
<script>
import LayoutContent from '@/components/business/LayoutContent'
import ComplexTable from '@/components/business/complex-table'
import UnionView from "@/views/dataset/data/UnionView";
import UpdateInfo from "@/views/dataset/data/UpdateInfo";
import DatasetTaskList from "@/views/system/task/DatasetTaskList";
import TaskRecord from "@/views/system/task/TaskRecord";
import TabDataPreview from "@/views/dataset/data/TabDataPreview";
import DatasetTableData from "@/views/dataset/common/DatasetTableData";
export default {
components: {DatasetTableData, LayoutContent, ComplexTable,UnionView, UpdateInfo, TabDataPreview, DatasetTaskList, TaskRecord},
data() {
return {
tabActive: 'DatasetTaskList',
}
},
methods: {
}
}
</script>
<style lang="scss" scoped>
.de-msg-radio-class {
padding: 0 5px;
>>>.el-radio-button__inner {
border-radius: 4px 4px 4px 4px !important;
border-left: 1px solid #dcdfe6 !important;
padding: 10px 10px;
}
>>>.el-radio-button__orig-radio:checked+.el-radio-button__inner {
color: #fff;
background-color: #0a7be0;
border-color: #0a7be0;
-webkit-box-shadow: 0px 0 0 0 #0a7be0;
box-shadow: 0px 0 0 0 #0a7be0;
}
}
.de-msg-a:hover {
text-decoration: underline !important;
color: #0a7be0 !important;
cursor: pointer !important;
}
</style>
frontend/src/views/system/task/index.vue 0 → 100644
浏览文件 @ 3af3e579
<template>
<router-view />
</template>
<script>
export default ({
data() {
return {
}
},
created() {
this.$store.dispatch('app/toggleSideBarHide', false)
},
method: {
}
})
</script>
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论