Skip to content

D
dataease
提交 e97297fd authored 作者: TaoJinlong's avatar TaoJinlong
浏览文件
操作

feat: 增量更新数据

上级 5f823af9
显示空白字符变更
内嵌 并排
正在显示 包含 1143 行增加 和 78 行删除
backend/pom.xml
浏览文件 @ e97297fd
...@@ -326,6 +326,11 @@ ...@@ -326,6 +326,11 @@
<artifactId>hbase-common</artifactId> <artifactId>hbase-common</artifactId>
<version>2.4.1</version> <version>2.4.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
<version>2.4.1</version>
</dependency>
<dependency> <dependency>
<groupId>org.testng</groupId> <groupId>org.testng</groupId>
......
backend/src/main/java/io/dataease/base/domain/DatasetTableIncrementalConfig.java 0 → 100644
浏览文件 @ e97297fd
package io.dataease.base.domain;
import java.io.Serializable;
import lombok.Data;
@Data
public class DatasetTableIncrementalConfig implements Serializable {
private String id;
private String tableId;
private String incrementalDelete;
private String incrementalAdd;
private static final long serialVersionUID = 1L;
}
\ No newline at end of file
backend/src/main/java/io/dataease/base/domain/DatasetTableIncrementalConfigExample.java 0 → 100644
浏览文件 @ e97297fd
package io.dataease.base.domain;
import java.util.ArrayList;
import java.util.List;
public class DatasetTableIncrementalConfigExample {
protected String orderByClause;
protected boolean distinct;
protected List<Criteria> oredCriteria;
public DatasetTableIncrementalConfigExample() {
oredCriteria = new ArrayList<Criteria>();
}
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
public String getOrderByClause() {
return orderByClause;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isDistinct() {
return distinct;
}
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria andIdIsNull() {
addCriterion("id is null");
return (Criteria) this;
}
public Criteria andIdIsNotNull() {
addCriterion("id is not null");
return (Criteria) this;
}
public Criteria andIdEqualTo(String value) {
addCriterion("id =", value, "id");
return (Criteria) this;
}
public Criteria andIdNotEqualTo(String value) {
addCriterion("id <>", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThan(String value) {
addCriterion("id >", value, "id");
return (Criteria) this;
}
public Criteria andIdGreaterThanOrEqualTo(String value) {
addCriterion("id >=", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThan(String value) {
addCriterion("id <", value, "id");
return (Criteria) this;
}
public Criteria andIdLessThanOrEqualTo(String value) {
addCriterion("id <=", value, "id");
return (Criteria) this;
}
public Criteria andIdLike(String value) {
addCriterion("id like", value, "id");
return (Criteria) this;
}
public Criteria andIdNotLike(String value) {
addCriterion("id not like", value, "id");
return (Criteria) this;
}
public Criteria andIdIn(List<String> values) {
addCriterion("id in", values, "id");
return (Criteria) this;
}
public Criteria andIdNotIn(List<String> values) {
addCriterion("id not in", values, "id");
return (Criteria) this;
}
public Criteria andIdBetween(String value1, String value2) {
addCriterion("id between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andIdNotBetween(String value1, String value2) {
addCriterion("id not between", value1, value2, "id");
return (Criteria) this;
}
public Criteria andTableIdIsNull() {
addCriterion("table_id is null");
return (Criteria) this;
}
public Criteria andTableIdIsNotNull() {
addCriterion("table_id is not null");
return (Criteria) this;
}
public Criteria andTableIdEqualTo(String value) {
addCriterion("table_id =", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotEqualTo(String value) {
addCriterion("table_id <>", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdGreaterThan(String value) {
addCriterion("table_id >", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdGreaterThanOrEqualTo(String value) {
addCriterion("table_id >=", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLessThan(String value) {
addCriterion("table_id <", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLessThanOrEqualTo(String value) {
addCriterion("table_id <=", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdLike(String value) {
addCriterion("table_id like", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotLike(String value) {
addCriterion("table_id not like", value, "tableId");
return (Criteria) this;
}
public Criteria andTableIdIn(List<String> values) {
addCriterion("table_id in", values, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotIn(List<String> values) {
addCriterion("table_id not in", values, "tableId");
return (Criteria) this;
}
public Criteria andTableIdBetween(String value1, String value2) {
addCriterion("table_id between", value1, value2, "tableId");
return (Criteria) this;
}
public Criteria andTableIdNotBetween(String value1, String value2) {
addCriterion("table_id not between", value1, value2, "tableId");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIsNull() {
addCriterion("incremental_delete is null");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIsNotNull() {
addCriterion("incremental_delete is not null");
return (Criteria) this;
}
public Criteria andIncrementalDeleteEqualTo(String value) {
addCriterion("incremental_delete =", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotEqualTo(String value) {
addCriterion("incremental_delete <>", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteGreaterThan(String value) {
addCriterion("incremental_delete >", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteGreaterThanOrEqualTo(String value) {
addCriterion("incremental_delete >=", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLessThan(String value) {
addCriterion("incremental_delete <", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLessThanOrEqualTo(String value) {
addCriterion("incremental_delete <=", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteLike(String value) {
addCriterion("incremental_delete like", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotLike(String value) {
addCriterion("incremental_delete not like", value, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteIn(List<String> values) {
addCriterion("incremental_delete in", values, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotIn(List<String> values) {
addCriterion("incremental_delete not in", values, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteBetween(String value1, String value2) {
addCriterion("incremental_delete between", value1, value2, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalDeleteNotBetween(String value1, String value2) {
addCriterion("incremental_delete not between", value1, value2, "incrementalDelete");
return (Criteria) this;
}
public Criteria andIncrementalAddIsNull() {
addCriterion("incremental_add is null");
return (Criteria) this;
}
public Criteria andIncrementalAddIsNotNull() {
addCriterion("incremental_add is not null");
return (Criteria) this;
}
public Criteria andIncrementalAddEqualTo(String value) {
addCriterion("incremental_add =", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotEqualTo(String value) {
addCriterion("incremental_add <>", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddGreaterThan(String value) {
addCriterion("incremental_add >", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddGreaterThanOrEqualTo(String value) {
addCriterion("incremental_add >=", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLessThan(String value) {
addCriterion("incremental_add <", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLessThanOrEqualTo(String value) {
addCriterion("incremental_add <=", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddLike(String value) {
addCriterion("incremental_add like", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotLike(String value) {
addCriterion("incremental_add not like", value, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddIn(List<String> values) {
addCriterion("incremental_add in", values, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotIn(List<String> values) {
addCriterion("incremental_add not in", values, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddBetween(String value1, String value2) {
addCriterion("incremental_add between", value1, value2, "incrementalAdd");
return (Criteria) this;
}
public Criteria andIncrementalAddNotBetween(String value1, String value2) {
addCriterion("incremental_add not between", value1, value2, "incrementalAdd");
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
}
}
\ No newline at end of file
backend/src/main/java/io/dataease/base/mapper/DatasetTableIncrementalConfigMapper.java 0 → 100644
浏览文件 @ e97297fd
package io.dataease.base.mapper;
import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.base.domain.DatasetTableIncrementalConfigExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DatasetTableIncrementalConfigMapper {
long countByExample(DatasetTableIncrementalConfigExample example);
int deleteByExample(DatasetTableIncrementalConfigExample example);
int insert(DatasetTableIncrementalConfig record);
int insertSelective(DatasetTableIncrementalConfig record);
List<DatasetTableIncrementalConfig> selectByExample(DatasetTableIncrementalConfigExample example);
int updateByExampleSelective(@Param("record") DatasetTableIncrementalConfig record, @Param("example") DatasetTableIncrementalConfigExample example);
int updateByExample(@Param("record") DatasetTableIncrementalConfig record, @Param("example") DatasetTableIncrementalConfigExample example);
}
\ No newline at end of file
backend/src/main/java/io/dataease/base/mapper/DatasetTableIncrementalConfigMapper.xml 0 → 100644
浏览文件 @ e97297fd
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="io.dataease.base.mapper.DatasetTableIncrementalConfigMapper">
<resultMap id="BaseResultMap" type="io.dataease.base.domain.DatasetTableIncrementalConfig">
<result column="id" jdbcType="VARCHAR" property="id" />
<result column="table_id" jdbcType="VARCHAR" property="tableId" />
<result column="incremental_delete" jdbcType="VARCHAR" property="incrementalDelete" />
<result column="incremental_add" jdbcType="VARCHAR" property="incrementalAdd" />
</resultMap>
<sql id="Example_Where_Clause">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Update_By_Example_Where_Clause">
<where>
<foreach collection="example.oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
<sql id="Base_Column_List">
id, table_id, incremental_delete, incremental_add
</sql>
<select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
<include refid="Base_Column_List" />
from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<delete id="deleteByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample">
delete from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfig">
insert into dataset_table_incremental_config (id, table_id, incremental_delete,
incremental_add)
values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{incrementalDelete,jdbcType=VARCHAR},
#{incrementalAdd,jdbcType=VARCHAR})
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfig">
insert into dataset_table_incremental_config
<trim prefix="(" suffix=")" suffixOverrides=",">
<if test="id != null">
id,
</if>
<if test="tableId != null">
table_id,
</if>
<if test="incrementalDelete != null">
incremental_delete,
</if>
<if test="incrementalAdd != null">
incremental_add,
</if>
</trim>
<trim prefix="values (" suffix=")" suffixOverrides=",">
<if test="id != null">
#{id,jdbcType=VARCHAR},
</if>
<if test="tableId != null">
#{tableId,jdbcType=VARCHAR},
</if>
<if test="incrementalDelete != null">
#{incrementalDelete,jdbcType=VARCHAR},
</if>
<if test="incrementalAdd != null">
#{incrementalAdd,jdbcType=VARCHAR},
</if>
</trim>
</insert>
<select id="countByExample" parameterType="io.dataease.base.domain.DatasetTableIncrementalConfigExample" resultType="java.lang.Long">
select count(*) from dataset_table_incremental_config
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
</select>
<update id="updateByExampleSelective" parameterType="map">
update dataset_table_incremental_config
<set>
<if test="record.id != null">
id = #{record.id,jdbcType=VARCHAR},
</if>
<if test="record.tableId != null">
table_id = #{record.tableId,jdbcType=VARCHAR},
</if>
<if test="record.incrementalDelete != null">
incremental_delete = #{record.incrementalDelete,jdbcType=VARCHAR},
</if>
<if test="record.incrementalAdd != null">
incremental_add = #{record.incrementalAdd,jdbcType=VARCHAR},
</if>
</set>
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
<update id="updateByExample" parameterType="map">
update dataset_table_incremental_config
set id = #{record.id,jdbcType=VARCHAR},
table_id = #{record.tableId,jdbcType=VARCHAR},
incremental_delete = #{record.incrementalDelete,jdbcType=VARCHAR},
incremental_add = #{record.incrementalAdd,jdbcType=VARCHAR}
<if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" />
</if>
</update>
</mapper>
\ No newline at end of file
backend/src/main/java/io/dataease/commons/constants/ScheduleType.java
浏览文件 @ e97297fd
package io.dataease.commons.constants; package io.dataease.commons.constants;
public enum ScheduleType { public enum ScheduleType {
CRON, SIMPLE CRON, SIMPLE, SIMPLE_COMPLETE
} }
backend/src/main/java/io/dataease/commons/constants/UpdateType.java 0 → 100644
浏览文件 @ e97297fd
package io.dataease.commons.constants;
public enum UpdateType {
all_scope, add_scope
}
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
浏览文件 @ e97297fd
...@@ -2,6 +2,7 @@ package io.dataease.controller.dataset; ...@@ -2,6 +2,7 @@ package io.dataease.controller.dataset;
import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.controller.request.dataset.DataSetTableRequest; import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.dto.TableFiled;
import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableService;
...@@ -70,4 +71,15 @@ public class DataSetTableController { ...@@ -70,4 +71,15 @@ public class DataSetTableController {
public Map<String, Object> getSQLPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { public Map<String, Object> getSQLPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception {
return dataSetTableService.getSQLPreview(dataSetTableRequest); return dataSetTableService.getSQLPreview(dataSetTableRequest);
} }
@PostMapping("incrementalConfig")
public DatasetTableIncrementalConfig incrementalConfig(@RequestBody DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
return dataSetTableService.incrementalConfig(datasetTableIncrementalConfig);
}
@PostMapping("save/incrementalConfig")
public void saveIncrementalConfig(@RequestBody DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
dataSetTableService.saveIncrementalConfig(datasetTableIncrementalConfig);
}
} }
backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
浏览文件 @ e97297fd
...@@ -8,6 +8,7 @@ public abstract class DeScheduleJob implements Job { ...@@ -8,6 +8,7 @@ public abstract class DeScheduleJob implements Job {
protected String datasetTableId; protected String datasetTableId;
protected String expression; protected String expression;
protected String taskId; protected String taskId;
protected String updateType;
@Override @Override
public void execute(JobExecutionContext context) throws JobExecutionException { public void execute(JobExecutionContext context) throws JobExecutionException {
...@@ -16,6 +17,7 @@ public abstract class DeScheduleJob implements Job { ...@@ -16,6 +17,7 @@ public abstract class DeScheduleJob implements Job {
this.datasetTableId = jobDataMap.getString("datasetTableId"); this.datasetTableId = jobDataMap.getString("datasetTableId");
this.expression = jobDataMap.getString("expression"); this.expression = jobDataMap.getString("expression");
this.taskId = jobDataMap.getString("taskId"); this.taskId = jobDataMap.getString("taskId");
this.updateType = jobDataMap.getString("updateType");
LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId); LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId);
LogUtil.info("CronExpression: " + expression); LogUtil.info("CronExpression: " + expression);
......
backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java
浏览文件 @ e97297fd
...@@ -16,7 +16,7 @@ public class ExtractDataJob extends DeScheduleJob{ ...@@ -16,7 +16,7 @@ public class ExtractDataJob extends DeScheduleJob{
@Override @Override
void businessExecute(JobExecutionContext context) { void businessExecute(JobExecutionContext context) {
extractDataService.extractData(datasetTableId, taskId); extractDataService.extractData(datasetTableId, taskId, updateType);
} }
} }
backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
浏览文件 @ e97297fd
...@@ -369,11 +369,12 @@ public class ScheduleManager { ...@@ -369,11 +369,12 @@ public class ScheduleManager {
addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null); addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null);
} }
public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId) { public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId, String updateType) {
JobDataMap jobDataMap = new JobDataMap(); JobDataMap jobDataMap = new JobDataMap();
jobDataMap.put("datasetTableId", resourceId); jobDataMap.put("datasetTableId", resourceId);
jobDataMap.put("taskId", taskId); jobDataMap.put("taskId", taskId);
jobDataMap.put("expression", expression); jobDataMap.put("expression", expression);
jobDataMap.put("updateType", updateType);
return jobDataMap; return jobDataMap;
} }
......
backend/src/main/java/io/dataease/service/ScheduleService.java
浏览文件 @ e97297fd
package io.dataease.service; package io.dataease.service;
import io.dataease.base.domain.DatasetTableTask; import io.dataease.base.domain.DatasetTableTask;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.job.sechedule.ExtractDataJob; import io.dataease.job.sechedule.ExtractDataJob;
import io.dataease.job.sechedule.ScheduleManager; import io.dataease.job.sechedule.ScheduleManager;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
...@@ -21,12 +22,13 @@ public class ScheduleService { ...@@ -21,12 +22,13 @@ public class ScheduleService {
private ScheduleManager scheduleManager; private ScheduleManager scheduleManager;
public void addSchedule(DatasetTableTask datasetTableTask) throws Exception { public void addSchedule(DatasetTableTask datasetTableTask) throws Exception {
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "0")) { if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), ScheduleType.SIMPLE.toString())) {
scheduleManager.addOrUpdateSingleJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()), scheduleManager.addOrUpdateSingleJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()), new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
ExtractDataJob.class, ExtractDataJob.class,
new Date(datasetTableTask.getStartTime()), scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId())); new Date(datasetTableTask.getStartTime()),
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "1")) { scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId(), datasetTableTask.getType()));
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), ScheduleType.CRON.toString())) {
Date endTime; Date endTime;
if (datasetTableTask.getEndTime() == null || datasetTableTask.getEndTime() == 0) { if (datasetTableTask.getEndTime() == null || datasetTableTask.getEndTime() == 0) {
endTime = null; endTime = null;
...@@ -38,7 +40,7 @@ public class ScheduleService { ...@@ -38,7 +40,7 @@ public class ScheduleService {
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()), new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
ExtractDataJob.class, ExtractDataJob.class,
datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime, datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime,
scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId())); scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId(), datasetTableTask.getType()));
} }
} }
......
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
浏览文件 @ e97297fd
...@@ -2,10 +2,8 @@ package io.dataease.service.dataset; ...@@ -2,10 +2,8 @@ package io.dataease.service.dataset;
import com.google.gson.Gson; import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.*;
import io.dataease.base.domain.DatasetTableExample; import io.dataease.base.mapper.DatasetTableIncrementalConfigMapper;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasourceMapper; import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.BeanUtils;
...@@ -40,6 +38,8 @@ public class DataSetTableService { ...@@ -40,6 +38,8 @@ public class DataSetTableService {
private DataSetTableFieldsService dataSetTableFieldsService; private DataSetTableFieldsService dataSetTableFieldsService;
@Resource @Resource
private DataSetTableTaskService dataSetTableTaskService; private DataSetTableTaskService dataSetTableTaskService;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
public void batchInsert(List<DatasetTable> datasetTable) throws Exception { public void batchInsert(List<DatasetTable> datasetTable) throws Exception {
for (DatasetTable table : datasetTable) { for (DatasetTable table : datasetTable) {
...@@ -261,6 +261,20 @@ public class DataSetTableService { ...@@ -261,6 +261,20 @@ public class DataSetTableService {
return data; return data;
} }
public List<String[]> getDataSetDataBySql(String datasourceId, String table, String sql) {
List<String[]> data = new ArrayList<>();
Datasource ds = datasourceMapper.selectByPrimaryKey(datasourceId);
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery(sql);
try {
return datasourceProvider.getData(datasourceRequest);
} catch (Exception e) {
}
return data;
}
public void saveTableField(DatasetTable datasetTable) throws Exception { public void saveTableField(DatasetTable datasetTable) throws Exception {
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DataSetTableRequest dataSetTableRequest = new DataSetTableRequest(); DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
...@@ -349,4 +363,35 @@ public class DataSetTableService { ...@@ -349,4 +363,35 @@ public class DataSetTableService {
return 0; return 0;
} }
} }
public DatasetTableIncrementalConfig incrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig){
if(StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())){return new DatasetTableIncrementalConfig();}
DatasetTableIncrementalConfigExample example = new DatasetTableIncrementalConfigExample();
example.createCriteria().andTableIdEqualTo(datasetTableIncrementalConfig.getTableId());
List<DatasetTableIncrementalConfig> configs = datasetTableIncrementalConfigMapper.selectByExample(example);
if(CollectionUtils.isNotEmpty(configs)){
return configs.get(0);
}else {
return new DatasetTableIncrementalConfig();
}
}
public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId){
DatasetTableIncrementalConfig datasetTableIncrementalConfig = new DatasetTableIncrementalConfig();
datasetTableIncrementalConfig.setTableId(datasetTableId);
return incrementalConfig(datasetTableIncrementalConfig);
}
public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig){
if(StringUtils.isEmpty(datasetTableIncrementalConfig.getId())){
datasetTableIncrementalConfig.setId(UUID.randomUUID().toString());
datasetTableIncrementalConfigMapper.insertSelective(datasetTableIncrementalConfig);
}else{
DatasetTableIncrementalConfigExample example = new DatasetTableIncrementalConfigExample();
example.createCriteria().andTableIdEqualTo(datasetTableIncrementalConfig.getTableId());
datasetTableIncrementalConfigMapper.updateByExample(datasetTableIncrementalConfig, example);
}
}
} }
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
浏览文件 @ e97297fd
...@@ -70,6 +70,10 @@ public class DataSetTableTaskService { ...@@ -70,6 +70,10 @@ public class DataSetTableTaskService {
return datasetTableTaskMapper.selectByPrimaryKey(id); return datasetTableTaskMapper.selectByPrimaryKey(id);
} }
public void update(DatasetTableTask datasetTableTask) {
datasetTableTaskMapper.updateByPrimaryKey(datasetTableTask);
}
public List<DatasetTableTask> list(DatasetTableTask datasetTableTask) { public List<DatasetTableTask> list(DatasetTableTask datasetTableTask) {
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria(); DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();
......
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
浏览文件 @ e97297fd
package io.dataease.service.dataset; package io.dataease.service.dataset;
import com.google.gson.Gson; import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.*;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableTaskLog;
import io.dataease.commons.constants.JobStatus; import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.commons.constants.UpdateType;
import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil; import io.dataease.commons.utils.LogUtil;
import io.dataease.dto.dataset.DataSetTaskLogDTO;
import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.dto.dataset.DataTableInfoDTO;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.*;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.security.MessageDigest;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
...@@ -28,30 +32,108 @@ public class ExtractDataService { ...@@ -28,30 +32,108 @@ public class ExtractDataService {
private DataSetTableFieldsService dataSetTableFieldsService; private DataSetTableFieldsService dataSetTableFieldsService;
@Resource @Resource
private DataSetTableTaskLogService dataSetTableTaskLogService; private DataSetTableTaskLogService dataSetTableTaskLogService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
private Long pageSize = 10000l; private Long pageSize = 10000l;
private static ExecutorService pool = Executors.newScheduledThreadPool(50); //设置连接池 private static ExecutorService pool = Executors.newScheduledThreadPool(50); //设置连接池
private Connection connection; private Connection connection;
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
private static String column_family = "dataease";
public void extractData(String datasetTableId, String taskId) { public void extractData(String datasetTableId, String taskId, String type) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
UpdateType updateType = UpdateType.valueOf(type);
try { try {
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
Admin admin = getConnection().getAdmin(); Admin admin = getConnection().getAdmin();
DatasetTable datasetTable = dataSetTableService.get(datasetTableId); DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build()); List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
TableName tableName = TableName.valueOf(table + "-" + datasetTable.getDataSourceId()); TableName tableName = TableName.valueOf(table + "-" + datasetTable.getDataSourceId());
switch (updateType){
// 全量更新
case all_scope:
writeDatasetTableTaskLog(datasetTableTaskLog,datasetTableId, taskId);
if(!admin.tableExists(tableName)){
creatHaseTable(tableName, admin);
}
extractAllData(admin, tableName, table, datasetTable, datasetTableFields);
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
break;
case add_scope:
// 增量更新
if(!admin.tableExists(tableName)){ if(!admin.tableExists(tableName)){
LogUtil.error("TableName error, dataaset: " + datasetTableId);
return;
}
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if(datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())){
return;
}
DatasetTableTaskLog request = new DatasetTableTaskLog();
request.setTableId(datasetTableId);
request.setStatus(JobStatus.Completed.name());
List<DataSetTaskLogDTO> dataSetTaskLogDTOS = dataSetTableTaskLogService.list(request);
if(CollectionUtils.isEmpty(dataSetTaskLogDTOS)){
return;
}
writeDatasetTableTaskLog(datasetTableTaskLog,datasetTableId, taskId);
// 增量添加
if(StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd())){
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
extractIncrementalData(tableName,table,datasetTable, datasetTableFields, sql, "add");
}
// 增量删除
if( StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())){
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
extractIncrementalData(tableName,table,datasetTable, datasetTableFields, sql, "delete");
}
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
break;
}
}catch (Exception e){
e.printStackTrace();
LogUtil.error("ExtractData error, dataaset: " + datasetTableId);
LogUtil.error(e.getMessage(), e);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
finally {
DatasetTableTask datasetTableTask = dataSetTableTaskService.get(taskId);
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){
datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
dataSetTableTaskService.update(datasetTableTask);
}
}
}
private void writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId){
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void creatHaseTable(TableName tableName, Admin admin)throws Exception{
TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(tableName); TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(column_family);
descBuilder.setColumnFamily(hcd); descBuilder.setColumnFamily(hcd);
TableDescriptor desc = descBuilder.build(); TableDescriptor desc = descBuilder.build();
admin.createTable(desc); admin.createTable(desc);
} }
private void extractAllData(Admin admin, TableName tableName, String table, DatasetTable datasetTable, List<DatasetTableField> datasetTableFields)throws Exception{
admin.disableTable(tableName); admin.disableTable(tableName);
admin.truncateTable(tableName, true); admin.truncateTable(tableName, true);
...@@ -61,32 +143,41 @@ public class ExtractDataService { ...@@ -61,32 +143,41 @@ public class ExtractDataService {
for (Long pageIndex = 1l; pageIndex <= pageCount; pageIndex++) { for (Long pageIndex = 1l; pageIndex <= pageCount; pageIndex++) {
List<String[]> data = dataSetTableService.getDataSetPageData(datasetTable.getDataSourceId(), table, datasetTableFields, pageIndex, pageSize); List<String[]> data = dataSetTableService.getDataSetPageData(datasetTable.getDataSourceId(), table, datasetTableFields, pageIndex, pageSize);
insertDataToHbaseTable(data,datasetTableFields,tab);
}
}
private void extractIncrementalData(TableName tableName, String table, DatasetTable datasetTable, List<DatasetTableField> datasetTableFields, String sql, String type)throws Exception{
Table tab = getConnection().getTable(tableName);
List<String[]> data = dataSetTableService.getDataSetDataBySql(datasetTable.getDataSourceId(), table, sql);
if (type.equalsIgnoreCase("add")){
insertDataToHbaseTable(data,datasetTableFields,tab);
}else {
deleteDataFromHbaseTable(data,datasetTableFields,tab);
}
}
private void insertDataToHbaseTable(List<String[]> data, List<DatasetTableField> datasetTableFields, Table tab)throws Exception{
for (String[] d : data) { for (String[] d : data) {
Put put = new Put(md5(generateStr(datasetTableFields.size(), d)).getBytes());
for(int i=0;i<datasetTableFields.size();i++){ for(int i=0;i<datasetTableFields.size();i++){
Put put = new Put(UUID.randomUUID().toString().getBytes());
String value = d[i]; String value = d[i];
if(value == null){ if(value == null){
value = "null"; value = "null";
} }
put.addColumn("cf".getBytes(), datasetTableFields.get(i).getOriginName().getBytes(), value.getBytes()); put.addColumn(column_family.getBytes(), datasetTableFields.get(i).getOriginName().getBytes(), value.getBytes());
tab.put(put);
} }
tab.put(put);
} }
} }
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis()); private void deleteDataFromHbaseTable(List<String[]> data, List<DatasetTableField> datasetTableFields, Table tab)throws Exception{
dataSetTableTaskLogService.save(datasetTableTaskLog); for (String[] d : data) {
}catch (Exception e){ Delete delete = new Delete(md5(generateStr(datasetTableFields.size(), d)).getBytes());
e.printStackTrace(); tab.delete(delete);
LogUtil.error("ExtractData error, dataaset: " + datasetTableId);
LogUtil.error(e.getMessage(), e);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
} }
} }
private synchronized Connection getConnection() throws Exception{ private synchronized Connection getConnection() throws Exception{
if(connection == null || connection.isClosed()){ if(connection == null || connection.isClosed()){
Configuration cfg = CommonBeanFactory.getBean(Configuration.class); Configuration cfg = CommonBeanFactory.getBean(Configuration.class);
...@@ -94,4 +185,43 @@ public class ExtractDataService { ...@@ -94,4 +185,43 @@ public class ExtractDataService {
} }
return connection; return connection;
} }
private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
private static final String UTF_8 = "UTF-8";
public static String md5(String src) {
return md5(src, UTF_8);
}
public static String md5(String src, String charset) {
try {
byte[] strTemp = io.micrometer.core.instrument.util.StringUtils.isEmpty(charset) ? src.getBytes() : src.getBytes(charset);
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(strTemp);
byte[] md = mdTemp.digest();
int j = md.length;
char[] str = new char[j * 2];
int k = 0;
for (byte byte0 : md) {
str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
str[k++] = HEX_DIGITS[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
throw new RuntimeException("MD5 encrypt error:", e);
}
}
public String generateStr(int size, String[] d ){
String str = null;
for(int i=0;i<size;i++){
str = str + d[i];
}
return str;
}
} }
backend/src/main/resources/db/migration/V13__dataset_tables_incre.sql 0 → 100644
浏览文件 @ e97297fd
DROP TABLE IF EXISTS `dataset_table_incremental_config`;
CREATE TABLE IF NOT EXISTS `dataset_table_incremental_config`
(
`id` varchar(50) NOT NULL COMMENT 'ID',
`table_id` varchar(50) NOT NULL COMMENT '表ID',
`incremental_delete` longtext COMMENT '详细信息',
`incremental_add` longtext COMMENT '详细信息',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4;
\ No newline at end of file
backend/src/main/resources/generatorConfig.xml
浏览文件 @ e97297fd
...@@ -37,11 +37,11 @@ ...@@ -37,11 +37,11 @@
</commentGenerator> </commentGenerator>
<!-- jdbc连接信息 --> <!-- EduLoanManage EduTestDataBase --> <!-- jdbc连接信息 --> <!-- EduLoanManage EduTestDataBase -->
<!--<jdbcConnection driverClass="com.mysql.jdbc.Driver" connectionURL="jdbc:mysql://192.168.20.180:3306/fit2cloud"--> <jdbcConnection driverClass="com.mysql.jdbc.Driver" connectionURL="jdbc:mysql://62.234.205.170:3306/dataease"
<!--userId="root" password="Fit2cloud2015!" />--> userId="root" password="Password123@mysql" />
<jdbcConnection driverClass="com.mysql.cj.jdbc.Driver" <!-- <jdbcConnection driverClass="com.mysql.cj.jdbc.Driver"-->
connectionURL="${spring.datasource.url}&amp;nullCatalogMeansCurrent=true" <!-- connectionURL="${spring.datasource.url}&amp;nullCatalogMeansCurrent=true"-->
userId="${spring.datasource.username}" password="${spring.datasource.password}"/> <!-- userId="${spring.datasource.username}" password="${spring.datasource.password}"/>-->
<!-- javaTypeResolver式类型转换的信息 --> <!-- javaTypeResolver式类型转换的信息 -->
<javaTypeResolver> <javaTypeResolver>
...@@ -64,8 +64,13 @@ ...@@ -64,8 +64,13 @@
<!--要生成的数据库表 --> <!--要生成的数据库表 -->
<!-- <table tableName="datasource"/>--> <table tableName="dataset_table_incremental_config" >
<table tableName="panel_group"/>
<!--以下为添加内容 -->
<columnOverride column="incremental_delete" javaType="java.lang.String" jdbcType="VARCHAR" />
<columnOverride column="incremental_add" javaType="java.lang.String" jdbcType="VARCHAR" />
</table>
</context> </context>
......
frontend/src/lang/zh.js
浏览文件 @ e97297fd
...@@ -671,7 +671,12 @@ export default { ...@@ -671,7 +671,12 @@ export default {
add_sql_table: '添加SQL', add_sql_table: '添加SQL',
preview: '预览', preview: '预览',
pls_input_name: '请输入名称', pls_input_name: '请输入名称',
connect_mode: '连接模式' connect_mode: '连接模式',
incremental_update_type: '增量更新方式:',
incremental_add: '增量添加:',
incremental_delete: '增量删除:',
last_update_time: '上次更新时间:',
current_update_time: '当前更新时间:'
}, },
datasource: { datasource: {
create: '新建数据连接', create: '新建数据连接',
...@@ -689,6 +694,7 @@ export default { ...@@ -689,6 +694,7 @@ export default {
please_input_port: '请输入端口', please_input_port: '请输入端口',
modify: '编辑数据连接', modify: '编辑数据连接',
validate_success: '校验成功', validate_success: '校验成功',
validate: '校验',
delete: '删除组织', delete: '删除组织',
delete_confirm: '删除该组织会关联删除该组织下的所有资源(如：相关工作空间，项目，测试用例等），确定要删除吗?', delete_confirm: '删除该组织会关联删除该组织下的所有资源(如：相关工作空间，项目，测试用例等），确定要删除吗?',
input_name: '请输入名称', input_name: '请输入名称',
......
frontend/src/views/dataset/data/UpdateInfo.vue
浏览文件 @ e97297fd
...@@ -79,12 +79,11 @@ ...@@ -79,12 +79,11 @@
<el-select v-model="taskForm.type" size="mini"> <el-select v-model="taskForm.type" size="mini">
<el-option <el-option
:label="$t('dataset.all_scope')" :label="$t('dataset.all_scope')"
value="0" value="all_scope"
/> />
<el-option <el-option
:label="$t('dataset.add_scope')" :label="$t('dataset.add_scope')"
value="1" value="add_scope"
:disabled="true"
/> />
</el-select> </el-select>
</el-form-item> </el-form-item>
...@@ -97,22 +96,22 @@ ...@@ -97,22 +96,22 @@
/> />
</el-form-item> </el-form-item>
<el-form-item :label="$t('dataset.execute_rate')" prop="rate"> <el-form-item :label="$t('dataset.execute_rate')" prop="rate">
<el-select v-model="taskForm.rate" size="mini" @change="onRateChange"> <el-select v-model="taskForm.rate" @change="onRateChange">
<el-option <el-option
:label="$t('dataset.execute_once')" :label="$t('dataset.execute_once')"
value="0" value="SIMPLE"
/> />
<el-option <el-option
:label="$t('dataset.cron_config')" :label="$t('dataset.cron_config')"
value="1" value="CRON"
/> />
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item v-if="taskForm.rate === '1'" label=""> <el-form-item v-if="taskForm.rate === 'CRON'" label="">
<el-input v-model="taskForm.cron" size="mini" style="width: 50%" /> <el-input v-model="taskForm.cron" size="mini" style="width: 50%" />
</el-form-item> </el-form-item>
<el-form-item :label="$t('dataset.end_time')" prop="end"> <el-form-item :label="$t('dataset.end_time')" prop="end">
<el-select v-model="taskForm.end" size="mini" :disabled="taskForm.rate === '0'"> <el-select v-model="taskForm.end" size="mini">
<el-option <el-option
:label="$t('dataset.no_limit')" :label="$t('dataset.no_limit')"
value="0" value="0"
...@@ -163,8 +162,9 @@ ...@@ -163,8 +162,9 @@
:label="$t('dataset.execute_rate')" :label="$t('dataset.execute_rate')"
> >
<template slot-scope="scope"> <template slot-scope="scope">
<span v-if="scope.row.rate === '0'">{{ $t('dataset.execute_once') }}</span> <span v-if="scope.row.rate === 'SIMPLE'">{{ $t('dataset.execute_once') }}</span>
<span v-if="scope.row.rate === '1'">{{ $t('dataset.cron_config') }}</span> <span v-if="scope.row.rate === 'SIMPLE_COMPLETE'">{{ $t('dataset.execute_once') }}</span>
<span v-if="scope.row.rate === 'CRON'">{{ $t('dataset.cron_config') }}</span>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column <el-table-column
...@@ -176,6 +176,7 @@ ...@@ -176,6 +176,7 @@
type="primary" type="primary"
icon="el-icon-edit" icon="el-icon-edit"
circle circle
:disabled="scope.row.rate === 'SIMPLE_COMPLETE'"
@click="addTask(scope.row)" @click="addTask(scope.row)"
/> />
<el-button <el-button
...@@ -190,10 +191,49 @@ ...@@ -190,10 +191,49 @@
</el-table> </el-table>
</el-row> </el-row>
<el-divider />
<el-row style="height: 26px;">
<el-row>
<el-col :span="6"><div>{{ $t('dataset.incremental_update_type') }}</div></el-col>
<el-col :span="18">
<el-radio-group v-model="incrementalUpdateType" size="small" @change="incrementalUpdateTypeChange">
<el-radio label="incrementalAdd" >{{ $t('dataset.incremental_add') }}</el-radio>
<el-radio label="incrementalDelete" >{{ $t('incremental_delete.incremental_update_type') }}</el-radio>
</el-radio-group>
</el-col>
</el-row>
</el-row>
<el-row style="height: 26px;">
<el-row>
<el-col :span="6" style="height: 26px;"><div style="height: 26px;">参数:</div></el-col>
<el-col :span="18">
<el-button type="text">{{ $t('dataset.last_update_time') }}</el-button>
<el-button type="text">{{ $t('dataset.current_update_time') }}</el-button>
</el-col>
</el-row>
</el-row>
<el-row>
<el-col style="min-width: 200px;">
<codemirror
ref="myCm"
v-model="sql"
class="codemirror"
:options="sqlOption"
@ready="onCmReady"
@focus="onCmFocus"
@input="onCmCodeChange"
/>
</el-col>
</el-row>
<div slot="footer" class="dialog-footer"> <div slot="footer" class="dialog-footer">
<!-- <el-button size="mini" @click="update_setting = false">{{ $t('dataset.cancel') }}</el-button>-->
<!-- <el-button type="primary" size="mini" @click="update_setting = false">{{ $t('dataset.confirm') }}</el-button>-->
<el-button size="mini" @click="update_setting = false">{{ $t('dataset.close') }}</el-button> <el-button size="mini" @click="update_setting = false">{{ $t('dataset.close') }}</el-button>
<el-button size="mini" @click="saveIncrementalConfig">{{ $t('dataset.confirm') }}</el-button>
</div> </div>
</el-dialog> </el-dialog>
</el-col> </el-col>
...@@ -201,9 +241,33 @@ ...@@ -201,9 +241,33 @@
<script> <script>
import { post } from '@/api/dataset/dataset' import { post } from '@/api/dataset/dataset'
import { codemirror } from 'vue-codemirror'
// 核心样式
import 'codemirror/lib/codemirror.css'
// 引入主题后还需要在 options 中指定主题才会生效
import 'codemirror/theme/solarized.css'
import 'codemirror/mode/sql/sql.js'
// require active-line.js
import 'codemirror/addon/selection/active-line.js'
// closebrackets
import 'codemirror/addon/edit/closebrackets.js'
// keyMap
import 'codemirror/mode/clike/clike.js'
import 'codemirror/addon/edit/matchbrackets.js'
import 'codemirror/addon/comment/comment.js'
import 'codemirror/addon/dialog/dialog.js'
import 'codemirror/addon/dialog/dialog.css'
import 'codemirror/addon/search/searchcursor.js'
import 'codemirror/addon/search/search.js'
import 'codemirror/keymap/emacs.js'
// 引入代码自动提示插件
import 'codemirror/addon/hint/show-hint.css'
import 'codemirror/addon/hint/sql-hint'
import 'codemirror/addon/hint/show-hint'
export default { export default {
name: 'UpdateInfo', name: 'UpdateInfo',
components: { codemirror },
props: { props: {
table: { table: {
type: Object, type: Object,
...@@ -216,9 +280,9 @@ export default { ...@@ -216,9 +280,9 @@ export default {
update_task: false, update_task: false,
taskForm: { taskForm: {
name: '', name: '',
type: '0', type: 'all_scope',
startTime: '', startTime: '',
rate: '0', rate: 'SIMPLE',
cron: '', cron: '',
endTime: '', endTime: '',
end: '0' end: '0'
...@@ -246,7 +310,21 @@ export default { ...@@ -246,7 +310,21 @@ export default {
end: [ end: [
{ required: true, message: this.$t('dataset.required'), trigger: 'change' } { required: true, message: this.$t('dataset.required'), trigger: 'change' }
] ]
},
sqlOption: {
tabSize: 2,
styleActiveLine: true,
lineNumbers: true,
line: true,
mode: 'text/x-sql',
theme: 'solarized',
hintOptions: { // 自定义提示选项
completeSingle: false // 当匹配只有一项的时候是否自动补全
} }
},
incrementalUpdateType: 'incrementalAdd',
sql: '',
incrementalConfig: {}
} }
}, },
watch: { watch: {
...@@ -255,10 +333,49 @@ export default { ...@@ -255,10 +333,49 @@ export default {
this.listTaskLog() this.listTaskLog()
} }
}, },
computed: {
codemirror() {
return this.$refs.myCm.codemirror
}
},
mounted() {
window.onresize = () => {
return (() => {
this.height = window.innerHeight / 2
})()
}
this.height = window.innerHeight / 2
},
methods: { methods: {
incrementalUpdateTypeChange: function (){
if(this.incrementalUpdateType === 'incrementalAdd'){
if(this.sql){
this.incrementalConfig.incrementalDelete = this.sql
}
if(this.incrementalConfig.incrementalAdd){
this.sql = this.incrementalConfig.incrementalAdd
}else {
this.sql = ''
}
}
if(this.incrementalUpdateType === 'incrementalDelete'){
if(this.sql){
this.incrementalConfig.incrementalAdd = this.sql
}
if(this.incrementalConfig.incrementalDelete){
this.sql = this.incrementalConfig.incrementalDelete
}else {
this.sql = ''
}
}
},
showConfig() { showConfig() {
this.update_setting = true this.update_setting = true
this.listTask() this.listTask()
this.getIncrementalConfig()
}, },
addTask(task) { addTask(task) {
if (!task) { if (!task) {
...@@ -275,6 +392,31 @@ export default { ...@@ -275,6 +392,31 @@ export default {
this.taskData = response.data this.taskData = response.data
}) })
}, },
getIncrementalConfig() {
post('/dataset/table/incrementalConfig', { tableId: this.table.id }).then(response => {
this.incrementalConfig = response.data
this.incrementalUpdateType = 'incrementalAdd'
if(this.incrementalConfig.incrementalAdd){
this.sql = this.incrementalConfig.incrementalAdd
}
})
},
saveIncrementalConfig() {
this.update_setting = false
if(this.incrementalUpdateType === 'incrementalAdd'){
this.incrementalConfig.incrementalAdd = this.sql
}else {
this.incrementalConfig.incrementalDelete = this.sql
}
this.incrementalConfig.tableId = this.table.id
post('/dataset/table/save/incrementalConfig', this.incrementalConfig).then(response => {
this.$message({
message: this.$t('dataset.save_success'),
type: 'success',
showClose: true
})
})
},
saveTask(task) { saveTask(task) {
task.startTime = new Date(task.startTime).getTime() task.startTime = new Date(task.startTime).getTime()
task.endTime = new Date(task.endTime).getTime() task.endTime = new Date(task.endTime).getTime()
...@@ -289,18 +431,6 @@ export default { ...@@ -289,18 +431,6 @@ export default {
this.resetTaskForm() this.resetTaskForm()
this.listTask() this.listTask()
}) })
// this.$refs['taskForm'].validate((valid) => {
// if (valid) {
//
// } else {
// this.$message({
// message: this.$t('dataset.input_content'),
// type: 'error',
// showClose: true
// })
// return false
// }
// })
}, },
deleteTask(task) { deleteTask(task) {
this.$confirm(this.$t('dataset.confirm_delete'), this.$t('dataset.tips'), { this.$confirm(this.$t('dataset.confirm_delete'), this.$t('dataset.tips'), {
...@@ -325,7 +455,7 @@ export default { ...@@ -325,7 +455,7 @@ export default {
this.resetTaskForm() this.resetTaskForm()
}, },
onRateChange() { onRateChange() {
if (this.taskForm.rate === '0') { if (this.taskForm.rate === 'SIMPLE') {
this.taskForm.end = '0' this.taskForm.end = '0'
this.taskForm.endTime = '' this.taskForm.endTime = ''
this.taskForm.cron = '' this.taskForm.cron = ''
...@@ -346,13 +476,25 @@ export default { ...@@ -346,13 +476,25 @@ export default {
resetTaskForm() { resetTaskForm() {
this.taskForm = { this.taskForm = {
name: '', name: '',
type: '0', type: 'all_scope',
startTime: '', startTime: '',
rate: '0', rate: 'SIMPLE',
endTime: '', endTime: '',
end: '0' end: '0'
} }
},
onCmReady(cm) {
this.codemirror.setSize('-webkit-fill-available', 'auto')
},
onCmFocus(cm) {
// console.log('the editor is focus!', cm)
},
onCmCodeChange(newCode) {
console.log(newCode)
this.sql = newCode
this.$emit('codeChange', this.sql)
} }
} }
} }
</script> </script>
...@@ -369,4 +511,13 @@ export default { ...@@ -369,4 +511,13 @@ export default {
.el-form-item { .el-form-item {
margin-bottom: 10px; margin-bottom: 10px;
} }
.codemirror {
height: 160px;
overflow-y: auto;
}
.codemirror >>> .CodeMirror-scroll {
height: 160px;
overflow-y: auto;
}
</style> </style>
frontend/src/views/system/datasource/index.vue
浏览文件 @ e97297fd
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
@search="search" @search="search"
> >
<template #buttons> <template #buttons>
<fu-table-button icon="el-icon-circle-plus-outline" :label="$t('datasource.create')" @click="create" /> <fu-table-button v-permission="['datasource:add']" icon="el-icon-circle-plus-outline" :label="$t('datasource.create')" @click="create" />
</template> </template>
<!-- <el-table-column type="selection" fix /> --> <!-- <el-table-column type="selection" fix /> -->
...@@ -62,7 +62,7 @@ ...@@ -62,7 +62,7 @@
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.data_base')" prop="configuration.dataBase" :rules="{required: true, message: $t('datasource.please_input_data_base'), trigger: 'blur'}"> <el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.data_base')" prop="configuration.dataBase" :rules="{required: true, message: $t('datasource.please_input_data_base'), trigger: 'blur'}">
<el-input v-model="form.configuration.dataBase" autocomplete="off" /> <el-input v-model="form.configuration.dataBase" autocomplete="off" />
</el-form-item> </el-form-item>
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')" prop="configuration.username"> <el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')" prop="configuration.username" :rules="{required: true, message: $t('datasource.please_input_user_name'), trigger: 'blur'}">
<el-input v-model="form.configuration.username" autocomplete="off" /> <el-input v-model="form.configuration.username" autocomplete="off" />
</el-form-item> </el-form-item>
<el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')" prop="configuration.password" :rules="{required: true, message: $t('datasource.please_input_password'), trigger: 'change'}"> <el-form-item v-show="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')" prop="configuration.password" :rules="{required: true, message: $t('datasource.please_input_password'), trigger: 'change'}">
...@@ -79,7 +79,8 @@ ...@@ -79,7 +79,8 @@
<div slot="footer" class="dialog-footer"> <div slot="footer" class="dialog-footer">
<el-button type="text" @click="dialogVisible = false">{{ $t('commons.cancel') }}</el-button> <el-button type="text" @click="dialogVisible = false">{{ $t('commons.cancel') }}</el-button>
<el-button type="primary" @click="saveDatasource('createDatasource')">确认</el-button> <el-button type="primary" @click="validaDatasource('createDatasource')">{{ $t('commons.validate') }}</el-button>
<el-button type="primary" @click="saveDatasource('createDatasource')">{{ $t('commons.confirm') }}</el-button>
</div> </div>
</el-dialog> </el-dialog>
...@@ -198,13 +199,12 @@ export default { ...@@ -198,13 +199,12 @@ export default {
this.$success(this.$t('commons.save_success')) this.$success(this.$t('commons.save_success'))
this.search() this.search()
this.dialogVisible = false this.dialogVisible = false
}) });
} else { } else {
return false return false
} }
}) })
}, },
validaDatasource(datasourceForm) { validaDatasource(datasourceForm) {
this.$refs[datasourceForm].validate(valid => { this.$refs[datasourceForm].validate(valid => {
if (valid) { if (valid) {
...@@ -230,7 +230,6 @@ export default { ...@@ -230,7 +230,6 @@ export default {
const result = {} const result = {}
if (condition && condition.quick) { if (condition && condition.quick) {
for (const [key, value] of Object.entries(condition)) { for (const [key, value] of Object.entries(condition)) {
// console.log(`${key}`)
if (`${key}` === 'quick') { if (`${key}` === 'quick') {
const v_new = Object.assign({}, value) const v_new = Object.assign({}, value)
v_new['field'] = 'name' v_new['field'] = 'name'
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论