feat: 增加 抽取模式视图 后台缓存

backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java

@@ -2,9 +2,16 @@ package io.dataease.base.mapper.ext;
 
 import io.dataease.controller.request.chart.ChartViewRequest;
 import io.dataease.dto.chart.ChartViewDTO;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
 
 import java.util.List;
 
+@Mapper
 public interface ExtChartViewMapper {
     List<ChartViewDTO> search(ChartViewRequest request);
+
+    @Select("select id from chart_view where table_id = #{tableId}")
+    List<String> allViewIds(@Param("tableId") String tableId);
 }

backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java

+package io.dataease.commons.constants;
+
+public class JdbcConstants {
+
+    public final static String VIEW_CACHE_KEY = "view_cache";
+}

backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java

@@ -10,7 +10,6 @@ import io.dataease.datasource.request.DatasourceRequest;
 import io.dataease.exception.DataEaseException;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
-
 import java.beans.PropertyVetoException;
 import java.sql.*;
 import java.util.*;
@@ -22,14 +21,29 @@ public class JdbcProvider extends DatasourceProvider {
     private static int initPoolSize = 5;
     private static int maxConnections = 200;
 
+    /**
+     * 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成，命中则使用缓存直接返回不再执行sql逻辑
+     * @param dsr
+     * @return
+     * @throws Exception
+     */
+    /**
+     * 这里使用声明式缓存不是很妥当
+     * 改为chartViewService中使用编程式缓存
+    @Cacheable(
+            value = JdbcConstants.JDBC_PROVIDER_KEY,
+            key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query",
+            condition = "#dsr.pageSize == null || #dsr.pageSize == 0L"
+    )
+     */
     @Override
-    public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
+    public List<String[]> getData(DatasourceRequest dsr) throws Exception {
         List<String[]> list = new LinkedList<>();
         Connection connection = null;
         try {
-            connection = getConnectionFromPool(datasourceRequest);
+            connection = getConnectionFromPool(dsr);
             Statement stat = connection.createStatement();
-            ResultSet rs = stat.executeQuery(datasourceRequest.getQuery());
+            ResultSet rs = stat.executeQuery(dsr.getQuery());
             list = fetchResult(rs);
         } catch (SQLException e) {
             DataEaseException.throwException(e);

backend/src/main/java/io/dataease/listener/util/CacheUtils.java

@@ -24,18 +24,19 @@ public class CacheUtils {
         return element.getObjectValue();
     }
 
-    private static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) {
+    public static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) {
         Element e = new Element(key, value);
         //不设置则使用xml配置
-        if (ttl != null)
+        if (ttl != null) {
             e.setEternal(false);
             e.setTimeToLive(ttl);
+        }
         if (tti != null)
             e.setTimeToIdle(tti);
         cache(cacheName).put(e);
     }
 
-    private static boolean remove(String cacheName, Object key) {
+    public static boolean remove(String cacheName, Object key) {
         return cache(cacheName).remove(key);
     }
 

backend/src/main/java/io/dataease/service/chart/ChartViewService.java

@@ -6,6 +6,7 @@ import io.dataease.base.domain.*;
 import io.dataease.base.mapper.ChartViewMapper;
 import io.dataease.base.mapper.ext.ExtChartGroupMapper;
 import io.dataease.base.mapper.ext.ExtChartViewMapper;
+import io.dataease.commons.constants.JdbcConstants;
 import io.dataease.commons.utils.AuthUtils;
 import io.dataease.commons.utils.BeanUtils;
 import io.dataease.commons.utils.CommonBeanFactory;
@@ -20,6 +21,7 @@ import io.dataease.datasource.service.DatasourceService;
 import io.dataease.dto.chart.*;
 import io.dataease.dto.dataset.DataTableInfoDTO;
 import io.dataease.i18n.Translator;
+import io.dataease.listener.util.CacheUtils;
 import io.dataease.provider.QueryProvider;
 import io.dataease.service.dataset.DataSetTableFieldsService;
 import io.dataease.service.dataset.DataSetTableService;
@@ -65,6 +67,10 @@ public class ChartViewService {
             chartView.setUpdateTime(timestamp);
             chartViewMapper.insertSelective(chartView);
         }
+        Optional.ofNullable(chartView.getId()).ifPresent(id -> {
+            CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, id);
+        });
+
         return chartView;
     }
 
@@ -188,6 +194,17 @@ public class ChartViewService {
                 }
             }
             data = datasourceProvider.getData(datasourceRequest);
+            /**
+             * 直连不实用缓存
+            String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery();
+            Object cache;
+            if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) {
+                data = datasourceProvider.getData(datasourceRequest);
+                CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null);
+            }else {
+                data = (List<String[]>) cache;
+            }
+             */
         } else if (table.getMode() == 1) {// 抽取
             // 连接doris，构建doris数据源查询
             Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
@@ -202,7 +219,15 @@ public class ChartViewService {
             } else {
                 datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList));
             }
-            data = datasourceProvider.getData(datasourceRequest);
+            // String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery();
+            // 定时抽取使用缓存
+            Object cache;
+            if ((cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, id)) == null) {
+                data = datasourceProvider.getData(datasourceRequest);
+                CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, id, data, null, null);
+            }else {
+                data = (List<String[]>) cache;
+            }
         }
         if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) {
             data = data.subList(0, 1000);

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -5,6 +5,8 @@ import io.dataease.base.domain.*;
 import io.dataease.base.mapper.DatasetTableMapper;
 import io.dataease.base.mapper.DatasetTableTaskMapper;
 import io.dataease.base.mapper.DatasourceMapper;
+import io.dataease.base.mapper.ext.ExtChartViewMapper;
+import io.dataease.commons.constants.JdbcConstants;
 import io.dataease.commons.constants.JobStatus;
 import io.dataease.commons.constants.ScheduleType;
 import io.dataease.commons.constants.UpdateType;
@@ -22,6 +24,7 @@ import io.dataease.datasource.provider.ProviderFactory;
 import io.dataease.datasource.request.DatasourceRequest;
 import io.dataease.dto.dataset.DataTableInfoDTO;
 import io.dataease.exception.DataEaseException;
+import io.dataease.listener.util.CacheUtils;
 import io.dataease.provider.QueryProvider;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.FileUtils;
@@ -96,6 +99,9 @@ public class ExtractDataService {
     @Resource
     private DatasetTableTaskMapper datasetTableTaskMapper;
 
+    @Resource
+    private ExtChartViewMapper extChartViewMapper;
+
     private static String lastUpdateTime = "${__last_update_time__}";
     private static String currentUpdateTime = "${__current_update_time__}";
     private static String separator = "|DE|";
@@ -277,7 +283,15 @@ public class ExtractDataService {
                     }
                 }
                 break;
-            }
+        }
+        //侵入式清除下属视图缓存
+        List<String> viewIds = extChartViewMapper.allViewIds(datasetTableId);
+        if (CollectionUtils.isNotEmpty(viewIds)){
+            viewIds.forEach(viewId -> {
+                CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId);
+            });
+        }
+
     }
 
     private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) {

backend/src/main/resources/ehcache/ehcache.xml

@@ -82,5 +82,17 @@
         <cacheEventListenerFactory class="io.dataease.listener.LicCacheEventListener" />
     </cache>
 
+    <cache
+            name="jdbc_provider_cache"
+            eternal="false"
+            maxElementsInMemory="100"
+            maxElementsOnDisk="10000"
+            overflowToDisk="true"
+            diskPersistent="true"
+            timeToIdleSeconds="28800"
+            timeToLiveSeconds="86400"
+            memoryStoreEvictionPolicy="LRU"
+    />
+
 
 </ehcache>
\ No newline at end of file