feat: 优化 oracle 增量更新

backend/src/main/java/io/dataease/provider/QueryProvider.java

@@ -43,4 +43,8 @@ public abstract class QueryProvider {
     public abstract String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
 
     public abstract String wrapSql(String sql);
+
+    public abstract String createRawQuerySQL(String table, List<DatasetTableField> fields);
+
+    public abstract String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields);
 }

backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java

@@ -348,6 +348,28 @@ public class DorisQueryProvider extends QueryProvider {
         return tmpSql;
     }
 
+    @Override
+    public String createRawQuerySQL(String table, List<DatasetTableField> fields){
+        String[] array = fields.stream().map(f -> {
+            StringBuilder stringBuilder = new StringBuilder();
+            stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
+            return stringBuilder.toString();
+        }).toArray(String[]::new);
+        return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
+    }
+
+    @Override
+    public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
+        return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
+    }
+
+    private String sqlFix(String sql) {
+        if (sql.lastIndexOf(";") == (sql.length() - 1)) {
+            sql = sql.substring(0, sql.length() - 1);
+        }
+        return sql;
+    }
+
     public String transMysqlFilterTerm(String term) {
         switch (term) {
             case "eq":

backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java

@@ -102,8 +102,6 @@ public class MysqlQueryProvider extends QueryProvider {
             }
             return stringBuilder.toString();
         }).toArray(String[]::new);
-
-//        return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getOriginName() : "null"), StringUtils.join(array, ","), table);
         return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
     }
 
@@ -356,6 +354,21 @@ public class MysqlQueryProvider extends QueryProvider {
         return tmpSql;
     }
 
+    @Override
+    public String createRawQuerySQL(String table, List<DatasetTableField> fields){
+        String[] array = fields.stream().map(f -> {
+            StringBuilder stringBuilder = new StringBuilder();
+            stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
+            return stringBuilder.toString();
+        }).toArray(String[]::new);
+        return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
+    }
+
+    @Override
+    public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
+        return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
+    }
+
     public String transMysqlFilterTerm(String term) {
         switch (term) {
             case "eq":

backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java

@@ -380,6 +380,31 @@ public class OracleQueryProvider extends QueryProvider {
         return getSQLSummary(" (" + sqlFix(sql) + ") tmp ", yAxis, customFilter, extFilterRequestList);
     }
 
+    @Override
+    public String wrapSql(String sql) {
+        sql = sql.trim();
+        if (sql.lastIndexOf(";") == (sql.length() - 1)) {
+            sql = sql.substring(0, sql.length() - 1);
+        }
+        String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0";
+        return tmpSql;
+    }
+
+    @Override
+    public String createRawQuerySQL(String table, List<DatasetTableField> fields){
+        String[] array = fields.stream().map(f -> {
+            StringBuilder stringBuilder = new StringBuilder();
+            stringBuilder.append(" ").append(f.getOriginName()).append("  AS ").append(f.getDataeaseName());
+            return stringBuilder.toString();
+        }).toArray(String[]::new);
+        return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
+    }
+
+    @Override
+    public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
+        return createRawQuerySQL(" (" + sqlFix(sql) + ") tmp ", fields);
+    }
+
     public String transMysqlFilterTerm(String term) {
         switch (term) {
             case "eq":
@@ -490,16 +515,6 @@ public class OracleQueryProvider extends QueryProvider {
         return sql;
     }
 
-    @Override
-    public String wrapSql(String sql) {
-        sql = sql.trim();
-        if (sql.lastIndexOf(";") == (sql.length() - 1)) {
-            sql = sql.substring(0, sql.length() - 1);
-        }
-        String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0";
-        return tmpSql;
-    }
-
     private String transDateFormat(String dateStyle, String datePattern) {
         String split = "-";
         if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {

backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java

@@ -270,6 +270,21 @@ public class SqlserverQueryProvider extends QueryProvider {
         return tmpSql;
     }
 
+    @Override
+    public String createRawQuerySQL(String table, List<DatasetTableField> fields){
+        String[] array = fields.stream().map(f -> {
+            StringBuilder stringBuilder = new StringBuilder();
+            stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName());
+            return stringBuilder.toString();
+        }).toArray(String[]::new);
+        return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
+    }
+
+    @Override
+    public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
+        return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields);
+    }
+
     public String transMysqlFilterTerm(String term) {
         switch (term) {
             case "eq":

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -206,13 +206,13 @@ public class ExtractDataService {
                     extractData(datasetTable, "all_scope");
                     replaceTable(DorisTableUtils.dorisName(datasetTableId));
                     saveSucessLog(datasetTableTaskLog);
-                    deleteFile("all_scope", datasetTableId);
+//                    deleteFile("all_scope", datasetTableId);
                     updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
                 }catch (Exception e){
                     saveErrorLog(datasetTableId, taskId, e);
                     updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
                     dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
-                    deleteFile("all_scope", datasetTableId);
+//                    deleteFile("all_scope", datasetTableId);
                 }finally {
                     if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
                         datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
@@ -267,15 +267,15 @@ public class ExtractDataService {
                             extractData(datasetTable, "incremental_delete");
                         }
                         saveSucessLog(datasetTableTaskLog);
-                        deleteFile("incremental_add", datasetTableId);
-                        deleteFile("incremental_delete", datasetTableId);
+//                        deleteFile("incremental_add", datasetTableId);
+//                        deleteFile("incremental_delete", datasetTableId);
                         updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
                     }
                 }catch (Exception e){
                     saveErrorLog(datasetTableId, taskId, e);
                     updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
-                    deleteFile("incremental_add", datasetTableId);
-                    deleteFile("incremental_delete", datasetTableId);
+//                    deleteFile("incremental_add", datasetTableId);
+//                    deleteFile("incremental_delete", datasetTableId);
                 }finally {
                     if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
                         datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString());
@@ -610,15 +610,7 @@ public class ExtractDataService {
                 dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
                 dataMeta.addExtraOption("MYSQL","characterEncoding", "UTF-8");
                 transMeta.addDatabase(dataMeta);
-                if (extractType.equalsIgnoreCase("all_scope")) {
-                    if(datasetTable.getType().equalsIgnoreCase("sql")){
-                        selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
-                    }else {
-                        String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
-                        QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
-                        selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
-                    }
-                }
+                selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
                 inputStep = inputStep(transMeta, selectSQL);
                 udjcStep = udjc(datasetTableFields, false);
                 break;
@@ -626,15 +618,7 @@ public class ExtractDataService {
                 SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfigration.class);
                 dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfigration.getHost(), sqlServerConfigration.getDataBase(), sqlServerConfigration.getPort().toString(), sqlServerConfigration.getUsername(), sqlServerConfigration.getPassword());
                 transMeta.addDatabase(dataMeta);
-                if (extractType.equalsIgnoreCase("all_scope")) {
-                    if(datasetTable.getType().equalsIgnoreCase("sql")){
-                        selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
-                    }else {
-                        String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
-                        QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
-                        selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
-                    }
-                }
+                selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
                 inputStep = inputStep(transMeta, selectSQL);
                 udjcStep = udjc(datasetTableFields, false);
                 break;
@@ -647,15 +631,8 @@ public class ExtractDataService {
                     dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword());
                 }
                transMeta.addDatabase(dataMeta);
-                if (extractType.equalsIgnoreCase("all_scope")) {
-                    if(datasetTable.getType().equalsIgnoreCase("sql")){
-                        selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
-                    }else {
-                        String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
-                        QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
-                        selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
-                    }
-                }
+
+                selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
                 inputStep = inputStep(transMeta, selectSQL);
                 udjcStep = udjc(datasetTableFields, false);
                 break;
@@ -702,6 +679,25 @@ public class ExtractDataService {
         FileUtils.writeStringToFile(file, transXml, "UTF-8");
     }
 
+    private String getSelectSQL(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) {
+        if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("db")) {
+            String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
+            QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
+            selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields);
+        }
+
+        if(extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")){
+            selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql();
+            QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
+            selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
+        }
+        if(!extractType.equalsIgnoreCase("all_scope")){
+            QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
+            selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields);
+        }
+        return selectSQL;
+    }
+
     private StepMeta inputStep(TransMeta transMeta, String selectSQL) {
         TableInputMeta tableInput = new TableInputMeta();
         DatabaseMeta database = transMeta.findDatabase("db");