fix: api 数据集补全字段

backend/src/main/java/io/dataease/auth/api/demo.java

@@ -3,13 +3,10 @@ package io.dataease.auth.api;
 
 import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
-import com.github.xiaoymin.knife4j.annotations.ApiSupport;
-import io.dataease.base.domain.DatasetTableFunction;
-import io.swagger.annotations.Api;
+
 import io.swagger.annotations.ApiOperation;
 import org.springframework.web.bind.annotation.*;
 
-import java.util.List;
 
 
 @RestController
@@ -19,10 +16,10 @@ public class demo {
     @GetMapping("demo")
     public Object listByTableId() {
         JSONArray jsonArray = new JSONArray();
-        for(int i=0;i<100;i++){
+        for(int i=0;i<10;i++){
             JSONObject jsonObject = new JSONObject();
-            for(int j=0;j<10;j++){
-                jsonObject.set("column" + j, "value"+j);
+            for(int j=0;j<1;j++){
+                jsonObject.set("column" +i + j, "value"+i+j);
             }
             jsonArray.put(jsonObject);
         }

backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java

 package io.dataease.commons.utils;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.entity.EntityBuilder;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -90,8 +90,7 @@ public class HttpClientUtil {
             httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset());
 
             HttpResponse response = httpClient.execute(httpGet);
-            HttpEntity entity = response.getEntity();
-            return getResponseStr(response, entity, config);
+            return getResponseStr(response, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
             throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@@ -136,8 +135,7 @@ public class HttpClientUtil {
             httpPost.setEntity(requestEntity);
 
             HttpResponse response = httpClient.execute(httpPost);
-            HttpEntity entity = response.getEntity();
-            return getResponseStr(response, entity, config);
+            return getResponseStr(response, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
             throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@@ -198,8 +196,7 @@ public class HttpClientUtil {
             }
 
             HttpResponse response = httpClient.execute(httpPost);
-            HttpEntity entity = response.getEntity();
-            return getResponseStr(response, entity, config);
+            return getResponseStr(response, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
             throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@@ -212,10 +209,14 @@ public class HttpClientUtil {
         }
     }
 
-    private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{
+    private static String getResponseStr(HttpResponse response, HttpClientConfig config) throws Exception{
         if(response.getStatusLine().getStatusCode() >= 400){
-            throw new Exception(EntityUtils.toString(entity, config.getCharset()));
+            String msg = EntityUtils.toString(response.getEntity(), config.getCharset());
+            if(StringUtils.isEmpty(msg)){
+                msg = "StatusCode: " + response.getStatusLine().getStatusCode();
+            }
+            throw new Exception(msg);
         }
-        return EntityUtils.toString(entity, config.getCharset());
+        return EntityUtils.toString(response.getEntity(), config.getCharset());
     }
 }

backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java

@@ -28,7 +28,7 @@ public class ApiProvider extends DatasourceProvider{
     public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
         ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
         String response = execHttpRequest(apiDefinition);
-        return fetchResult(response, apiDefinition.getDataPath());
+        return fetchResult(response, apiDefinition);
     }
 
     @Override
@@ -64,7 +64,7 @@ public class ApiProvider extends DatasourceProvider{
 
         fieldList = getTableFileds(datasourceRequest);
         result.put("fieldList", fieldList);
-        dataList = fetchResult(response, apiDefinition.getDataPath());
+        dataList = fetchResult(response, apiDefinition);
         result.put("dataList", dataList);
         return result;
     }
@@ -170,16 +170,14 @@ public class ApiProvider extends DatasourceProvider{
         return response;
     }
 
-    private List<String[]> fetchResult(String result, String path){
+    private List<String[]> fetchResult(String result, ApiDefinition apiDefinition){
         List<String[]> dataList = new LinkedList<>();
-        List<LinkedHashMap> datas = JsonPath.read(result, path);
+        List<LinkedHashMap> datas = JsonPath.read(result, apiDefinition.getDataPath());
         for (LinkedHashMap data : datas) {
-            String[] row = new String[data.entrySet().size()];
-            Iterator it = data.entrySet().iterator();
+            String[] row = new String[apiDefinition.getFields().size()];
             int i = 0;
-            while (it.hasNext()){
-                Map.Entry entry = (Map.Entry)it.next();
-                row[i] = Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
+            for (DatasetTableField field : apiDefinition.getFields()) {
+                row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
                 i++;
             }
             dataList.add(row);

backend/src/main/java/io/dataease/service/datasource/DatasourceService.java

@@ -362,18 +362,16 @@ public class DatasourceService {
 
         List<JSONObject> dataList = new ArrayList<>();
         List<DatasetTableField> fields = new ArrayList<>();
-        Boolean getFileds = true;
-
+        Set<String> fieldKeys = new HashSet<>();
+        //第一遍获取 field
         for (LinkedHashMap data : datas) {
-            JSONObject jsonObject = new JSONObject();
-            Iterator it = data.entrySet().iterator();
-            while (it.hasNext()){
-                Map.Entry entry = (Map.Entry)it.next();
-                jsonObject.put((String) entry.getKey(), Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
-                if(getFileds) {
+            Set<String> keys = data.keySet();
+            for (String key : keys) {
+                if(!fieldKeys.contains(key)){
+                    fieldKeys.add(key);
                     DatasetTableField tableField = new DatasetTableField();
-                    tableField.setOriginName((String) entry.getKey());
-                    tableField.setName((String) entry.getKey());
+                    tableField.setOriginName(key);
+                    tableField.setName(key);
                     tableField.setSize(65535);
                     tableField.setDeExtractType(0);
                     tableField.setDeType(0);
@@ -381,7 +379,13 @@ public class DatasourceService {
                     fields.add(tableField);
                 }
             }
-            getFileds = false;
+        }
+        //第二遍获取 data
+        for (LinkedHashMap data : datas) {
+            JSONObject jsonObject = new JSONObject();
+            for (String key : fieldKeys) {
+                jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
+            }
             dataList.add(jsonObject);
         }
         apiDefinition.setDatas(dataList);