From 9b0f8f1e40c6b89f3cd2cd95e50be9378d1dcca1 Mon Sep 17 00:00:00 2001
From: Adam Collins <adam.collins@csiro.au>
Date: Mon, 25 Mar 2024 17:00:47 +1000
Subject: [PATCH] #891 appending list kvp to downloads stability

---
 .../ala/biocache/dao/SolrIndexDAOImpl.java    |  5 ++-
 .../ala/biocache/service/ListsService.java    | 38 ++++++++++++++++---
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/src/main/java/au/org/ala/biocache/dao/SolrIndexDAOImpl.java b/src/main/java/au/org/ala/biocache/dao/SolrIndexDAOImpl.java
index ef3e5a214..9485b0601 100644
--- a/src/main/java/au/org/ala/biocache/dao/SolrIndexDAOImpl.java
+++ b/src/main/java/au/org/ala/biocache/dao/SolrIndexDAOImpl.java
@@ -1116,6 +1116,7 @@ private TupleStream openStream(SolrParams params) throws IOException {
     @Override
     public int streamingQuery(SolrQuery query, ProcessInterface procSearch, ProcessInterface procFacet, SolrQuery endemicFacetSuperset) throws SolrServerException {
         int tupleCount = 0;
+        boolean finished = false;
         try {
             if (logger.isDebugEnabled()) {
                 logger.debug("SOLR query:" + query.toString());
@@ -1125,9 +1126,9 @@ public int streamingQuery(SolrQuery query, ProcessInterface procSearch, ProcessI
             if (procSearch != null && query.getRows() != 0) {
                 try (TupleStream solrStream = openStream(buildSearchExpr(query));) {
                     Tuple tuple;
-                    while (!(tuple = solrStream.read()).EOF && (tupleCount < query.getRows() || query.getRows() < 0)) {
+                    while (!(tuple = solrStream.read()).EOF && (tupleCount < query.getRows() || query.getRows() < 0) && !finished) {
                         tupleCount++;
-                        procSearch.process(tuple);
+                        finished = procSearch.process(tuple);
                     }
                     procSearch.flush();
                 }
diff --git a/src/main/java/au/org/ala/biocache/service/ListsService.java b/src/main/java/au/org/ala/biocache/service/ListsService.java
index afa7742f6..15c9a3c27 100644
--- a/src/main/java/au/org/ala/biocache/service/ListsService.java
+++ b/src/main/java/au/org/ala/biocache/service/ListsService.java
@@ -19,6 +19,7 @@
 import au.org.ala.biocache.service.ListsService.SpeciesListSearchDTO.SpeciesListDTO;
 import au.org.ala.biocache.util.SearchUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
+import groovyjarjarantlr4.v4.misc.OrderedHashMap;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.springframework.beans.factory.annotation.Value;
@@ -176,6 +177,8 @@ public List<Kvp> getKvp(String dataResourceUid) {
         int max = 400;  // response size can be limited by api gateway
         int offset = 0;
 
+        Map<String, Integer> kvpKeys = new OrderedHashMap<>();
+
         try {
             while (hasAnotherPage) {
                 SpeciesListItemsDTO speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesListItems/" + dataResourceUid + "?includeKVP=true&max=" + max + "&offset=" + offset), SpeciesListItemsDTO.class);
@@ -188,12 +191,23 @@ public List<Kvp> getKvp(String dataResourceUid) {
                         // ignore species list item when there are no lft rgt values for the LSID
                         String fq = searchUtils.getTaxonSearch(item.lsid)[0];
                         if (fq.startsWith("lft:[")) {
-                            List<String> keys = new ArrayList<>();
-                            List<String> values = new ArrayList<>();
+                            List<String> keys = new ArrayList<>(kvpKeys.size());
+                            List<String> values = new ArrayList<>(kvpKeys.size());
 
                             for (KvpDTO kvp : item.kvpValues) {
-                                keys.add(kvp.key);
-                                values.add(kvp.value);
+                                int pos = kvpKeys.getOrDefault(kvp.key, kvpKeys.size());
+                                if (pos == kvpKeys.size()) {
+                                    kvpKeys.put(kvp.key, pos);
+                                }
+
+                                // padding. keys and length is updated before return
+                                while (keys.size() <= pos) {
+                                    keys.add("");
+                                    values.add("");
+                                }
+
+                                keys.set(pos, kvp.key);
+                                values.set(pos, kvp.value);
                             }
 
                             long lft = Long.parseLong(fq.replaceAll("(.*\\[| TO.*)", ""));
@@ -212,6 +226,20 @@ public List<Kvp> getKvp(String dataResourceUid) {
 
         if (list.size() > 0) {
             list.sort(Kvp.KvpComparator);
+
+            // kvp array padding and key standardisation
+            for (Kvp kvp : list) {
+                // padding
+                while (kvp.keys.size() < kvpKeys.size()) {
+                    kvp.keys.add("");
+                    kvp.values.add("");
+                }
+
+                // standardise key
+                for (Map.Entry<String, Integer> entity : kvpKeys.entrySet()) {
+                    kvp.keys.set(entity.getValue(), entity.getKey());
+                }
+            }
             return list;
         } else {
             return null;
@@ -307,7 +335,7 @@ public String getKvpValue(int idx, List<Kvp> kvps, Kvp lftrgt) {
 
         if (kvps != null && kvps.size() > idx) {
             Kvp kvp = find(kvps, lftrgt);
-            if (kvp != null) {
+            if (kvp != null && idx < kvp.values.size()) {
                 value = kvp.values.get(idx);
             }
         }