Clear only when starting search, added lastDoc for pagination (#3967)

Co-authored-by: Ramin Haeri Azad <[email protected]>
obiba · Oct 26, 2024 · 3cb4e87 · 3cb4e87
1 parent dfc45f3
commit 3cb4e87
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 60 deletions.
diff --git a/opal-search/src/main/java/org/obiba/opal/search/AbstractSearchUtility.java b/opal-search/src/main/java/org/obiba/opal/search/AbstractSearchUtility.java
@@ -31,22 +31,28 @@ public abstract class AbstractSearchUtility {
 
   abstract protected String getSearchPath();
 
-  protected QuerySettings buildQuerySearch(String query, int offset, int limit, Collection<String> fields,
+  protected QuerySettings buildQuerySearch(String query, String lastDoc, int limit, Collection<String> fields,
                                            Collection<String> facets, String sortField, String sortDir) {
     String sortBy = Strings.isNullOrEmpty(sortField) ? DEFAULT_SORT_FIELD : sortField;
     String sortOrder = Strings.isNullOrEmpty(sortDir) ? SortDir.DESC.toString() : sortDir;
-    return buildQuerySearch(query, offset, limit, fields, facets, Lists.newArrayList(sortBy + ":" + sortOrder));
+    return buildQuerySearch(query, lastDoc, limit, fields, facets, Lists.newArrayList(sortBy + ":" + sortOrder));
   }
 
-  protected QuerySettings buildQuerySearch(String query, int offset, int limit, Collection<String> fields,
+  protected QuerySettings buildQuerySearch(String query, String lastDoc, int limit, Collection<String> fields,
                                            Collection<String> facets, List<String> sortWithOrder) {
 
     List<String> safeFields = Lists.newArrayList();
     if (fields != null)
       safeFields.addAll(fields);
-    return QuerySettings.newSettings(query)
-        .fields(safeFields).facets(facets).from(offset).size(limit) //
-        .sortWithOrder(sortWithOrder);
+    QuerySettings querySettings = QuerySettings.newSettings(query)
+      .fields(safeFields).facets(facets).size(limit) //
+      .sortWithOrder(sortWithOrder);
+
+    if (!Strings.isNullOrEmpty(lastDoc)) {
+      querySettings.lastDoc(lastDoc);
+    }
+
+    return querySettings;
   }
 
   protected boolean searchServiceAvailable() {

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/QuerySettings.java b/opal-search/src/main/java/org/obiba/opal/search/service/QuerySettings.java
@@ -61,6 +61,8 @@ public class QuerySettings {
 
   private boolean withDefaultQueryFields = true;
 
+  private String lastDoc;
+
   //
   // Public methods
   //
@@ -83,6 +85,15 @@ public QuerySettings from(int value) {
     return this;
   }
 
+  public String getLastDoc() {
+    return lastDoc;
+  }
+
+  public QuerySettings lastDoc(String value) {
+    lastDoc = value;
+    return this;
+  }
+
   public int getSize() {
     return size;
   }

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/ContentQueryExecutor.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/ContentQueryExecutor.java
@@ -10,6 +10,7 @@
 
 package org.obiba.opal.search.service.impl;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.Sets;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
@@ -27,6 +28,7 @@
 import org.obiba.opal.search.service.QuerySettings;
 import org.obiba.opal.search.service.SearchException;
 import org.obiba.opal.search.service.SearchQueryExecutor;
+import org.obiba.opal.search.service.support.ScoreDocSerializer;
 import org.obiba.opal.web.model.Opal;
 import org.obiba.opal.web.model.Search;
 import org.slf4j.Logger;
@@ -63,13 +65,17 @@ public Search.QueryResultDto execute(QuerySettings querySettings) throws SearchE
 
       // Parse a query (search for books with "Lucene" in the title)
       Query query = parser.parse(querySettings.getQuery());
+      String lastDoc = querySettings.getLastDoc();
+      ScoreDoc lastScoreDoc = Strings.isNullOrEmpty(lastDoc) ? null : ScoreDocSerializer.deserialize(lastDoc);
 
       // Search for the top results
-      TopDocs results = searcher.search(query, querySettings.getSize());
+      TopDocs results = searcher.searchAfter(lastScoreDoc, query, querySettings.getSize());
       ScoreDoc[] hits = results.scoreDocs;
 
       // Build results
       Search.QueryResultDto.Builder builder = Search.QueryResultDto.newBuilder().setTotalHits((int) results.totalHits.value);
+      if (hits.length > 0) builder.setLastDoc(ScoreDocSerializer.serialize(hits[hits.length - 1]));
+
       StoredFields storedFields = reader.storedFields();
       for (ScoreDoc hit : hits) {
         Document doc = storedFields.document(hit. doc);

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/support/ScoreDocSerializer.java b/opal-search/src/main/java/org/obiba/opal/search/service/support/ScoreDocSerializer.java
@@ -0,0 +1,19 @@
+package org.obiba.opal.search.service.support;
+
+import org.apache.lucene.search.ScoreDoc;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class ScoreDocSerializer {
+  public static String serialize(ScoreDoc scoreDoc) {
+    String doc = scoreDoc.doc + ":" + scoreDoc.score + ":" + scoreDoc.shardIndex;
+    return Base64.getEncoder().encodeToString(doc.getBytes(StandardCharsets.UTF_8));
+  }
+
+  public static ScoreDoc deserialize(String serialized) {
+    byte[] decoded = Base64.getDecoder().decode(serialized);
+    String[] parts = new String(decoded, StandardCharsets.UTF_8).split(":");
+    return new ScoreDoc(Integer.parseInt(parts[0]), Float.parseFloat(parts[1]), Integer.parseInt(parts[2]));
+  }
+}
diff --git a/opal-search/src/main/java/org/obiba/opal/web/search/DatasourcesVariablesSearchResource.java b/opal-search/src/main/java/org/obiba/opal/web/search/DatasourcesVariablesSearchResource.java
@@ -44,14 +44,14 @@ public class DatasourcesVariablesSearchResource extends AbstractSearchUtility {
   @GET
   @Path("_search")
   @Transactional(readOnly = true)
-  public Response search(@QueryParam("query") String query, @QueryParam("offset") @DefaultValue("0") int offset,
-      @QueryParam("limit") @DefaultValue("10") int limit, @QueryParam("sort") List<String> sorts, @QueryParam("order") String order,
-      @SuppressWarnings("TypeMayBeWeakened") @QueryParam("field") List<String> fields,
-      @SuppressWarnings("TypeMayBeWeakened") @QueryParam("facet") List<String> facets) {
+  public Response search(@QueryParam("query") String query, @QueryParam("limit") @DefaultValue("10") int limit,
+                         @QueryParam("lastDoc") String lastDoc, @QueryParam("sort") List<String> sorts, @QueryParam("order") String order,
+                         @SuppressWarnings("TypeMayBeWeakened") @QueryParam("field") List<String> fields,
+                         @SuppressWarnings("TypeMayBeWeakened") @QueryParam("facet") List<String> facets) {
 
     try {
-      if(!searchServiceAvailable()) return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
-      String defaultOrder = Strings.isNullOrEmpty(order) ? SortDir.ASC.toString() : order;
+      if (!searchServiceAvailable()) return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
+     String defaultOrder = Strings.isNullOrEmpty(order) ? SortDir.ASC.toString() : order;
       List<String> sortsWithOrder = Lists.newArrayList();
       if (sorts == null || sorts.isEmpty()) {
         sortsWithOrder.add(DEFAULT_SORT_FIELD + ":desc");
@@ -63,10 +63,10 @@ public Response search(@QueryParam("query") String query, @QueryParam("offset")
           sortsWithOrder.add(sortField + ":" + sortOrder);
         }
       }
-      Search.QueryResultDto dtoResponse = opalSearchService.executeQuery(buildQuerySearch(query, offset, limit, fields, facets, sortsWithOrder),
-          getSearchPath(), null);
+      Search.QueryResultDto dtoResponse = opalSearchService.executeQuery(buildQuerySearch(query, lastDoc, limit, fields, facets, sortsWithOrder),
+        getSearchPath(), null);
       return Response.ok().entity(dtoResponse).build();
-    } catch(Exception e) {
+    } catch (Exception e) {
       log.error("Unable to perform variables search", e);
       return Response.status(Response.Status.BAD_REQUEST).build();
     }
@@ -78,9 +78,9 @@ protected String getSearchPath() {
   }
 
   @Override
-  protected QuerySettings buildQuerySearch(String query, int offset, int limit, Collection<String> fields,
+  protected QuerySettings buildQuerySearch(String query, String lastDoc, int limit, Collection<String> fields,
                                            Collection<String> facets, List<String> sortWithOrder) {
-    return super.buildQuerySearch(query, offset, limit, fields, facets, sortWithOrder).filterReferences(getTableReferencesFilter());
+    return super.buildQuerySearch(query, lastDoc, limit, fields, facets, sortWithOrder).filterReferences(getTableReferencesFilter());
   }
 
   //
@@ -89,8 +89,8 @@ protected QuerySettings buildQuerySearch(String query, int offset, int limit, Co
 
   private Collection<String> getTableReferencesFilter() {
     Collection<String> references = new ArrayList<>();
-    for(Datasource datasource : MagmaEngine.get().getDatasources()) {
-      for(ValueTable valueTable : datasource.getValueTables()) {
+    for (Datasource datasource : MagmaEngine.get().getDatasources()) {
+      for (ValueTable valueTable : datasource.getValueTables()) {
         references.add(valueTable.getTableReference());
       }
     }

diff --git a/opal-search/src/main/java/org/obiba/opal/web/search/TableVariablesSearchResource.java b/opal-search/src/main/java/org/obiba/opal/web/search/TableVariablesSearchResource.java
@@ -23,6 +23,7 @@
 
 import jakarta.ws.rs.*;
 import jakarta.ws.rs.core.Response;
+
 import java.util.List;
 
 @Component
@@ -39,20 +40,19 @@ public class TableVariablesSearchResource extends AbstractSearchUtility {
   @GET
   @Transactional(readOnly = true)
   @SuppressWarnings("PMD.ExcessiveParameterList")
-  public Response search(@QueryParam("query") String query, @QueryParam("offset") @DefaultValue("0") int offset,
-                         @QueryParam("limit") @DefaultValue("10") int limit,
-                         @QueryParam("variable") @DefaultValue("false") boolean addVariableDto, @QueryParam("field") List<String> fields,
-                         @QueryParam("facet") List<String> facets, @QueryParam("sortField") String sortField,
-                         @QueryParam("sortDir") String sortDir) {
+  public Response search(@QueryParam("query") String query, @QueryParam("limit") @DefaultValue("10") int limit,
+                         @QueryParam("lastDoc") String lastDoc, @QueryParam("variable") @DefaultValue("false") boolean addVariableDto,
+                         @QueryParam("field") List<String> fields, @QueryParam("facet") List<String> facets,
+                         @QueryParam("sortField") String sortField, @QueryParam("sortDir") String sortDir) {
 
     try {
       if (!canQueryEsIndex()) return Response.status(Response.Status.SERVICE_UNAVAILABLE).build();
       if (!opalSearchService.getVariablesIndexManager().hasIndex(getValueTable()))
         return Response.status(Response.Status.NOT_FOUND).build();
       String esQuery = "reference:\"" + getValueTable().getTableReference() + "\" AND " + query;
-      Search.QueryResultDto dtoResponse = opalSearchService.executeQuery(buildQuerySearch(esQuery, offset, limit, fields, facets, sortField, sortDir),
-          getSearchPath(),
-          addVariableDto ? new ItemResultDtoStrategy(getValueTable()) : null);
+      Search.QueryResultDto dtoResponse = opalSearchService.executeQuery(buildQuerySearch(esQuery, lastDoc, limit, fields, facets, sortField, sortDir),
+        getSearchPath(),
+        addVariableDto ? new ItemResultDtoStrategy(getValueTable()) : null);
       return Response.ok().entity(dtoResponse).build();
     } catch (NoSuchValueSetException | NoSuchDatasourceException e) {
       return Response.status(Response.Status.NOT_FOUND).build();
@@ -69,7 +69,7 @@ protected String getSearchPath() {
 
   private boolean canQueryEsIndex() {
     return searchServiceAvailable() && opalSearchService.getVariablesIndexManager().isReady() &&
-        opalSearchService.getVariablesIndexManager().isIndexUpToDate(getValueTable());
+      opalSearchService.getVariablesIndexManager().isIndexUpToDate(getValueTable());
   }
 
   private ValueTable getValueTable() {

diff --git a/opal-ui/src/pages/SearchVariablesPage.vue b/opal-ui/src/pages/SearchVariablesPage.vue
@@ -20,31 +20,31 @@
               :label="$t('query')"
               flat
               dense
-              size="sm"
               style="min-width: 400px"
-              @update:model-value="onClear"
-              @keyup.enter="onSubmit"
-            />
-            <q-btn
-              :label="$t('search')"
-              icon="search"
-              color="primary"
-              size="sm"
-              @click="onSubmit"
-              :disable="loading || !isValid"
-              class="q-mt-lg"
-              style="height: 2.5em"
-            />
-            <q-btn
-              outline
-              color="secondary"
-              icon="cleaning_services"
-              :label="$t('clear')"
-              size="sm"
-              style="height: 2.5em"
-              @click="onClearAndReset"
-              class="q-mt-lg"
+              @keyup.enter="onClearAndSubmit"
             />
+            <div class="q-gutter-x-md q-mt-lg">
+              <q-btn
+                :label="$t('search')"
+                icon="search"
+                color="primary"
+                size="sm"
+                @click="onClearAndSubmit"
+                :disable="loading || !isValid"
+                class=""
+                style="height: 2.5em"
+              />
+              <q-btn
+                outline
+                color="secondary"
+                icon="search_off"
+                :label="$t('clear')"
+                size="sm"
+                style="height: 2.5em"
+                @click="onClearAndReset"
+                class=""
+              />
+            </div>
           </div>
         </q-card-section>
         <q-card-section class="q-pt-none">
@@ -158,6 +158,7 @@ const { t, locale } = useI18n({ useScope: 'global' });
 const loading = ref<boolean>(false);
 const showResults = ref(false);
 const results = ref<QueryResultDto>();
+const lastDoc = ref();
 const limit = ref<number>(10);
 const tables = ref<TableDto[]>([]);
 
@@ -224,17 +225,26 @@ onMounted(() => {
 function onClear() {
   showResults.value = false;
   results.value = undefined;
+  lastDoc.value = undefined;
   limit.value = 10;
 }
 
 function onSubmit() {
   if (isValid.value) {
     loading.value = true;
     searchStore
-      .searchVariables(limit.value)
+      .searchVariables(limit.value, lastDoc.value)
       .then((res) => {
-        showResults.value = res.totalHits > 0;
-        results.value = res;
+        const totalHits = res.totalHits;
+        showResults.value = totalHits > 0;
+        lastDoc.value = res.lastDoc;
+
+        if (totalHits > 0 && !!results.value && !!lastDoc.value) {
+          results.value.totalHits = totalHits;
+          results.value.hits.push(...res.hits);
+        } else {
+          results.value = res;
+        }
       })
       .finally(() => (loading.value = false));
   } else {
@@ -272,7 +282,6 @@ function goToVariable(item: ItemFieldsResultDto) {
 }
 
 function addLimit() {
-  limit.value += 10;
   onSubmit();
 }
 

diff --git a/opal-ui/src/stores/search.ts b/opal-ui/src/stores/search.ts
@@ -34,7 +34,7 @@ export const useSearchStore = defineStore('search', () => {
     };
   }
 
-  async function searchVariables(limit: number) {
+  async function searchVariables(limit: number, lastDoc: string | undefined) {
     let fullQuery = variablesQuery.value.query?.trim() || '';
     Object.keys(variablesQuery.value.criteria).forEach((key) => {
       const terms = variablesQuery.value.criteria[key];
@@ -43,13 +43,13 @@ export const useSearchStore = defineStore('search', () => {
         fullQuery = fullQuery.length === 0 ? statement : `${fullQuery} AND ${statement}`;
       }
     });
-    return search(fullQuery, limit, ['label', 'label-en']);
+    return search(fullQuery, limit, ['label', 'label-en'], lastDoc);
   }
 
-  async function search(query: string, limit: number, fields: string[] | undefined) {
+  async function search(query: string, limit: number, fields: string[] | undefined, lastDoc: string | undefined) {
     return api
       .get('/datasources/variables/_search', {
-        params: { query, limit, field: fields },
+        params: { query, lastDoc: lastDoc , limit, field: fields },
         paramsSerializer: {
           indexes: null, // no brackets at all
         },

diff --git a/opal-web-model/src/main/protobuf/Search.proto b/opal-web-model/src/main/protobuf/Search.proto
@@ -86,6 +86,7 @@ message QueryResultDto {
   repeated FacetResultDto facets = 1;
   required int32 totalHits = 2;
   repeated ItemResultDto hits = 3;
+  optional string lastDoc = 4;
 }
 
 message FacetResultDto {