✨(backend) keep ordering from fulltext search in results

Keep ordering by score from Find API on search/ results when the
fulltext search is enabled.
Refactor pagination to work with a list instead of a queryset
Fix Changelog

Signed-off-by: Fabre Florian <[email protected]>

Author: joehybird

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to
 - ✨(front) update to ui-kit v2
 - 🌐 add dutch translation
 - ✨(back) add resource server routes
+- ✨(backend) add async indexation of items on save (or access save)
+- ✨(backend) add throttle mechanism to limit indexation job
+- ✨(api) modify items/search endpoint to use indexed items in Find
 
 ### Fixed
 

src/backend/core/api/viewsets.py

@@ -961,7 +961,7 @@ def tree(self, request, pk=None):
         )
 
     # pylint: disable-next=too-many-arguments,too-many-positional-arguments
-    def _fulltext_search_queryset(self, queryset, indexer, request, text):
+    def _fulltext_search(self, queryset, indexer, request, text):
         """
         Returns a queryset from the results the fulltext search of Find
         """
@@ -975,10 +975,27 @@ def _fulltext_search_queryset(self, queryset, indexer, request, text):
             token=token,
             visited=get_visited_items_ids_of(queryset, user),
             page=1,
-            page_size=200,
+            page_size=100,
         )
 
-        return queryset.filter(pk__in=results)
+        queryset = queryset.filter(pk__in=results)
+        queryset = self.annotate_user_roles(queryset)
+        queryset = self.annotate_is_favorite(queryset)
+
+        files_by_uuid = {str(d.pk): d for d in queryset}
+        ordered_files = [files_by_uuid[id] for id in results if id in files_by_uuid]
+
+        page = self.paginate_queryset(ordered_files)
+
+        if page is not None:
+            items = self._compute_parents(page)
+            serializer = self.get_serializer(items, many=True)
+            result = self.get_paginated_response(serializer.data)
+            return result
+
+        items = self._compute_parents(ordered_files)
+        serializer = self.get_serializer(items, many=True)
+        return drf.response.Response(serializer.data)
 
     @drf.decorators.action(
         detail=False,
@@ -1036,7 +1053,7 @@ def search(self, request, *args, **kwargs):
         if indexer:
             # When the indexer is configured pop "title" from queryset search and use
             # fulltext results instead.
-            queryset = self._fulltext_search_queryset(
+            return self._fulltext_search(
                 queryset,
                 indexer,
                 request,

src/backend/core/management/commands/index.py

@@ -22,7 +22,8 @@ def handle(self, *args, **options):
         indexer = get_file_indexer()
 
         if not indexer:
-            raise CommandError("The indexer is not enabled or properly configured.")
+            logger.warning("The indexer is not enabled or properly configured.")
+            return
 
         logger.info("Starting to regenerate Find index...")
         start = time.perf_counter()

src/backend/core/services/search_indexers.py

@@ -44,7 +44,7 @@ def get_ancestor_to_descendants_map(items):
     """
     Given a list of items, return a mapping of ancestor_path -> set of descendant_paths.
 
-    Each path is assumed to use materialized path format with fixed-length segments.
+    Each path is assumed to be a list of uid.
 
     Args:
         items (list of Item): List of items.
@@ -55,11 +55,11 @@ def get_ancestor_to_descendants_map(items):
     ancestor_map = defaultdict(set)
 
     for item in items:
-        path = str(item.path)
-        ancestors = path.split(".")
+        fullpath = str(item.path)
+        ancestors = item.path
         for i in range(1, len(ancestors) + 1):
             ancestor = ".".join(ancestors[:i])
-            ancestor_map[ancestor].add(path)
+            ancestor_map[ancestor].add(fullpath)
 
     return ancestor_map
 
@@ -187,10 +187,9 @@ def index(self, queryset=None):
 
             last_id = items_batch[-1].id
             accesses_by_item_path = get_batch_accesses_by_users_and_teams(items_batch)
-            serialize = self.serialize_item
 
             serialized_batch = [
-                serialize(item, accesses_by_item_path) for item in items_batch
+                self.serialize_item(item, accesses_by_item_path) for item in items_batch
             ]
 
             self.push(serialized_batch)
@@ -242,8 +241,6 @@ def search(self, text, token, visited=(), page=1, page_size=50):
                 "services": ["drive"],
                 "page_number": page,
                 "page_size": page_size,
-                "order_by": "updated_at",
-                "order_direction": "desc",
             },
             token=token,
         )
@@ -317,7 +314,7 @@ def serialize_item(self, item, accesses):
             "users": list(accesses.get(doc_path, {}).get("users", set())),
             "groups": list(accesses.get(doc_path, {}).get("teams", set())),
             "reach": str(item.link_reach),
-            "size": item.size,
+            "size": item.size or 0,
             "is_active": not is_deleted,
         }
 

src/backend/core/tests/commands/test_index.py

@@ -2,10 +2,11 @@
 Unit test for `index` command.
 """
 
+import logging
 from operator import itemgetter
 from unittest import mock
 
-from django.core.management import CommandError, call_command
+from django.core.management import call_command
 from django.db import transaction
 
 import pytest
@@ -67,11 +68,21 @@ def test_index():
 
 @pytest.mark.django_db
 @pytest.mark.usefixtures("indexer_settings")
-def test_index_improperly_configured(indexer_settings):
+def test_index_improperly_configured(indexer_settings, caplog):
     """The command should raise an exception if the indexer is not configured"""
     indexer_settings.SEARCH_INDEXER_CLASS = None
 
-    with pytest.raises(CommandError) as err:
-        call_command("index")
+    with transaction.atomic():
+        factories.ItemFactory(
+            upload_bytes=b"This is a text file content",
+            mimetype="text/plain",
+            type=models.ItemTypeChoices.FILE,
+        )
+
+    with caplog.at_level(logging.WARN):
+        with mock.patch.object(SearchIndexer, "push") as mock_push:
+            call_command("index")
+
+    mock_push.assert_not_called()
 
-    assert str(err.value) == "The indexer is not enabled or properly configured."
+    assert "The indexer is not enabled or properly configured." in caplog.messages