✨(backend) Fix trashbin & mimetype issues

In Drive the search should find files in the trashbin for a limited amount
of time. So the soft-delete cannot disable a indexed entry
Be more strict on mimetype patterns.

Signed-off-by: Fabre Florian <[email protected]>

Author: joehybird

src/backend/core/services/search_indexers.py

@@ -115,6 +115,32 @@ def get_visited_items_ids_of(queryset, user):
     return [str(id) for id in docs.values_list("pk", flat=True)]
 
 
+def match_mimetype_glob(mimetype, pattern):
+    """
+    Returns true if the mimetype match with the pattern.
+    If a pattern ends with / all subtypes are valid, if not it have to
+    perfectly match. e.g :
+      - application/pdf only match "application/pdf" and not "application/pdf+bin"
+      - application/ match any "application/*"
+    """
+    if len(mimetype) < 1:
+        return False
+
+    if pattern.endswith("/"):
+        return mimetype.startswith(pattern)
+
+    return mimetype == pattern
+
+
+def is_allowed_mimetype(mimetype, patterns):
+    """
+    Returns true if the mimetype is not empty and matches any of the allowed patterns.
+    """
+    return len(mimetype) > 0 and any(
+        match_mimetype_glob(mimetype, pattern) for pattern in patterns
+    )
+
+
 class BaseItemIndexer(ABC):
     """
     Base class for item indexers.
@@ -166,17 +192,17 @@ def index(self, queryset=None, batch_size=None):
 
         Args:
             queryset (optional): Document queryset
-                Defaults to all documents without filter.
+                Defaults to all documents without the main workspaces.
             batch_size (int, optional): Number of documents per batch.
                 Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
         """
         last_id = None
         count = 0
         batch_size = batch_size or self.batch_size
-        queryset = queryset or models.Item.objects.all()
-        queryset = queryset.filter(
+        queryset = queryset or models.Item.objects.filter(
             main_workspace=False,
-        ).order_by("id")
+        )
+        queryset = queryset.order_by("id")
 
         while True:
             if last_id is not None:
@@ -280,8 +306,7 @@ def has_text(self, item):
         return (
             item.upload_state == models.ItemUploadStateChoices.READY
             and item.type == models.ItemTypeChoices.FILE
-            and len(mimetype) > 0
-            and any(mimetype.startswith(allowed) for allowed in self.allowed_mimetypes)
+            and is_allowed_mimetype(mimetype, self.allowed_mimetypes)
         )
 
     def serialize_item(self, item, accesses):
@@ -296,12 +321,15 @@ def serialize_item(self, item, accesses):
             dict: A JSON-serializable dictionary.
         """
         doc_path = str(item.path)
-        is_deleted = bool(item.ancestors_deleted_at or item.deleted_at)
         content = ""
 
-        # There is no endpoint in Find API for deleted items so we index it
+        # The deleted items are still accessible in Drive (not in Docs !)
+        # See in V2 for handling hard deleted ones
+        is_active = True
+
+        # There is no endpoint in Find API for inactive items so we index it
         # again with an empty content.
-        if not is_deleted and self.has_text(item):
+        if is_active and self.has_text(item):
             content = self.to_text(item)
 
         return {
@@ -319,7 +347,7 @@ def serialize_item(self, item, accesses):
             "groups": list(accesses.get(doc_path, {}).get("teams", set())),
             "reach": str(item.link_reach),
             "size": item.size or 0,
-            "is_active": not is_deleted,
+            "is_active": is_active,
         }
 
     def search_query(self, data, token) -> requests.Response:

src/backend/core/tasks/search.py

@@ -55,7 +55,10 @@ def batch_file_indexer_task(timestamp):
     indexer = get_file_indexer()
 
     if indexer:
-        queryset = models.Item.objects.filter(updated_at__gte=timestamp)
+        queryset = models.Item.objects.filter(
+            updated_at__gte=timestamp,
+            main_workspace=False,
+        )
 
         count = indexer.index(queryset)
         logger.info("Indexed %d files", count)

src/backend/core/tests/test_services_search_indexers.py

@@ -20,6 +20,7 @@
     get_ancestor_to_descendants_map,
     get_file_indexer,
     get_visited_items_ids_of,
+    is_allowed_mimetype,
 )
 
 pytestmark = pytest.mark.django_db
@@ -251,15 +252,43 @@ def test_services_search_allowed_mimetypes_is_invalid(
     assert expected in str(exc_info.value)
 
 
+def test_services_is_allowed_mimetype():
+    """
+    Check if the given mimetype matches with the allowed ones.
+    """
+    assert is_allowed_mimetype("", ["text/"]) is False
+    assert is_allowed_mimetype("text/plain", []) is False
+    assert is_allowed_mimetype("text/plain", ["text/html"]) is False
+    assert is_allowed_mimetype("text/plain", ["text/"]) is True
+    assert is_allowed_mimetype("application/pdf", ["text/", "application/pdf"]) is True
+    assert (
+        is_allowed_mimetype("application/pdf+bin", ["text/", "application/pdf"])
+        is False
+    )
+    assert is_allowed_mimetype("application/pdf+bin", ["text/", "application/"]) is True
+
+
 @pytest.mark.parametrize(
     "kwargs, expected",
     [
         ({"mimetype": "text/plain"}, True),
-        ({"mimetype": "text/html"}, True),
-        ({"mimetype": "application/html"}, False),
         ({"mimetype": "application/pdf"}, True),
-        ({"mimetype": "application/"}, False),
-        ({"mimetype": ""}, False),
+        ({"mimetype": "application/pdf+bin"}, False),
+        (
+            {
+                "mimetype": "text/plain",
+                "type": models.ItemTypeChoices.FOLDER,
+                "upload_bytes": None,
+            },
+            False,
+        ),
+        (
+            {
+                "mimetype": "text/plain",
+                "update_upload_state": models.ItemUploadStateChoices.ANALYZING,
+            },
+            False,
+        ),
     ],
 )
 def test_services_search_has_text(indexer_settings, kwargs, expected):
@@ -272,11 +301,16 @@ def test_services_search_has_text(indexer_settings, kwargs, expected):
         "application/pdf",
     ]
 
+    params = {
+        "upload_bytes": b"This is a text file content",
+        "type": models.ItemTypeChoices.FILE,
+        "update_upload_state": models.ItemUploadStateChoices.READY,
+    }
+
+    params.update(kwargs)
+
     item = factories.ItemFactory(
-        upload_bytes=b"This is a text file content",
-        type=models.ItemTypeChoices.FILE,
-        update_upload_state=models.ItemUploadStateChoices.READY,
-        **kwargs,
+        **params,
     )
 
     assert expected == SearchIndexer().has_text(item)
@@ -422,13 +456,18 @@ def test_services_search_indexers_serialize_item_textfile():
 
 
 @pytest.mark.usefixtures("indexer_settings")
-def test_services_search_indexers_serialize_document_deleted():
-    """Deleted documents are marked as just in the serialized json."""
+def test_services_search_indexers_serialize_document_soft_deleted():
+    """
+    Deleted documents are NOT marked as inactive in the serialized json because
+    they can be accessed through the trashbin for several days.
+    """
     folder = factories.ItemFactory(type=models.ItemTypeChoices.FOLDER)
     item = factories.ItemFactory(
         parent=folder,
         mimetype="text/plain",
         type=models.ItemTypeChoices.FILE,
+        upload_bytes=b"This is a text file content",
+        update_upload_state=models.ItemUploadStateChoices.READY,
     )
 
     folder.soft_delete()
@@ -437,7 +476,9 @@ def test_services_search_indexers_serialize_document_deleted():
     indexer = SearchIndexer()
     result = indexer.serialize_item(item, {})
 
-    assert result["is_active"] is False
+    # Still accessible through the thrashbin
+    assert result["is_active"] is True
+    assert result["content"] == "This is a text file content"
 
 
 @responses.activate