✨(backend) enable limit for the indexed content

Use SEARCH_INDEXER_CONTENT_MAX_SIZE as limit (in bytes) for the file content.
Fix default configuration of OIDC_STORE_ACCESS_TOKEN

Signed-off-by: Fabre Florian <[email protected]>

Author: joehybird

docs/env.md

@@ -91,7 +91,7 @@ This document lists all configurable environment variables for the Drive applica
 | `SEARCH_INDEXER_MIMETYPES` | Find application endpoint for search | `None` |
 | `SEARCH_INDEXER_QUERY_URL` | Find application endpoint for search | `None` |
 | `SEARCH_INDEXER_SECRET` | Token for indexation queries | `None` |
-| `SEARCH_INDEXER_UPLOAD_MAX_SIZE` | Maximum size for an indexable file | `2097152` |
+| `SEARCH_INDEXER_CONTENT_MAX_SIZE` | Maximum size for an indexable file | `2097152` |
 | `SEARCH_INDEXER_URL` | Find application endpoint for indexation | `None` |
 | `SEARCH_INDEXER_QUERY_LIMIT` | Maximum number of results expected from search endpoint | 50 |
 | `SENTRY_DSN` | Sentry DSN for error tracking | `None` |

docs/setup-find.md

@@ -32,7 +32,7 @@ SEARCH_INDEXER_QUERY_LIMIT=50
 
 # Limit the mimetypes and size of indexable files
 SEARCH_INDEXER_ALLOWED_MIMETYPES=["text/"]
-SEARCH_INDEXER_UPLOAD_MAX_SIZE=2 * 2**20  # 2Mb
+SEARCH_INDEXER_CONTENT_MAX_SIZE=2 * 2**20  # 2Mb
 ```
 
 We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.

env.d/development/common

@@ -79,3 +79,7 @@ WOPI_SRC_BASE_URL=http://app-dev:8000
 SEARCH_INDEXER_SECRET=find-api-key-for-driv-with-exactly-50-chars-length   # Key generated by create_demo in Find app.
 SEARCH_INDEXER_URL="http://find:8000/api/v1.0/documents/index/"
 SEARCH_INDEXER_QUERY_URL="http://find:8000/api/v1.0/documents/search/"
+
+# Store OIDC tokens in the session
+OIDC_STORE_ACCESS_TOKEN = True
+OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session.

src/backend/core/api/viewsets.py

@@ -1009,6 +1009,10 @@ def search(self, request, *args, **kwargs):
          - A fulltext search through the opensearch indexation app "find" if the backend is
            enabled (see SEARCH_INDEXER_CLASS)
          - A filtering by the model fields 'title' & 'type'.
+
+        Note : Even if the indexer is disabled this view will do OIDC refresh calls
+        anyway. Think about using a decorator with args to prevent this when the
+        SEARCH_INDEXER_CLASS setting is not configured.
         """
         queryset = self.queryset
         indexer = get_file_indexer()

src/backend/core/services/search_indexers.py

@@ -154,7 +154,7 @@ def __init__(self):
         Initialize the indexer.
         """
         self.batch_size = settings.SEARCH_INDEXER_BATCH_SIZE
-        self.max_upload_size = settings.SEARCH_INDEXER_UPLOAD_MAX_SIZE
+        self.max_content_size = settings.SEARCH_INDEXER_CONTENT_MAX_SIZE
         self.indexer_url = settings.SEARCH_INDEXER_URL
         self.indexer_secret = settings.SEARCH_INDEXER_SECRET
         self.search_url = settings.SEARCH_INDEXER_QUERY_URL
@@ -297,15 +297,17 @@ def to_text(self, item):
 
         raise SuspiciousFileOperation(f"Unrecognized mimetype {mimetype}")
 
-    def has_text(self, item):
+    def can_serialize_content(self, item):
         """
         Return True if the file mimetype can be converted into text for indexation
         """
         mimetype = item.mimetype or ""
+        filesize = item.size or 0
 
         return (
             item.upload_state == models.ItemUploadStateChoices.READY
             and item.type == models.ItemTypeChoices.FILE
+            and filesize < self.max_content_size
             and is_allowed_mimetype(mimetype, self.allowed_mimetypes)
         )
 
@@ -329,7 +331,7 @@ def serialize_item(self, item, accesses):
 
         # There is no endpoint in Find API for inactive items so we index it
         # again with an empty content.
-        if is_active and self.has_text(item):
+        if is_active and self.can_serialize_content(item):
             content = self.to_text(item)
 
         return {

src/backend/core/tests/test_services_search_indexers.py

@@ -291,7 +291,7 @@ def test_services_is_allowed_mimetype():
         ),
     ],
 )
-def test_services_search_has_text(indexer_settings, kwargs, expected):
+def test_services_search_can_serialize_content(indexer_settings, kwargs, expected):
     """
     Only allowed mimetypes of uploaded files in ready state can have an indexable
     content.
@@ -313,7 +313,7 @@ def test_services_search_has_text(indexer_settings, kwargs, expected):
         **params,
     )
 
-    assert expected == SearchIndexer().has_text(item)
+    assert expected == SearchIndexer().can_serialize_content(item)
 
 
 @pytest.mark.usefixtures("indexer_settings")
@@ -646,6 +646,43 @@ def test_services_search_indexers_ignore_content_if_not_ready(mock_push):
     }
 
 
+@patch.object(SearchIndexer, "push")
+@pytest.mark.usefixtures("indexer_settings")
+def test_services_search_indexers_ignore_content_if_too_big(
+    mock_push, indexer_settings
+):
+    """
+    Should not fill the content data when the file is over the limit
+    setting SEARCH_INDEXER_CONTENT_MAX_SIZE
+    """
+    indexer_settings.SEARCH_INDEXER_CONTENT_MAX_SIZE = 50
+
+    item = factories.ItemFactory(
+        mimetype="text/plain",
+        type=models.ItemTypeChoices.FILE,
+        update_upload_state=models.ItemUploadStateChoices.READY,
+        upload_bytes="a" * 49,
+    )
+
+    # too big
+    too_big_item = factories.ItemFactory(
+        mimetype="text/plain",
+        type=models.ItemTypeChoices.FILE,
+        update_upload_state=models.ItemUploadStateChoices.READY,
+        upload_bytes="a" * 50,
+    )
+
+    assert SearchIndexer().index() == 2
+
+    assert mock_push.call_count == 1
+
+    results = {item["id"]: item["content"] for item in mock_push.call_args[0][0]}
+    assert results == {
+        str(item.id): "a" * 49,
+        str(too_big_item.id): "",
+    }
+
+
 @patch.object(SearchIndexer, "push")
 @pytest.mark.usefixtures("indexer_settings")
 def test_services_search_indexers_ancestors_link_reach(mock_push):

src/backend/drive/settings.py

@@ -124,9 +124,9 @@ class Base(Configuration):
     SEARCH_INDEXER_QUERY_LIMIT = values.PositiveIntegerValue(
         default=50, environ_name="SEARCH_INDEXER_QUERY_LIMIT", environ_prefix=None
     )
-    SEARCH_INDEXER_UPLOAD_MAX_SIZE = values.PositiveIntegerValue(
+    SEARCH_INDEXER_CONTENT_MAX_SIZE = values.PositiveIntegerValue(
         2 * (2**20),  # 2MB
-        environ_name="SEARCH_INDEXER_UPLOAD_MAX_SIZE",
+        environ_name="SEARCH_INDEXER_CONTENT_MAX_SIZE",
         environ_prefix=None,
     )
     SEARCH_INDEXER_ALLOWED_MIMETYPES = values.ListValue(