implement real time suggestion

.github/workflows/ci-cd.yaml

@@ -242,6 +242,7 @@ jobs:
         run: |
           # Backend
           echo "DJANGO_ALGOLIA_APPLICATION_ID=${{ secrets.DJANGO_ALGOLIA_APPLICATION_ID }}" > .env.backend
+          echo "DJANGO_ALGOLIA_APPLICATION_REGION=${{ secrets.DJANGO_ALGOLIA_APPLICATION_REGION }}" >> .env.backend
           echo "DJANGO_ALGOLIA_WRITE_API_KEY=${{ secrets.DJANGO_ALGOLIA_WRITE_API_KEY }}" >> .env.backend
           echo "DJANGO_ALLOWED_HOSTS=${{ secrets.DJANGO_ALLOWED_HOSTS }}" >> .env.backend
           echo "DJANGO_AWS_ACCESS_KEY_ID=${{ secrets.DJANGO_AWS_ACCESS_KEY_ID }}" >> .env.backend

CONTRIBUTING.md

@@ -81,6 +81,7 @@ Follow these steps to set up the OWASP Nest application:
       ```plaintext
       DJANGO_ALGOLIA_APPLICATION_ID=<your-algolia-application-id>
       DJANGO_ALGOLIA_WRITE_API_KEY=<your-algolia-write-api-key>
+      DJANGO_ALGOLIA_APPLICATION_REGION=<your-algolia-application-region> // eu or us
       ```
 
     - Update your `frontend/.env` file with the following keys from your Algolia app (use **search** API key for frontend):

backend/.env.example

@@ -1,4 +1,5 @@
 DJANGO_ALGOLIA_APPLICATION_ID=None
+DJANGO_ALGOLIA_APPLICATION_REGION=None
 DJANGO_ALGOLIA_WRITE_API_KEY=None
 DJANGO_ALLOWED_HOSTS=*
 DJANGO_AWS_ACCESS_KEY_ID=None

backend/Makefile

@@ -36,6 +36,7 @@ index-data:
 	@echo "Indexing Nest data"
 	@CMD="poetry run python manage.py algolia_reindex" $(MAKE) exec-backend-command
 	@CMD="poetry run python manage.py algolia_update_synonyms" $(MAKE) exec-backend-command
+	@CMD="poetry run python manage.py algolia_update_suggestions" $(MAKE) exec-backend-command
 
 load-data:
 	@echo "Loading Nest data"

backend/apps/common/management/commands/algolia_update_suggestions.py

@@ -0,0 +1,105 @@
+"""A command to update OWASP Nest suggestions index."""
+
+from algoliasearch.query_suggestions.client import QuerySuggestionsClientSync
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Create query suggestions for Algolia indices"
+
+    def handle(self, *args, **kwargs):
+        client = QuerySuggestionsClientSync(
+            settings.ALGOLIA_APPLICATION_ID,
+            settings.ALGOLIA_WRITE_API_KEY,
+            settings.ALGOLIA_APPLICATION_REGION,
+        )
+
+        entity_configs = {
+            "chapters": {
+                "facets": [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_tags"},
+                    {"attribute": "idx_country"},
+                    {"attribute": "idx_region"},
+                    {"attribute": "idx_suggested_location"},
+                ],
+                "generate": [
+                    ["idx_name"],
+                    ["idx_tags"],
+                    ["idx_country"],
+                    ["idx_region"],
+                    ["idx_suggested_location"],
+                ],
+            },
+            "committees": {
+                "facets": [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_tags"},
+                ],
+                "generate": [
+                    ["idx_name"],
+                    ["idx_tags"],
+                ],
+            },
+            "issues": {
+                "facets": [
+                    {"attribute": "idx_title"},
+                    {"attribute": "idx_project_name"},
+                    {"attribute": "idx_repository_name"},
+                    {"attribute": "idx_project_tags"},
+                    {"attribute": "idx_repository_topics"},
+                ],
+                "generate": [
+                    ["idx_title"],
+                    ["idx_project_name"],
+                    ["idx_repository_name"],
+                    ["idx_project_tags"],
+                    ["idx_repository_topics"],
+                ],
+            },
+            "projects": {
+                "facets": [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_repository_names"},
+                    {"attribute": "idx_tags"},
+                ],
+                "generate": [
+                    ["idx_name"],
+                    ["idx_tags"],
+                    ["idx_repository_names"],
+                ],
+            },
+            "users": {
+                "facets": [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_title"},
+                ],
+                "generate": [
+                    ["idx_name"],
+                    ["idx_title"],
+                ],
+            },
+        }
+
+        for entity, suggestion_settings in entity_configs.items():
+            source_index_name = f"{settings.ENVIRONMENT.lower()}_{entity}"
+            suggestions_index_name = f"{settings.ENVIRONMENT.lower()}_{entity}_suggestions"
+
+            configuration = {
+                "sourceIndices": [
+                    {
+                        "indexName": source_index_name,
+                        **suggestion_settings,
+                    }
+                ]
+            }
+            client.update_config(
+                index_name=suggestions_index_name,
+                configuration=configuration,
+            )
+            print(f"Updated query suggestions index for {entity.capitalize()}")

backend/apps/github/index/issue.py

@@ -39,6 +39,13 @@ class IssueIndex(AlgoliaIndex, IndexBase):
     )
 
     settings = {
+        "attributesForFaceting": [
+            "idx_title",
+            "idx_project_name",
+            "idx_repository_name",
+            "idx_project_tags",
+            "idx_repository_topics",
+        ],
         "attributeForDistinct": "idx_project_name",
         "minProximity": 4,
         "indexLanguages": ["en"],

backend/apps/github/index/user.py

@@ -32,6 +32,11 @@ class UserIndex(AlgoliaIndex, IndexBase):
     )
 
     settings = {
+        "attributesForFaceting": [
+            "idx_key",
+            "idx_name",
+            "idx_title",
+        ],
         "attributeForDistinct": "idx_login",
         "minProximity": 4,
         "customRanking": [

backend/apps/owasp/index/chapter.py

@@ -34,6 +34,9 @@ class ChapterIndex(AlgoliaIndex):
     settings = {
         "attributesForFaceting": [
             "filterOnly(idx_key)",
+            "idx_country",
+            "idx_name",
+            "idx_tags",
         ],
         "indexLanguages": ["en"],
         "customRanking": [

backend/apps/owasp/index/committee.py

@@ -28,6 +28,8 @@ class CommitteeIndex(AlgoliaIndex):
     settings = {
         "attributesForFaceting": [
             "filterOnly(idx_key)",
+            "idx_name",
+            "idx_tags",
         ],
         "indexLanguages": ["en"],
         "customRanking": [

backend/apps/owasp/index/project.py

@@ -41,6 +41,9 @@ class ProjectIndex(AlgoliaIndex, IndexBase):
     settings = {
         "attributesForFaceting": [
             "filterOnly(idx_key)",
+            "idx_name",
+            "idx_tags",
+            "idx_repository_names",
         ],
         "indexLanguages": ["en"],
         "customRanking": [

backend/settings/base.py

@@ -91,6 +91,7 @@ class Base(Configuration):
     WSGI_APPLICATION = "wsgi.application"
 
     ALGOLIA_APPLICATION_ID = values.SecretValue(environ_name="ALGOLIA_APPLICATION_ID")
+    ALGOLIA_APPLICATION_REGION = values.SecretValue(environ_name="ALGOLIA_APPLICATION_REGION")
     ALGOLIA_WRITE_API_KEY = values.SecretValue(environ_name="ALGOLIA_WRITE_API_KEY")
 
     ALGOLIA = {

backend/tests/common/management/commands/algolia_update_synonyms_tests.py

@@ -2,30 +2,122 @@
 
 import pytest
 
-from apps.common.management.commands.algolia_update_synonyms import Command
-
-
-@pytest.mark.parametrize(
-    "indexes",
-    [["IssueIndex", "ProjectIndex"]],
-)
-class TestAlgoliaUpdateSynonyms:
-    @patch("builtins.print")
-    @patch("apps.common.management.commands.algolia_update_synonyms.ProjectIndex")
-    @patch("apps.common.management.commands.algolia_update_synonyms.IssueIndex")
-    def test_handle(self, mock_issue_index, mock_project_index, mock_print, indexes):
-        mock_indexes = {
-            "IssueIndex": mock_issue_index,
-            "ProjectIndex": mock_project_index,
-        }
-        for index_name, index_instance in mock_indexes.items():
-            index_instance.update_synonyms = MagicMock()
-            index_instance.index_name = index_name
+from apps.common.management.commands.algolia_update_suggestions import Command
+
+
+class TestUpdateSuggestionsCommand:
+    @pytest.mark.parametrize(
+        ("entity", "facets", "generate"),
+        [
+            (
+                "chapters",
+                [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_tags"},
+                    {"attribute": "idx_country"},
+                    {"attribute": "idx_region"},
+                    {"attribute": "idx_suggested_location"},
+                ],
+                [
+                    ["idx_name"],
+                    ["idx_tags"],
+                    ["idx_country"],
+                    ["idx_region"],
+                    ["idx_suggested_location"],
+                ],
+            ),
+            (
+                "committees",
+                [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_tags"},
+                ],
+                [
+                    ["idx_name"],
+                    ["idx_tags"],
+                ],
+            ),
+            (
+                "issues",
+                [
+                    {"attribute": "idx_title"},
+                    {"attribute": "idx_project_name"},
+                    {"attribute": "idx_repository_name"},
+                    {"attribute": "idx_project_tags"},
+                    {"attribute": "idx_repository_topics"},
+                ],
+                [
+                    ["idx_title"],
+                    ["idx_project_name"],
+                    ["idx_repository_name"],
+                    ["idx_project_tags"],
+                    ["idx_repository_topics"],
+                ],
+            ),
+            (
+                "projects",
+                [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_repository_names"},
+                    {"attribute": "idx_tags"},
+                ],
+                [
+                    ["idx_name"],
+                    ["idx_tags"],
+                    ["idx_repository_names"],
+                ],
+            ),
+            (
+                "users",
+                [
+                    {"attribute": "idx_key"},
+                    {"attribute": "idx_name"},
+                    {"attribute": "idx_title"},
+                ],
+                [
+                    ["idx_name"],
+                    ["idx_title"],
+                ],
+            ),
+        ],
+    )
+    @patch("apps.common.management.commands.algolia_update_suggestions.settings")
+    @patch("apps.common.management.commands.algolia_update_suggestions.QuerySuggestionsClientSync")
+    def test_handle(self, mock_query_suggestions_client, mock_settings, entity, facets, generate):
+        # mocks
+        mock_settings.ENVIRONMENT = "testenv"
+        mock_settings.ALGOLIA_APPLICATION_ID = "mock_app_id"
+        mock_settings.ALGOLIA_WRITE_API_KEY = "mock_api_key"
+        mock_settings.ALGOLIA_APPLICATION_REGION = "eu"
+
+        mock_client = MagicMock()
+        mock_query_suggestions_client.return_value = mock_client
+
+        mock_client.update_config.return_value = None
+
+        expected_call_count = 5
+        # Act
 
         command = Command()
         command.handle()
 
-        for index_name in indexes:
-            index_instance = mock_indexes[index_name]
-            index_instance.update_synonyms.assert_called_once()
-            mock_print.assert_any_call(f"Updated {index_name.capitalize()} synonyms")
+        mock_query_suggestions_client.assert_called_once_with("mock_app_id", "mock_api_key", "eu")
+
+        mock_client.update_config.assert_any_call(
+            index_name=f"testenv_{entity}_suggestions",
+            configuration={
+                "sourceIndices": [
+                    {
+                        "indexName": f"testenv_{entity}",
+                        "facets": facets,
+                        "generate": generate,
+                    }
+                ]
+            },
+        )
+
+        # Use constant for the expected call count
+        assert mock_client.update_config.call_count == expected_call_count

frontend/Makefile

@@ -17,5 +17,5 @@ shell-frontend:
 	@CMD="/bin/bash" $(MAKE) exec-frontend-command-it
 
 test-frontend:
-	@docker build -f frontend/Dockerfile.test frontend -t nest-frontend-test 2>/dev/null
-	@docker run nest-frontend-test npm run test
+	@docker build -f frontend/Dockerfile.test frontend -t nest-frontend-test
+	@docker run --env-file frontend/.env.example nest-frontend-test npm run test

frontend/__tests__/src/pages/Chapters.test.tsx

@@ -91,12 +91,10 @@ describe('ChaptersPage Component', () => {
     const loadingSpinner = screen.getAllByAltText('Loading indicator')
     await waitFor(() => {
       expect(loadingSpinner.length).toBeGreaterThan(0)
-      expect(screen.queryByPlaceholderText('Search for OWASP chapters...')).not.toBeInTheDocument()
       expect(screen.queryByText('Next Page')).not.toBeInTheDocument()
     })
     await waitFor(() => {
       expect(screen.getByPlaceholderText('Search for OWASP chapters...')).toBeInTheDocument()
-      expect(screen.getByPlaceholderText('Search for OWASP chapters...')).toHaveFocus()
       expect(screen.getByText('Chapter 1')).toBeInTheDocument()
       expect(screen.getByText('Next Page')).toBeInTheDocument()
     })

frontend/__tests__/src/pages/Committees.test.tsx

@@ -56,15 +56,12 @@ describe('Committees Component', () => {
     const loadingSpinner = screen.getAllByAltText('Loading indicator')
     await waitFor(() => {
       expect(loadingSpinner.length).toBeGreaterThan(0)
-      expect(
-        screen.queryByPlaceholderText('Search for OWASP committees...')
-      ).not.toBeInTheDocument()
+
       expect(screen.queryByText('Next Page')).not.toBeInTheDocument()
     })
 
     await waitFor(() => {
       expect(screen.getByPlaceholderText('Search for OWASP committees...')).toBeInTheDocument()
-      expect(screen.getByPlaceholderText('Search for OWASP committees...')).toHaveFocus()
       expect(screen.getByText('Committee 1')).toBeInTheDocument()
       expect(screen.getByText('Next Page')).toBeInTheDocument()
     })

frontend/__tests__/src/pages/Projects.test.tsx

@@ -55,12 +55,10 @@ describe('ProjectPage Component', () => {
     const loadingSpinner = screen.getAllByAltText('Loading indicator')
     await waitFor(() => {
       expect(loadingSpinner.length).toBeGreaterThan(0)
-      expect(screen.queryByPlaceholderText('Search for OWASP projects...')).not.toBeInTheDocument()
       expect(screen.queryByText('Next Page')).not.toBeInTheDocument()
     })
     await waitFor(() => {
       expect(screen.getByPlaceholderText('Search for OWASP projects...')).toBeInTheDocument()
-      expect(screen.getByPlaceholderText('Search for OWASP projects...')).toHaveFocus()
       expect(screen.getByText('Project 1')).toBeInTheDocument()
       expect(screen.getByText('Next Page')).toBeInTheDocument()
     })