From 96271f01202b84fe36f3913db60a45dbc930cf6f Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 26 Jun 2024 09:48:24 -0500 Subject: [PATCH 1/3] update metadata index and add it to search --- backend/app/search/config.py | 4 +++- frontend/src/components/search/EmbeddedPublicSearch.tsx | 2 +- frontend/src/components/search/EmbeddedSearch.tsx | 2 +- frontend/src/components/search/PublicSearch.tsx | 7 ++++++- frontend/src/components/search/Search.tsx | 7 ++++++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/app/search/config.py b/backend/app/search/config.py index 99100c125..18d6c61c3 100644 --- a/backend/app/search/config.py +++ b/backend/app/search/config.py @@ -16,9 +16,11 @@ class IndexSettings: "dataset_id": {"type": "text", "index": False}, "folder_id": {"type": "text", "index": False}, "bytes": {"type": "long"}, - # metadata fields + # metadata fields cast to plain string to enable search "metadata": { "type": "object", + "dynamic": True, + "properties": {"*": {"type": "text"}}, }, } } diff --git a/frontend/src/components/search/EmbeddedPublicSearch.tsx b/frontend/src/components/search/EmbeddedPublicSearch.tsx index 85e09eefd..b79b2649f 100644 --- a/frontend/src/components/search/EmbeddedPublicSearch.tsx +++ b/frontend/src/components/search/EmbeddedPublicSearch.tsx @@ -46,7 +46,7 @@ export function EmbeddedPublicSearch() { showFilter={true} showClear renderNoSuggestion="No suggestions found." - dataField={["name", "description"]} + dataField={["name", "description", "creator.keyword", "metadata"]} // placeholder="Search for Dataset" innerClass={{ title: "search-title", diff --git a/frontend/src/components/search/EmbeddedSearch.tsx b/frontend/src/components/search/EmbeddedSearch.tsx index 3119fcd39..dd7216d52 100644 --- a/frontend/src/components/search/EmbeddedSearch.tsx +++ b/frontend/src/components/search/EmbeddedSearch.tsx @@ -46,7 +46,7 @@ export function EmbeddedSearch() { showFilter={true} showClear renderNoSuggestion="No suggestions found." - dataField={["name", "description", "creator.keyword"]} + dataField={["name", "description", "creator.keyword", "metadata"]} // placeholder="Search for Dataset" innerClass={{ title: "search-title", diff --git a/frontend/src/components/search/PublicSearch.tsx b/frontend/src/components/search/PublicSearch.tsx index deb16fa3b..6f7f2bb20 100644 --- a/frontend/src/components/search/PublicSearch.tsx +++ b/frontend/src/components/search/PublicSearch.tsx @@ -95,7 +95,12 @@ export function PublicSearch() { showFilter={true} showClear={true} renderNoSuggestion="No suggestions found." - dataField={["name", "description"]} + dataField={[ + "name", + "description", + "creator.keyword", + "metadata", + ]} fieldWeights={[3, 2, 1]} innerClass={{ title: "search-title", diff --git a/frontend/src/components/search/Search.tsx b/frontend/src/components/search/Search.tsx index e8773c3ac..0c621867a 100644 --- a/frontend/src/components/search/Search.tsx +++ b/frontend/src/components/search/Search.tsx @@ -122,7 +122,12 @@ export function Search() { showFilter={true} showClear={true} renderNoSuggestion="No suggestions found." - dataField={["name", "description", "creator.keyword"]} + dataField={[ + "name", + "description", + "creator.keyword", + "metadata", + ]} fieldWeights={[3, 2, 1]} innerClass={{ title: "search-title", From 34d72271792456ce03340da6d314ab9929f72b7d Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 26 Jun 2024 11:02:19 -0500 Subject: [PATCH 2/3] alternative ways to allow metadata searchable --- backend/app/models/search.py | 1 + backend/app/search/config.py | 5 +++-- backend/app/search/index.py | 4 ++++ frontend/src/components/search/EmbeddedPublicSearch.tsx | 8 +++++++- frontend/src/components/search/EmbeddedSearch.tsx | 8 +++++++- frontend/src/components/search/PublicSearch.tsx | 4 ++-- frontend/src/components/search/Search.tsx | 4 ++-- 7 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/app/models/search.py b/backend/app/models/search.py index 3c9aac6de..012e0657d 100644 --- a/backend/app/models/search.py +++ b/backend/app/models/search.py @@ -44,4 +44,5 @@ class ElasticsearchEntry(BaseModel): bytes: Optional[int] # metadata fields metadata: Optional[List[dict]] = [] + metadata_stringify: Optional[str] status: Optional[str] diff --git a/backend/app/search/config.py b/backend/app/search/config.py index 18d6c61c3..2a97387f1 100644 --- a/backend/app/search/config.py +++ b/backend/app/search/config.py @@ -16,12 +16,13 @@ class IndexSettings: "dataset_id": {"type": "text", "index": False}, "folder_id": {"type": "text", "index": False}, "bytes": {"type": "long"}, - # metadata fields cast to plain string to enable search + # metadata fields "metadata": { "type": "object", "dynamic": True, - "properties": {"*": {"type": "text"}}, }, + # metadata fields cast to plain string to enable search + "metadata_stringify": {"type": "text"}, } } diff --git a/backend/app/search/index.py b/backend/app/search/index.py index 60c512e0c..b9482d76a 100644 --- a/backend/app/search/index.py +++ b/backend/app/search/index.py @@ -1,3 +1,4 @@ +import json from typing import List, Optional, Union from app.config import settings @@ -51,6 +52,7 @@ async def index_dataset( downloads=dataset.downloads, user_ids=authorized_user_ids, metadata=metadata, + metadata_stringify=json.dumps(metadata), status=dataset_status, ).dict() @@ -103,6 +105,7 @@ async def index_file( folder_id=str(file.folder_id), bytes=file.bytes, metadata=metadata, + metadata_stringify=json.dumps(metadata), status=file.status, ).dict() if update: @@ -198,6 +201,7 @@ async def index_thumbnail( folder_id=str(file.folder_id), bytes=thumbnail.bytes, metadata=metadata, + metadata_stringify=json.dumps(metadata), downloads=thumbnail.downloads, ).dict() if update: diff --git a/frontend/src/components/search/EmbeddedPublicSearch.tsx b/frontend/src/components/search/EmbeddedPublicSearch.tsx index b79b2649f..f47fe8f08 100644 --- a/frontend/src/components/search/EmbeddedPublicSearch.tsx +++ b/frontend/src/components/search/EmbeddedPublicSearch.tsx @@ -46,7 +46,13 @@ export function EmbeddedPublicSearch() { showFilter={true} showClear renderNoSuggestion="No suggestions found." - dataField={["name", "description", "creator.keyword", "metadata"]} + dataField={[ + "name", + "description", + "metadata_stringify", + "creator.keyword", + ]} + fieldWeights={[3, 2, 2, 1]} // placeholder="Search for Dataset" innerClass={{ title: "search-title", diff --git a/frontend/src/components/search/EmbeddedSearch.tsx b/frontend/src/components/search/EmbeddedSearch.tsx index dd7216d52..2f29decf1 100644 --- a/frontend/src/components/search/EmbeddedSearch.tsx +++ b/frontend/src/components/search/EmbeddedSearch.tsx @@ -46,7 +46,13 @@ export function EmbeddedSearch() { showFilter={true} showClear renderNoSuggestion="No suggestions found." - dataField={["name", "description", "creator.keyword", "metadata"]} + dataField={[ + "name", + "description", + "metadata_stringify", + "creator.keyword", + ]} + fieldWeights={[3, 2, 2, 1]} // placeholder="Search for Dataset" innerClass={{ title: "search-title", diff --git a/frontend/src/components/search/PublicSearch.tsx b/frontend/src/components/search/PublicSearch.tsx index 6f7f2bb20..44011eddc 100644 --- a/frontend/src/components/search/PublicSearch.tsx +++ b/frontend/src/components/search/PublicSearch.tsx @@ -98,10 +98,10 @@ export function PublicSearch() { dataField={[ "name", "description", + "metadata_stringify", "creator.keyword", - "metadata", ]} - fieldWeights={[3, 2, 1]} + fieldWeights={[3, 2, 2, 1]} innerClass={{ title: "search-title", input: "search-input", diff --git a/frontend/src/components/search/Search.tsx b/frontend/src/components/search/Search.tsx index 0c621867a..51bd2468e 100644 --- a/frontend/src/components/search/Search.tsx +++ b/frontend/src/components/search/Search.tsx @@ -125,10 +125,10 @@ export function Search() { dataField={[ "name", "description", + "metadata_text", "creator.keyword", - "metadata", ]} - fieldWeights={[3, 2, 1]} + fieldWeights={[3, 2, 2, 1]} innerClass={{ title: "search-title", input: "search-input", From 7b35c2f9c2028906917befc80495d31c66b2af84 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 26 Jun 2024 11:09:13 -0500 Subject: [PATCH 3/3] mising one spot --- frontend/src/components/search/Search.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/search/Search.tsx b/frontend/src/components/search/Search.tsx index 51bd2468e..27de9183d 100644 --- a/frontend/src/components/search/Search.tsx +++ b/frontend/src/components/search/Search.tsx @@ -125,7 +125,7 @@ export function Search() { dataField={[ "name", "description", - "metadata_text", + "metadata_stringify", "creator.keyword", ]} fieldWeights={[3, 2, 2, 1]}