From a080993e97897c3ccd26e2e18a3ce378e8dae251 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:12:13 +0530 Subject: [PATCH 1/7] feat: create the suggestions API for suggestions --- .../discovery/ESBasedSuggestionService.java | 100 ++++++++++++++++++ .../apache/atlas/web/rest/DiscoveryREST.java | 11 +- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java new file mode 100644 index 0000000000..0cff416bc5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java @@ -0,0 +1,100 @@ +package org.apache.atlas.discovery; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.atlas.type.AtlasType; +import org.apache.http.util.EntityUtils; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.SuggestBuilders; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class ESBasedSuggestionService { + private static final Logger LOG = LoggerFactory.getLogger(ESBasedSuggestionService.class); + + private RestClient esRestClient; + + public static final String INDEX_NAME = "suggest"; + + public ESBasedSuggestionService(RestClient lowLevelClient) { + this.esRestClient = lowLevelClient; + } + public static SearchRequest buildSuggestQuery(String userQuery, String field, String suggestion_name, int fuzziness) { + SearchRequest searchRequest = new SearchRequest(INDEX_NAME); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + Fuzziness fuzzinessObj = Fuzziness.build(fuzziness); + CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(field) + .prefix(userQuery, fuzzinessObj); + + SuggestBuilder suggestBuilder = new SuggestBuilder() + .addSuggestion(suggestion_name, completionSuggestionBuilder); + + searchSourceBuilder.suggest(suggestBuilder); + searchRequest.source(searchSourceBuilder); + + return searchRequest; + } + + public SuggestionResponse searchSuggestions(String query, String fieldName, int fuzziness) throws IOException { + // Build the suggestor query for ES + String suggestorName = "suggest_keyword"; + SearchRequest searchRequest = buildSuggestQuery(query, fieldName, suggestorName, fuzziness); + String queryStr = searchRequest.source().toString(); + Request queryRequest = new Request("POST", "/suggest/_search"); + queryRequest.setJsonEntity(queryStr); + Response response = esRestClient.performRequest(queryRequest); + + SuggestionResponse suggestionResponse = new SuggestionResponse(); + String esResponseString = EntityUtils.toString(response.getEntity()); + + // Parse the response and return the suggestions + Map responseMap = AtlasType.fromJson(esResponseString, Map.class); + Map suggestMap = (Map) responseMap.get("suggest"); + ArrayList suggestionMap = (ArrayList) suggestMap.get(suggestorName); + LinkedHashMap suggestions = suggestionMap.get(0); + List options = (List) suggestions.get("options"); + for (LinkedHashMap option : options) { + suggestionResponse.addSuggestion((String) option.get("text")); + } + + return suggestionResponse; + } + + @JsonAutoDetect(getterVisibility=JsonAutoDetect.Visibility.PUBLIC_ONLY, setterVisibility=JsonAutoDetect.Visibility.PUBLIC_ONLY, fieldVisibility=JsonAutoDetect.Visibility.PUBLIC_ONLY) + @JsonSerialize(include=JsonSerialize.Inclusion.ALWAYS) + @JsonIgnoreProperties(ignoreUnknown=true) + public class SuggestionResponse { + + public SuggestionResponse() { } + private List suggestions = new ArrayList<>(); + + public List getSuggestions() { + return suggestions; + } + + public void addSuggestion(String suggestion) { + this.suggestions.add(suggestion); + } + + public void setSuggestions(List suggestions) { + this.suggestions = suggestions; + } + } + + +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 590e3cb0bf..423e36a119 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -25,6 +25,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.discovery.AtlasDiscoveryService; +import org.apache.atlas.discovery.ESBasedSuggestionService; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.*; @@ -34,6 +35,7 @@ import org.apache.atlas.model.searchlog.SearchLogSearchResult; import org.apache.atlas.model.searchlog.SearchRequestLogData.SearchRequestLogDataBuilder; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchDatabase; import org.apache.atlas.searchlog.SearchLoggingManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; @@ -92,6 +94,8 @@ public class DiscoveryREST { private final AtlasDiscoveryService discoveryService; private final SearchLoggingManagement loggerManagement; + private final ESBasedSuggestionService esBasedSuggestionService = new ESBasedSuggestionService(AtlasElasticsearchDatabase.getLowLevelClient()); + private static final String INDEXSEARCH_TAG_NAME = "indexsearch"; private static final Set TRACKING_UTM_TAGS = new HashSet<>(Arrays.asList("ui_main_list", "ui_popup_searchbar")); private static final String UTM_TAG_FROM_PRODUCT = "project_webapp"; @@ -829,7 +833,8 @@ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParam @Path("suggestions") @GET @Timed - public AtlasSuggestionsResult getSuggestions(@QueryParam("prefixString") String prefixString, @QueryParam("fieldName") String fieldName) { + public ESBasedSuggestionService.SuggestionResponse getSuggestions(@QueryParam("prefixString") String prefixString, @QueryParam("fieldName") String fieldName, + @QueryParam("fuzziness") int fuzziness) { AtlasPerfTracer perf = null; try { @@ -837,7 +842,9 @@ public AtlasSuggestionsResult getSuggestions(@QueryParam("prefixString") String perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSuggestions(" + prefixString + "," + fieldName + ")"); } - return discoveryService.getSuggestions(prefixString, fieldName); + return esBasedSuggestionService.searchSuggestions(prefixString, fieldName, fuzziness); + } catch (IOException e) { + throw new RuntimeException(e); } finally { AtlasPerfTracer.log(perf); } From 408e5caca56ea0cceaf2738c5b57e5942e2fd42f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:12:38 +0530 Subject: [PATCH 2/7] Update maven.yml --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..f0515e66cb 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - autocomplete jobs: build: From 3d6913a6fa89a19bdd2b7b6ae81d066bee0831ca Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 12 Aug 2024 11:52:48 +0530 Subject: [PATCH 3/7] feat: add direct query option --- .../discovery/ESBasedSuggestionService.java | 22 ++----------------- .../apache/atlas/web/rest/DiscoveryREST.java | 10 ++++----- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java index 0cff416bc5..e5b67ab3f1 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java @@ -33,29 +33,11 @@ public class ESBasedSuggestionService { public ESBasedSuggestionService(RestClient lowLevelClient) { this.esRestClient = lowLevelClient; } - public static SearchRequest buildSuggestQuery(String userQuery, String field, String suggestion_name, int fuzziness) { - SearchRequest searchRequest = new SearchRequest(INDEX_NAME); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - Fuzziness fuzzinessObj = Fuzziness.build(fuzziness); - CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(field) - .prefix(userQuery, fuzzinessObj); - - SuggestBuilder suggestBuilder = new SuggestBuilder() - .addSuggestion(suggestion_name, completionSuggestionBuilder); - - searchSourceBuilder.suggest(suggestBuilder); - searchRequest.source(searchSourceBuilder); - - return searchRequest; - } - - public SuggestionResponse searchSuggestions(String query, String fieldName, int fuzziness) throws IOException { + public SuggestionResponse searchSuggestions(String queryStr) throws IOException { // Build the suggestor query for ES String suggestorName = "suggest_keyword"; - SearchRequest searchRequest = buildSuggestQuery(query, fieldName, suggestorName, fuzziness); - String queryStr = searchRequest.source().toString(); - Request queryRequest = new Request("POST", "/suggest/_search"); + Request queryRequest = new Request("POST", "/autocomplete/_search"); queryRequest.setJsonEntity(queryStr); Response response = esRestClient.performRequest(queryRequest); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 423e36a119..7223ab0c06 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -39,6 +39,7 @@ import org.apache.atlas.searchlog.SearchLoggingManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; @@ -831,18 +832,17 @@ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParam } @Path("suggestions") - @GET + @POST @Timed - public ESBasedSuggestionService.SuggestionResponse getSuggestions(@QueryParam("prefixString") String prefixString, @QueryParam("fieldName") String fieldName, - @QueryParam("fuzziness") int fuzziness) { + public ESBasedSuggestionService.SuggestionResponse getSuggestions(Object queryStr) { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSuggestions(" + prefixString + "," + fieldName + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSuggestions(" + queryStr + ")"); } - return esBasedSuggestionService.searchSuggestions(prefixString, fieldName, fuzziness); + return esBasedSuggestionService.searchSuggestions(AtlasType.toJson(queryStr)); } catch (IOException e) { throw new RuntimeException(e); } finally { From a0dbc6cea1df5c675e9c60f453ed3561a46d0623 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:08:05 +0530 Subject: [PATCH 4/7] Update maven.yml --- .github/workflows/maven.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f0515e66cb..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - autocomplete + - lineageondemand jobs: build: @@ -58,7 +58,7 @@ jobs: [{ "id": "github", "username": "atlan-ci", - "password": "${{ secrets.my_pat }}" + "password": "${{ secrets.ORG_PAT_GITHUB }}" }] - name: Build with Maven @@ -77,7 +77,7 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx @@ -89,7 +89,7 @@ jobs: with: registry: ghcr.io username: $GITHUB_ACTOR - password: ${{ secrets.my_pat }} + password: ${{ secrets.ORG_PAT_GITHUB }} - name: Build and push id: docker_build From 6e8f8a658db1139d4798c7ee61bcb3d04f68edd5 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 19 Aug 2024 21:02:09 +0530 Subject: [PATCH 5/7] feat: update suggestion response --- .../discovery/ESBasedSuggestionService.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java index e5b67ab3f1..899361ec81 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java @@ -51,7 +51,7 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException LinkedHashMap suggestions = suggestionMap.get(0); List options = (List) suggestions.get("options"); for (LinkedHashMap option : options) { - suggestionResponse.addSuggestion((String) option.get("text")); + suggestionResponse.addSuggestion((LinkedHashMap) option.get("_source")); } return suggestionResponse; @@ -63,19 +63,25 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException public class SuggestionResponse { public SuggestionResponse() { } - private List suggestions = new ArrayList<>(); + private List suggestions = new ArrayList<>(); + private final List sources = new ArrayList<>(); - public List getSuggestions() { + public List getSuggestions() { return suggestions; } - public void addSuggestion(String suggestion) { + public void addSuggestion(LinkedHashMap suggestion) { this.suggestions.add(suggestion); } - public void setSuggestions(List suggestions) { + public void addSource(LinkedHashMap source) { + this.sources.add(source); + } + + public void setSuggestions(List suggestions) { this.suggestions = suggestions; } + } From 9b0f14a438c1d87907c057f69a88accb5447a3a4 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:22:44 +0530 Subject: [PATCH 6/7] feat: just fetch texts from the suggester --- .../discovery/ESBasedSuggestionService.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java index 899361ec81..ba83ed2c91 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java @@ -5,15 +5,9 @@ import org.apache.http.util.EntityUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +31,7 @@ public ESBasedSuggestionService(RestClient lowLevelClient) { public SuggestionResponse searchSuggestions(String queryStr) throws IOException { // Build the suggestor query for ES String suggestorName = "suggest_keyword"; - Request queryRequest = new Request("POST", "/autocomplete/_search"); + Request queryRequest = new Request("POST", "/autocomplete/_search?_source=false"); queryRequest.setJsonEntity(queryStr); Response response = esRestClient.performRequest(queryRequest); @@ -51,7 +45,7 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException LinkedHashMap suggestions = suggestionMap.get(0); List options = (List) suggestions.get("options"); for (LinkedHashMap option : options) { - suggestionResponse.addSuggestion((LinkedHashMap) option.get("_source")); + suggestionResponse.addSuggestion((String) option.get("text")); } return suggestionResponse; @@ -63,14 +57,14 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException public class SuggestionResponse { public SuggestionResponse() { } - private List suggestions = new ArrayList<>(); + private List suggestions = new ArrayList<>(); private final List sources = new ArrayList<>(); - public List getSuggestions() { + public List getSuggestions() { return suggestions; } - public void addSuggestion(LinkedHashMap suggestion) { + public void addSuggestion(String suggestion) { this.suggestions.add(suggestion); } @@ -78,7 +72,7 @@ public void addSource(LinkedHashMap source) { this.sources.add(source); } - public void setSuggestions(List suggestions) { + public void setSuggestions(List suggestions) { this.suggestions = suggestions; } From dc6f4e3c4632968c5de85e2d074baedbe953f3fb Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 4 Sep 2024 10:01:09 +0530 Subject: [PATCH 7/7] feat: add a way to just return the complete response to the user --- .../discovery/ESBasedSuggestionService.java | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java index ba83ed2c91..91fcdf0bee 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/ESBasedSuggestionService.java @@ -29,9 +29,7 @@ public ESBasedSuggestionService(RestClient lowLevelClient) { } public SuggestionResponse searchSuggestions(String queryStr) throws IOException { - // Build the suggestor query for ES - String suggestorName = "suggest_keyword"; - Request queryRequest = new Request("POST", "/autocomplete/_search?_source=false"); + Request queryRequest = new Request("POST", "/autocomplete/_search"); queryRequest.setJsonEntity(queryStr); Response response = esRestClient.performRequest(queryRequest); @@ -40,13 +38,7 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException // Parse the response and return the suggestions Map responseMap = AtlasType.fromJson(esResponseString, Map.class); - Map suggestMap = (Map) responseMap.get("suggest"); - ArrayList suggestionMap = (ArrayList) suggestMap.get(suggestorName); - LinkedHashMap suggestions = suggestionMap.get(0); - List options = (List) suggestions.get("options"); - for (LinkedHashMap option : options) { - suggestionResponse.addSuggestion((String) option.get("text")); - } + suggestionResponse.setResponseMap(responseMap); return suggestionResponse; } @@ -57,23 +49,14 @@ public SuggestionResponse searchSuggestions(String queryStr) throws IOException public class SuggestionResponse { public SuggestionResponse() { } - private List suggestions = new ArrayList<>(); - private final List sources = new ArrayList<>(); - - public List getSuggestions() { - return suggestions; - } - - public void addSuggestion(String suggestion) { - this.suggestions.add(suggestion); - } + private Map responseMap = new LinkedHashMap<>(); - public void addSource(LinkedHashMap source) { - this.sources.add(source); + public Map getResponseMap() { + return responseMap; } - public void setSuggestions(List suggestions) { - this.suggestions = suggestions; + public void setResponseMap(Map responseMap) { + this.responseMap = responseMap; } }