Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Read Nested Property Mappings #46

Merged
merged 2 commits into from
Feb 10, 2025
Merged

Read Nested Property Mappings #46

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
204d2ac
collectPage now reads a Map of property mappings and can map nested p…
JoonLeeNIH Jan 15, 2025
df05f80
Restructured property mappings between GraphQL and Opensearch
JoonLeeNIH Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 56 additions & 16 deletions service/ESService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,17 +310,17 @@ public int getTotalHits(JsonObject jsonObject) {
return jsonObject.get("hits").getAsJsonObject().get("total").getAsJsonObject().get("value").getAsInt();
}

public List<Map<String, Object>> collectPage(Map<String, Object> params, String endpoint, String[][] properties) throws IOException {
public List<Map<String, Object>> collectPage(Map<String, Object> params, String endpoint, List<Map<String, Object>> properties) throws IOException {
Map<String, Object> query = buildListQuery(params, Set.of(),false);
Request request = new Request("GET", endpoint);
return collectPage(request, query, properties);
}

public List<Map<String, Object>> collectPage(Request request, Map<String, Object> query, String[][] properties) throws IOException {
public List<Map<String, Object>> collectPage(Request request, Map<String, Object> query, List<Map<String, Object>> properties) throws IOException {
return collectPage(request, query, properties, ESService.MAX_ES_SIZE, 0);
}

public List<Map<String, Object>> collectPage(Request request, Map<String, Object> query, String[][] properties, int pageSize, int offset) throws IOException {
public List<Map<String, Object>> collectPage(Request request, Map<String, Object> query, List<Map<String, Object>> properties, int pageSize, int offset) throws IOException {
// Make sure page size is less than max allowed size
if (pageSize > MAX_ES_SIZE) {
throw new IOException("Parameter 'first' must not exceeded " + MAX_ES_SIZE);
Expand Down Expand Up @@ -352,7 +352,7 @@ public List<Map<String, Object>> collectPage(Request request, Map<String, Object
* @throws IOException
*/
private List<Map<String, Object>> collectPageWithScroll(
Request request, Map<String, Object> query, String[][] properties, int pageSize, int offset) throws IOException {
Request request, Map<String, Object> query, List<Map<String, Object>> properties, int pageSize, int offset) throws IOException {
query.put("size", SCROLL_SIZE);
String jsonizedQuery = gson.toJson(query);
request.setJsonEntity(jsonizedQuery);
Expand Down Expand Up @@ -428,15 +428,25 @@ private JsonObject rollToPage(Request request, int pageSize, int offset) throws
}

// Collect a page of data, result will be of pageSize or less if not enough data remains
public List<Map<String, Object>> collectPage(JsonObject jsonObject, String[][] properties, int pageSize) throws IOException {
public List<Map<String, Object>> collectPage(JsonObject jsonObject, List<Map<String, Object>> properties, int pageSize) throws IOException {
return collectPage(jsonObject, properties, pageSize, 0);
}

private List<Map<String, Object>> collectPage(JsonObject jsonObject, String[][] properties, int pageSize, int offset) throws IOException {
private List<Map<String, Object>> collectPage(JsonObject jsonObject, List<Map<String, Object>> properties, int pageSize, int offset) throws IOException {
return collectPage(jsonObject, properties, null, pageSize, offset);
}

public List<Map<String, Object>> collectPage(JsonObject jsonObject, String[][] properties, String[][] highlights, int pageSize, int offset) throws IOException {
/**
* Transforms Opensearch results to a desired mapping
* @param jsonObject The Opensearch results
* @param properties The desired mapping of GraphQL properties to Opensearch properties
* @param highlights A mapping of properties whose results need to be formatted for highlighting
* @param pageSize The maximum number of results to return
* @param offset How many results to skip at first
* @return
* @throws IOException
*/
public List<Map<String, Object>> collectPage(JsonObject jsonObject, List<Map<String, Object>> properties, String[][] highlights, int pageSize, int offset) throws IOException {
List<Map<String, Object>> data = new ArrayList<>();

JsonArray searchHits = jsonObject.getAsJsonObject("hits").getAsJsonArray("hits");
Expand All @@ -445,13 +455,21 @@ public List<Map<String, Object>> collectPage(JsonObject jsonObject, String[][] p
if (i + 1 <= offset) {
continue;
}

Map<String, Object> row = new HashMap<>();
for (String[] prop: properties) {
String propName = prop[0];
String dataField = prop[1];
JsonElement element = searchHits.get(i).getAsJsonObject().get("_source").getAsJsonObject().get(dataField);
row.put(propName, getValue(element));
for (Map<String, Object> prop: properties) {
List<Map<String, String>> nestedProps = prop.containsKey("nested") ? (List<Map<String, String>>) prop.get("nested") : null; // Mapping of nested property names
String propNameGql = (String) prop.get("gqlName"); // Name of the property in GraphQL
String propNameOs = (String) prop.get("osName"); // Name of the property in Opensearch document
JsonElement element = searchHits.get(i).getAsJsonObject().get("_source").getAsJsonObject().get(propNameOs);

if (nestedProps == null) {
row.put(propNameGql, getValue(element));
} else {
row.put(propNameGql, getValue(element, nestedProps));
}
}

if (highlights != null) {
for (String[] highlight: highlights) {
String hlName = highlight[0];
Expand All @@ -462,7 +480,9 @@ public List<Map<String, Object>> collectPage(JsonObject jsonObject, String[][] p
}
}
}

data.add(row);

if (data.size() >= pageSize) {
break;
}
Expand Down Expand Up @@ -498,16 +518,36 @@ public List<Map<String, Object>> getFilteredGroupCount(Map<String, Object> param
return groupCounts;
}

// Convert JsonElement into Java collections and primitives
// Wrapper method for getValue()
private Object getValue(JsonElement element) {
return getValue(element, null);
}

/**
* Convert JsonElement into Java collections and primitives
* @param element The JsonElement to transform
* @param propNames Property name mapping for the JsonElement
* @return
*/
private Object getValue(JsonElement element, List<Map<String, String>> propNames) {
Object value = null;
if (element == null || element.isJsonNull()) {
return null;
} else if (element.isJsonObject()) {
}

if (element.isJsonObject()) {
value = new HashMap<String, Object>();
JsonObject object = element.getAsJsonObject();
for (String key: object.keySet()) {
((Map<String, Object>) value).put(key, getValue(object.get(key)));
if (propNames != null) { // Nested properties need to be renamed
for (Map<String, String> propMap: propNames) {
String gqlName = propMap.get("gqlName");
String osName = propMap.get("osName");
((Map<String, Object>) value).put(gqlName, getValue(object.get(osName)));
}
} else { // Nested properties don't need to be renamed
for (String key: object.keySet()) {
((Map<String, Object>) value).put(key, getValue(object.get(key)));
}
}
} else if (element.isJsonArray()) {
value = new ArrayList<>();
Expand Down