From 0779b9f3dbd4161c4c2d2cd2f2f77e0933f75c10 Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Thu, 15 Aug 2024 15:34:42 +0700 Subject: [PATCH] MAINT-2649 Allow RELEASE_LEASE to run the "EN_GB import" endpoint --- ...dditionalLanguageRefsetUpgradeService.java | 157 ++++++++++-------- 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/src/main/java/org/snomed/snowstorm/extension/ExtensionAdditionalLanguageRefsetUpgradeService.java b/src/main/java/org/snomed/snowstorm/extension/ExtensionAdditionalLanguageRefsetUpgradeService.java index 7143cae8a..a6d45d9a2 100644 --- a/src/main/java/org/snomed/snowstorm/extension/ExtensionAdditionalLanguageRefsetUpgradeService.java +++ b/src/main/java/org/snomed/snowstorm/extension/ExtensionAdditionalLanguageRefsetUpgradeService.java @@ -8,6 +8,7 @@ import io.kaicode.elasticvc.domain.Commit; import io.kaicode.elasticvc.domain.Metadata; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.snomed.snowstorm.core.data.domain.CodeSystem; @@ -19,9 +20,9 @@ import org.snomed.snowstorm.core.data.services.NotFoundException; import org.snomed.snowstorm.core.data.services.ReferenceSetMemberService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.SearchHitsIterator; -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder; import org.springframework.data.elasticsearch.core.query.FetchSourceFilter; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Service; @@ -30,10 +31,12 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool; +import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.range; import static com.google.common.collect.Iterables.partition; import static io.kaicode.elasticvc.api.ComponentService.LARGE_PAGE; -import static co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.*; -import static io.kaicode.elasticvc.helper.QueryHelper.*; +import static io.kaicode.elasticvc.helper.QueryHelper.termQuery; +import static io.kaicode.elasticvc.helper.QueryHelper.termsQuery; import static org.snomed.snowstorm.core.data.domain.Description.Fields.DESCRIPTION_ID; import static org.snomed.snowstorm.core.data.domain.Description.Fields.TYPE_ID; import static org.snomed.snowstorm.core.data.domain.ReferenceSetMember.Fields.*; @@ -67,7 +70,7 @@ public class ExtensionAdditionalLanguageRefsetUpgradeService { private final Logger logger = LoggerFactory.getLogger(ExtensionAdditionalLanguageRefsetUpgradeService.class); - @PreAuthorize("hasPermission('ADMIN', #codeSystem.branchPath)") + @PreAuthorize("hasPermission('ADMIN', #codeSystem.branchPath) || hasPermission('RELEASE_LEAD', #codeSystem.branchPath)") public void generateAdditionalLanguageRefsetDelta(CodeSystem codeSystem, String branchPath, String languageRefsetId, Boolean completeCopy) { logger.info("Start updating additional language refset on branch {} for {}.", branchPath, codeSystem); AdditionalRefsetExecutionConfig config = createExecutionConfig(codeSystem, completeCopy); @@ -166,57 +169,17 @@ private List addOrUpdateLanguageRefsetComponents(String bran /* Step 2 */ // Get all existing language reference set members - Map> existingLanguageRefsetMembersToUpdate = new Long2ObjectOpenHashMap<>(); - for (List batch : partition(languageRefsetMembersToCopy.keySet(), 10_000)) { - NativeQueryBuilder queryBuilder = new NativeQueryBuilder() - .withQuery(bool(b -> b - .must(branchCriteria.getEntityBranchCriteria(ReferenceSetMember.class)) - .must(termQuery(REFSET_ID, config.getDefaultEnglishLanguageRefsetId())) - .must(termsQuery(ACCEPTABILITY_ID_FIELD_PATH, List.of(Concepts.PREFERRED, Concepts.ACCEPTABLE))) - .must(termsQuery(ReferenceSetMember.Fields.CONCEPT_ID, batch))) - ).withPageable(LARGE_PAGE); - - try (final SearchHitsIterator searchHitsIterator = elasticsearchTemplate.searchForStream(queryBuilder.build(), ReferenceSetMember.class)) { - searchHitsIterator.forEachRemaining(hit -> { - existingLanguageRefsetMembersToUpdate.computeIfAbsent(Long.valueOf(hit.getContent().getConceptId()), k -> new ArrayList <>()).add(hit.getContent()); - descriptionIds.add(Long.valueOf(hit.getContent().getReferencedComponentId())); - }); - } - } + Map> existingLanguageRefsetMembersToUpdate = getAllExistingLanguageRefsetMembers(config, languageRefsetMembersToCopy, branchCriteria, descriptionIds); /* Step 3 */ // Get all relevant descriptions (requires only SYN and DEF) - Map> conceptToDescriptionsMap = new Long2ObjectOpenHashMap<>(); - for (List batch : partition(descriptionIds, 10_000)) { - NativeQueryBuilder queryBuilder = new NativeQueryBuilder() - .withQuery(bool(b -> b - .must(branchCriteria.getEntityBranchCriteria(Description.class)) - .must(termsQuery(TYPE_ID, List.of(Concepts.SYNONYM, Concepts.TEXT_DEFINITION))))) - .withFilter(termsQuery(DESCRIPTION_ID, batch)) - .withPageable(LARGE_PAGE); - - try (final SearchHitsIterator searchHitsIterator = elasticsearchTemplate.searchForStream(queryBuilder.build(), Description.class)) { - searchHitsIterator.forEachRemaining(hit -> conceptToDescriptionsMap.computeIfAbsent(Long.valueOf(hit.getContent().getConceptId()), k -> new HashSet<>()).add(hit.getContent())); - } - } + Map> conceptToDescriptionsMap = getAllRelevantDescriptions(descriptionIds, branchCriteria); /* Step 4 */ // Update the existing language reference set members if needed List result = new ArrayList<>(); Set memberIdsToSkipCopy = new HashSet<>(); - for (Long conceptId : existingLanguageRefsetMembersToUpdate.keySet()) { - List existingRefsetMembers = existingLanguageRefsetMembersToUpdate.get(conceptId); - if (languageRefsetMembersToCopy.containsKey(conceptId)) { - Map languageRefsetMembersToCopyMap = convertMembersListToMap(languageRefsetMembersToCopy.get(conceptId)); - Map existingLanguageRefsetMembersMap = convertMembersListToMap(existingRefsetMembers); - for (ReferenceSetMember existingRefsetMember : existingRefsetMembers) { - if (languageRefsetMembersToCopyMap.containsKey(existingRefsetMember.getReferencedComponentId()) - && isAbleToAddOrUpdate(existingRefsetMember.getReferencedComponentId(), config.getDefaultModuleId(), languageRefsetMembersToCopyMap, existingLanguageRefsetMembersMap, conceptToDescriptionsMap)) { - update(existingRefsetMember, languageRefsetMembersToCopyMap, result, memberIdsToSkipCopy); - } - } - } - } + updateExistingMembersIfNeeded(config, languageRefsetMembersToCopy, existingLanguageRefsetMembersToUpdate, conceptToDescriptionsMap, result, memberIdsToSkipCopy); Set updated = result.stream().map(ReferenceSetMember::getReferencedComponentId).collect(Collectors.toSet()); logger.info("{} existing components to be updated", updated.size()); @@ -224,6 +187,12 @@ && isAbleToAddOrUpdate(existingRefsetMember.getReferencedComponentId(), config.g logger.info("{} components to skip copy as change exists in extension already.", memberIdsToSkipCopy.size()); } // add new ones + addNewLanguageRefsetMembers(config, languageRefsetMembersToCopy, existingLanguageRefsetMembersToUpdate, memberIdsToSkipCopy, updated, conceptToDescriptionsMap, result); + logger.info("{} new components to be added", result.size() - updated.size()); + return result; + } + + private void addNewLanguageRefsetMembers(AdditionalRefsetExecutionConfig config, Map> languageRefsetMembersToCopy, Map> existingLanguageRefsetMembersToUpdate, Set memberIdsToSkipCopy, Set updated, Map> conceptToDescriptionsMap, List result) { for (Long conceptId : languageRefsetMembersToCopy.keySet()) { List toCopyLanguageRefsetMembers = languageRefsetMembersToCopy.get(conceptId); Map languageRefsetMembersToCopyMap = convertMembersListToMap(toCopyLanguageRefsetMembers); @@ -241,8 +210,62 @@ && isAbleToAddOrUpdate(toCopyMember.getReferencedComponentId(), config.getDefaul } } } - logger.info("{} new components to be added", result.size() - updated.size()); - return result; + } + + private void updateExistingMembersIfNeeded(AdditionalRefsetExecutionConfig config, Map> languageRefsetMembersToCopy, Map> existingLanguageRefsetMembersToUpdate, Map> conceptToDescriptionsMap, List result, Set memberIdsToSkipCopy) { + for (Long conceptId : existingLanguageRefsetMembersToUpdate.keySet()) { + List existingRefsetMembers = existingLanguageRefsetMembersToUpdate.get(conceptId); + if (languageRefsetMembersToCopy.containsKey(conceptId)) { + Map languageRefsetMembersToCopyMap = convertMembersListToMap(languageRefsetMembersToCopy.get(conceptId)); + Map existingLanguageRefsetMembersMap = convertMembersListToMap(existingRefsetMembers); + for (ReferenceSetMember existingRefsetMember : existingRefsetMembers) { + if (languageRefsetMembersToCopyMap.containsKey(existingRefsetMember.getReferencedComponentId()) + && isAbleToAddOrUpdate(existingRefsetMember.getReferencedComponentId(), config.getDefaultModuleId(), languageRefsetMembersToCopyMap, existingLanguageRefsetMembersMap, conceptToDescriptionsMap)) { + update(existingRefsetMember, languageRefsetMembersToCopyMap, result, memberIdsToSkipCopy); + } + } + } + } + } + + @NotNull + private Map> getAllRelevantDescriptions(Set descriptionIds, BranchCriteria branchCriteria) { + Map> conceptToDescriptionsMap = new Long2ObjectOpenHashMap<>(); + for (List batch : partition(descriptionIds, 10_000)) { + NativeQueryBuilder queryBuilder = new NativeQueryBuilder() + .withQuery(bool(b -> b + .must(branchCriteria.getEntityBranchCriteria(Description.class)) + .must(termsQuery(TYPE_ID, List.of(Concepts.SYNONYM, Concepts.TEXT_DEFINITION))))) + .withFilter(termsQuery(DESCRIPTION_ID, batch)) + .withPageable(LARGE_PAGE); + + try (final SearchHitsIterator searchHitsIterator = elasticsearchTemplate.searchForStream(queryBuilder.build(), Description.class)) { + searchHitsIterator.forEachRemaining(hit -> conceptToDescriptionsMap.computeIfAbsent(Long.valueOf(hit.getContent().getConceptId()), k -> new HashSet<>()).add(hit.getContent())); + } + } + return conceptToDescriptionsMap; + } + + @NotNull + private Map> getAllExistingLanguageRefsetMembers(AdditionalRefsetExecutionConfig config, Map> languageRefsetMembersToCopy, BranchCriteria branchCriteria, Set descriptionIds) { + Map> existingLanguageRefsetMembersToUpdate = new Long2ObjectOpenHashMap<>(); + for (List batch : partition(languageRefsetMembersToCopy.keySet(), 10_000)) { + NativeQueryBuilder queryBuilder = new NativeQueryBuilder() + .withQuery(bool(b -> b + .must(branchCriteria.getEntityBranchCriteria(ReferenceSetMember.class)) + .must(termQuery(REFSET_ID, config.getDefaultEnglishLanguageRefsetId())) + .must(termsQuery(ACCEPTABILITY_ID_FIELD_PATH, List.of(Concepts.PREFERRED, Concepts.ACCEPTABLE))) + .must(termsQuery(ReferenceSetMember.Fields.CONCEPT_ID, batch))) + ).withPageable(LARGE_PAGE); + + try (final SearchHitsIterator searchHitsIterator = elasticsearchTemplate.searchForStream(queryBuilder.build(), ReferenceSetMember.class)) { + searchHitsIterator.forEachRemaining(hit -> { + existingLanguageRefsetMembersToUpdate.computeIfAbsent(Long.valueOf(hit.getContent().getConceptId()), k -> new ArrayList <>()).add(hit.getContent()); + descriptionIds.add(Long.valueOf(hit.getContent().getReferencedComponentId())); + }); + } + } + return existingLanguageRefsetMembersToUpdate; } private boolean isAbleToAddOrUpdate(String referencedComponentId, String defaultModuleId, Map languageRefsetMembersToCopyMap, Map existingLanguageRefsetMembersMap, Map> conceptToDescriptionsMap) { @@ -250,27 +273,27 @@ private boolean isAbleToAddOrUpdate(String referencedComponentId, String default ReferenceSetMember languageReferenceSetMemberToCopy = languageRefsetMembersToCopyMap.get(referencedComponentId); if (languageReferenceSetMemberToCopy.isActive() && Concepts.PREFERRED.equals(languageReferenceSetMemberToCopy.getAdditionalField(ACCEPTABILITY_ID))) { Set descriptions = conceptToDescriptionsMap.get(Long.valueOf(languageReferenceSetMemberToCopy.getConceptId())); - if (descriptions != null) { - Description descriptionOfReferenceSetMemberToCopy = descriptions.stream().filter(description -> description.getDescriptionId().equals(referencedComponentId)).findFirst().orElse(null); - if (descriptionOfReferenceSetMemberToCopy != null) { - for (Description description : descriptions) { - if (!descriptionOfReferenceSetMemberToCopy.getDescriptionId().equals(description.getDescriptionId()) - && description.getModuleId().equals(defaultModuleId) - && descriptionOfReferenceSetMemberToCopy.getType().equals(description.getType()) /* make sure that we're checking for the same description type */) { - // Check if the extension descriptions already have any PREFERRED term. - // If yes, skip the EN-GB Reference Set add/update. - if (!description.getDescriptionId().equals(languageReferenceSetMemberToCopy.getReferencedComponentId())) { - ReferenceSetMember existing = existingLanguageRefsetMembersMap.get(description.getDescriptionId()); - if (existing != null && existing.isActive() && Concepts.PREFERRED.equals(existing.getAdditionalField(ACCEPTABILITY_ID))) { - return false; - } - } - } + return descriptions == null || (!isPreferredTermInExtensionFound(referencedComponentId, defaultModuleId, existingLanguageRefsetMembersMap, descriptions, languageReferenceSetMemberToCopy)); + } + return true; + } + return false; + } + + private boolean isPreferredTermInExtensionFound(String referencedComponentId, String defaultModuleId, Map existingLanguageRefsetMembersMap, Set descriptions, ReferenceSetMember languageReferenceSetMemberToCopy) { + Description descriptionOfReferenceSetMemberToCopy = descriptions.stream().filter(description -> description.getDescriptionId().equals(referencedComponentId)).findFirst().orElse(null); + if (descriptionOfReferenceSetMemberToCopy != null) { + for (Description description : descriptions) { + if (!descriptionOfReferenceSetMemberToCopy.getDescriptionId().equals(description.getDescriptionId()) + && description.getModuleId().equals(defaultModuleId) + && descriptionOfReferenceSetMemberToCopy.getType().equals(description.getType()) /* make sure that we're checking for the same description type */ && (!description.getDescriptionId().equals(languageReferenceSetMemberToCopy.getReferencedComponentId()))) { + ReferenceSetMember existing = existingLanguageRefsetMembersMap.get(description.getDescriptionId()); + if (existing != null && existing.isActive() && Concepts.PREFERRED.equals(existing.getAdditionalField(ACCEPTABILITY_ID))) { + return true; } - } + } } - return true; } return false; }