From af4f1781b93ebdd4b1be96d855a4b42aeae7e081 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Tue, 29 Nov 2022 15:26:35 +0100 Subject: [PATCH 01/44] #4376 WIP. StudyTableSource interface added. New property sourceURN replaces opal project/table in dataset's study tables and variables. Requires ES index config update. --- .../mica/core/domain/BaseStudyTable.java | 81 ++++++++- .../org/obiba/mica/core/domain/OpalTable.java | 88 --------- .../mica/core/domain/OpalTableSource.java | 113 ++++++++++++ .../obiba/mica/core/domain/StudyTable.java | 6 +- .../StudyTableSourceServiceRegistry.java | 50 ++++++ .../obiba/mica/dataset/domain/Dataset.java | 2 +- .../mica/dataset/domain/DatasetVariable.java | 99 +++++------ .../dataset/domain/HarmonizationDataset.java | 15 +- .../service/CollectedDatasetService.java | 36 +--- .../mica/dataset/service/DatasetService.java | 118 +++---------- .../service/HarmonizedDatasetService.java | 167 ++++++------------ .../dataset/service/VariableSetService.java | 33 ++-- .../mica/micaConfig/service/CacheService.java | 2 +- .../micaConfig/service/PluginsService.java | 16 +- .../org/obiba/mica/web/model/DatasetDtos.java | 143 ++++++++------- .../java/org/obiba/mica/web/model/Dtos.java | 12 +- .../service/CollectedDatasetServiceTest.java | 4 +- .../service/HarmonizedDatasetServiceTest.java | 4 +- .../support/HarmonizedDatasetHelperTest.java | 6 +- .../obiba/mica/web/model/DatasetDtosTest.java | 4 +- .../rest/PublishedDatasetResource.java | 4 +- .../DraftCollectedDatasetResource.java | 7 - ...DraftCollectedDatasetVariableResource.java | 6 - .../entity/rql/RQLCriteriaOpalConverter.java | 3 +- .../rest/entity/rql/RQLFieldReferences.java | 2 +- ...raftDataschemaDatasetVariableResource.java | 18 +- .../DraftHarmonizedDatasetResource.java | 7 - ...raftHarmonizedDatasetVariableResource.java | 22 +-- .../DraftDatasetVariableResource.java | 3 +- .../AbstractPublishedDatasetResource.java | 76 ++++---- ...ishedCollectedDatasetVariableResource.java | 9 - .../harmonization/CsvContingencyWriter.java | 5 +- .../CsvHarmonizationVariablesWriter.java | 5 +- ...shedDataschemaDatasetVariableResource.java | 43 +---- .../PublishedHarmonizedDatasetResource.java | 7 +- ...shedHarmonizedDatasetVariableResource.java | 60 ++----- .../PublishedDatasetVariableResource.java | 3 +- .../mica/spi/dataset/StudyTableSource.java | 53 ++++++ .../spi/dataset/StudyTableSourceService.java | 24 +++ .../StudyTableSourceServiceLoader.java | 28 +++ .../mica/spi/search/SearchEngineService.java | 2 +- mica-web-model/src/main/protobuf/Mica.proto | 11 +- .../web/controller/VariableController.java | 21 ++- 43 files changed, 704 insertions(+), 714 deletions(-) delete mode 100644 mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java index b85fb007f5..15ebe196d1 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java @@ -13,9 +13,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Strings; -import java.io.Serializable; - -public class BaseStudyTable extends OpalTable { +public class BaseStudyTable { protected String studyId; @@ -23,6 +21,22 @@ public class BaseStudyTable extends OpalTable { protected int populationWeight; + private LocalizedString name; + + private LocalizedString description; + + private LocalizedString additionalInformation; + + private int weight; + + private String sourceURN; + + // legacy + private String project; + + // legacy + private String table; + public String getStudyId() { return studyId; } @@ -72,13 +86,66 @@ public void setPopulationWeight(int populationWeight) { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("project", getProject()).add("table", getTable()) + return MoreObjects.toStringHelper(this).add("source", getSourceURN()) .add("studyId", getStudyId()).add("populationId", getPopulationId()) .toString(); } - @Override - protected String getEntityId() { - return studyId; + public void setName(LocalizedString name) { + this.name = name; + } + + public LocalizedString getName() { + return name; + } + + public void setDescription(LocalizedString description) { + this.description = description; + } + + public LocalizedString getDescription() { + return description; + } + + public LocalizedString getAdditionalInformation() { + return additionalInformation; + } + + public void setAdditionalInformation(LocalizedString additionalInformation) { + this.additionalInformation = additionalInformation; + } + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } + + public boolean isFor(String studyId, String sourceURN) { + return this.studyId.equals(studyId) && getSourceURN().equals(sourceURN); + } + + public void setSourceURN(String sourceURN) { + this.sourceURN = sourceURN; + } + + public String getSourceURN() { + // legacy + if (Strings.isNullOrEmpty(sourceURN)) { + this.sourceURN = OpalTableSource.newSource(project, table).getURN(); + } + return sourceURN; + } + + @Deprecated + public void setProject(String project) { + this.project = project; + } + + @Deprecated + public void setTable(String table) { + this.table = table; } } diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java deleted file mode 100644 index 9b17df54ce..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.core.domain; - -import javax.validation.constraints.NotNull; - -/** - * Represents a table in opal. - */ -public abstract class OpalTable { - - @NotNull - private String project; - - @NotNull - private String table; - - private LocalizedString name; - - private LocalizedString description; - - private LocalizedString additionalInformation; - - private int weight; - - public String getProject() { - return project; - } - - public void setProject(String project) { - this.project = project; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public void setName(LocalizedString name) { - this.name = name; - } - - public LocalizedString getName() { - return name; - } - - public void setDescription(LocalizedString description) { - this.description = description; - } - - public LocalizedString getDescription() { - return description; - } - - public LocalizedString getAdditionalInformation() { - return additionalInformation; - } - - public void setAdditionalInformation(LocalizedString additionalInformation) { - this.additionalInformation = additionalInformation; - } - - public int getWeight() { - return weight; - } - - public void setWeight(int weight) { - this.weight = weight; - } - - public boolean isFor(String entityId, String project, String table) { - return getEntityId().equals(entityId) && getProject().equals(project) && getTable().equals(table); - } - - protected abstract String getEntityId(); - -} diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java new file mode 100644 index 0000000000..fe19508893 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.domain; + +import com.google.common.base.Strings; +import org.obiba.magma.ValueTable; +import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.spi.dataset.StudyTableSource; +import org.obiba.opal.rest.client.magma.RestDatasource; +import org.obiba.opal.rest.client.magma.RestValueTable; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +import javax.validation.constraints.NotNull; + +/** + * Connector to an Opal server, to retrieve value table and summary statistics. + */ +public class OpalTableSource implements StudyTableSource { + + private OpalService opalService; + + private String opalUrl; + + @NotNull + private String project; + + @NotNull + private String table; + + public String getProject() { + return project; + } + + public void setProject(String project) { + this.project = project; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + @Override + public ValueTable getValueTable() { + return getDatasource().getValueTable(table); + } + + @Override + public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { + return getRestValueTable().getFacets(query); + } + + @Override + public Math.SummaryStatisticsDto getVariableSummary(String variableName) { + return ((RestValueTable.RestVariableValueSource)getRestValueTable().getVariableValueSource(variableName)).getSummary(); + } + + @Override + public String getURN() { + return String.format("urn:opal:%s.%s", project, table); + } + + public static boolean isFor(String sourceURN) { + return !Strings.isNullOrEmpty(sourceURN) && sourceURN.startsWith("urn:opal:"); + } + + public static OpalTableSource newSource(String project, String table) { + OpalTableSource source = new OpalTableSource(); + source.setProject(project); + source.setTable(table); + return source; + } + + public static OpalTableSource fromURN(String sourceURN) { + if (Strings.isNullOrEmpty(sourceURN) || !sourceURN.startsWith("urn:opal:")) + throw new IllegalArgumentException("Not a valid Opal table source URN: " + sourceURN); + + String fullName = toTableName(sourceURN); + int sep = fullName.indexOf("."); + String project = fullName.substring(0, sep); + String table = fullName.substring(sep + 1); + return OpalTableSource.newSource(project, table); + } + + public static String toTableName(String sourceURN) { + return sourceURN.replace("urn:opal:", ""); + } + + public void init(OpalService opalService, String opalUrl) { + this.opalService = opalService; + this.opalUrl = opalUrl; + } + + private RestDatasource getDatasource() { + return opalService.getDatasource(opalUrl, project); + } + + private RestValueTable getRestValueTable() { + return (RestValueTable) getDatasource().getValueTable(table); + } +} diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java index 14ec461824..e3036a78dc 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java @@ -10,10 +10,10 @@ package org.obiba.mica.core.domain; -import java.io.Serializable; - import com.google.common.base.MoreObjects; +import java.io.Serializable; + /** * Represents a opal table that is associated to a {@link org.obiba.mica.study.domain.Study} * {@link org.obiba.mica.study.domain.Population} {@link org.obiba.mica.study.domain.DataCollectionEvent}. @@ -63,7 +63,7 @@ public boolean appliesTo(String studyId, String populationId, String dataCollect @Override public String toString() { - return MoreObjects.toStringHelper(this).add("project", getProject()).add("table", getTable()) + return MoreObjects.toStringHelper(this).add("source", getSourceURN()) .add("dceId", getDataCollectionEventUId()).toString(); } diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java new file mode 100644 index 0000000000..7ad425a408 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.service; + +import org.obiba.mica.core.domain.BaseStudyTable; +import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.spi.dataset.StudyTableSource; +import org.obiba.mica.spi.dataset.StudyTableSourceService; +import org.obiba.mica.study.domain.BaseStudy; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.util.NoSuchElementException; +import java.util.Optional; + +@Service +public class StudyTableSourceServiceRegistry { + + @Inject + private PluginsService pluginsService; + + @Inject + private OpalService opalService; + + public StudyTableSource makeSource(BaseStudy study, String sourceURN) { + if (OpalTableSource.isFor(sourceURN)) { + OpalTableSource source = OpalTableSource.fromURN(sourceURN); + source.init(opalService, study.getOpal()); + return source; + } + Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() + .filter(service -> service.isFor(sourceURN)).findFirst(); + if (serviceOptional.isPresent()) { + // TODO add a context to the study table source + return serviceOptional.get().makeSource(sourceURN); + } + throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + sourceURN); + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java index b1aabd45fe..6e32ab5fb1 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java @@ -28,7 +28,7 @@ import java.util.Map; /** - * Proxy to Opal tables. + * Proxy to value tables. */ public abstract class Dataset extends AbstractModelAware implements AttributeAware, Indexable { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index c4884ebd59..6618a77d98 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -89,9 +89,7 @@ public enum OpalTableType { private int index; - private String project; - - private String table; + private String sourceURN; private OpalTableType opalTableType; @@ -132,25 +130,17 @@ public DatasetVariable(HarmonizationDataset dataset, Variable variable) { setContainerId(table.getStudyId()); } - public DatasetVariable(HarmonizationDataset dataset, Variable variable, OpalTable opalTable) { + public DatasetVariable(HarmonizationDataset dataset, Variable variable, BaseStudyTable studyTable) { this(dataset, Type.Harmonized, variable); - if (opalTable instanceof BaseStudyTable) { - studyId = ((BaseStudyTable)opalTable).getStudyId(); - setContainerId(studyId); - opalTableType = opalTable instanceof StudyTable ? OpalTableType.Study : OpalTableType.Harmonization; + studyId = studyTable.getStudyId(); + setContainerId(studyId); + opalTableType = studyTable instanceof StudyTable ? OpalTableType.Study : OpalTableType.Harmonization; - if (opalTable instanceof HarmonizationStudyTable) { - populationId = ((HarmonizationStudyTable) opalTable).getPopulationUId(); - dceId = ((HarmonizationStudyTable) opalTable).getDataCollectionEventUId(); - } else { - populationId = ((BaseStudyTable) opalTable).getPopulationUId(); - dceId = ((BaseStudyTable) opalTable).getDataCollectionEventUId(); - } - } + populationId = studyTable.getPopulationUId(); + dceId = studyTable.getDataCollectionEventUId(); - project = opalTable.getProject(); - table = opalTable.getTable(); + sourceURN = studyTable.getSourceURN(); } private DatasetVariable(Dataset dataset, Type type, Variable variable) { @@ -187,7 +177,7 @@ public String getId() { if (Type.Harmonized == variableType) { String entityId = studyId; String tableType = opalTableType == OpalTableType.Study ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + project + ID_SEPARATOR + table; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + sourceURN; } return id; @@ -202,8 +192,7 @@ public void setId(String id) { opalTableType = Strings.isNullOrEmpty(tableType) ? null : OpalTableType.valueOf(tableType); if (resolver.hasStudyId()) studyId = resolver.getStudyId(); - if (resolver.hasProject()) project = resolver.getProject(); - if (resolver.hasTable()) table = resolver.getTable(); + if (resolver.hasSourceURN()) sourceURN = resolver.getSourceURN(); } public String getDatasetId() { @@ -327,12 +316,8 @@ public int getIndex() { return index; } - public String getProject() { - return project; - } - - public String getTable() { - return table; + public String getSourceURN() { + return sourceURN; } public OpalTableType getOpalTableType() { @@ -483,7 +468,7 @@ private static String encodeDecode(Map map, Pattern pattern, Str public static class IdResolver { - private final String id; + private String id; private final Type type; @@ -493,12 +478,10 @@ public static class IdResolver { private final String studyId; - private final String project; - - private final String table; - private final String tableType; + private final String sourceURN; + public static IdResolver from(String id) { return new IdResolver(id); } @@ -507,31 +490,33 @@ public static IdResolver from(String datasetId, String variableName, Type variab return from(encode(datasetId, variableName, variableType, null)); } + public static String encode(String datasetId, String variableName, Type variableType) { + return encode(datasetId, variableName, variableType, null, null, null); + } + public static String encode(String datasetId, String variableName, Type variableType, String studyId, - String project, String table, String tableType) { + String sourceURN, String tableType) { String id = datasetId + ID_SEPARATOR + IdEncoderDecoder.encode(variableName) + ID_SEPARATOR + variableType; String entityId; if (Type.Harmonized == variableType) { entityId = studyId; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + project + ID_SEPARATOR + table; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + sourceURN; } return id; } - public static String encode(String datasetId, String variableName, Type variableType, OpalTable opalTable) { - String tableType = opalTable instanceof StudyTable ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; - BaseStudyTable studyTable = (BaseStudyTable)opalTable; + public static String encode(String datasetId, String variableName, Type variableType, BaseStudyTable studyTable) { + String tableType = studyTable instanceof StudyTable ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; return studyTable == null - ? encode(datasetId, variableName, variableType, null, null, null, null) + ? encode(datasetId, variableName, variableType) : encode(datasetId, variableName, variableType, studyTable.getStudyId(), - opalTable.getProject(), - opalTable.getTable(), + studyTable.getSourceURN(), tableType); } @@ -540,7 +525,9 @@ private IdResolver(String id) { this.id = IdEncoderDecoder.encode(id); boolean encoded = !this.id.equals(id); - String[] tokens = id.split(ID_SEPARATOR); + String[] parts = id.split(":urn:"); + + String[] tokens = parts[0].split(ID_SEPARATOR); if (tokens.length < 3) throw new IllegalArgumentException("Not a valid dataset variable ID: " + id); datasetId = tokens[0]; @@ -550,8 +537,16 @@ private IdResolver(String id) { tableType = tokens.length > 3 ? tokens[3] : null; studyId = tokens.length > 4 ? tokens[4] : null; - project = tokens.length > 5 ? tokens[5] : null; - table = tokens.length > 6 ? tokens[6] : null; + if (parts.length>1) { + sourceURN = "urn:" + parts[1]; + } else if (tokens.length > 6) { + // legacy + sourceURN = String.format("urn:opal:%s.%s", tokens[5], tokens[6]); + // need to rewrite id + this.id = IdEncoderDecoder.encode(encode(datasetId, name, type,studyId, sourceURN, tableType)); + } else { + sourceURN = null; + } } public String getId() { @@ -578,26 +573,18 @@ public boolean hasStudyId() { return !Strings.isNullOrEmpty(studyId); } - public String getProject() { - return project; - } - - public boolean hasProject() { - return !Strings.isNullOrEmpty(project); - } - - public String getTable() { - return table; + public boolean hasSourceURN() { + return !Strings.isNullOrEmpty(sourceURN); } - public boolean hasTable() { - return !Strings.isNullOrEmpty(table); + public String getSourceURN() { + return sourceURN; } @Override public String toString() { String tableType = type == Type.Dataschema ? OPAL_HARMONIZATION_TABLE_PREFIX : OPAL_STUDY_TABLE_PREFIX; - return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + project + ", " + table + "]"; + return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + sourceURN + "]"; } public String getTableType() { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java index 124cc5b0c2..6ee28b065f 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java @@ -10,21 +10,18 @@ package org.obiba.mica.dataset.domain; -import java.io.Serializable; -import java.util.*; -import java.util.stream.Collectors; - -import javax.validation.constraints.NotNull; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; - import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTable; import org.obiba.mica.core.domain.StudyTable; +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.*; +import java.util.stream.Collectors; + /** * Dataset that relies on Study Opal servers summaries. */ @@ -106,6 +103,6 @@ public Map parts() { @JsonIgnore public List getBaseStudyTables() { return Lists.newArrayList(Iterables.concat(getStudyTables(), getHarmonizationTables())).stream()// - .sorted(Comparator.comparingInt(OpalTable::getWeight)).collect(Collectors.toList()); + .sorted(Comparator.comparingInt(BaseStudyTable::getWeight)).collect(Collectors.toList()); } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 29511f5a61..28b01f76b2 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -15,9 +15,9 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; -import org.joda.time.DateTime; import org.obiba.magma.NoSuchValueTableException; import org.obiba.magma.NoSuchVariableException; +import org.obiba.magma.ValueTable; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.AbstractGitPersistable; import org.obiba.mica.core.domain.PublishCascadingScope; @@ -50,7 +50,6 @@ import org.obiba.mica.study.service.IndividualStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestValueTable; import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +66,7 @@ import javax.inject.Inject; import javax.validation.Valid; import javax.validation.constraints.NotNull; +import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -74,8 +74,6 @@ import static java.util.stream.Collectors.toList; -import java.time.LocalDateTime; - @Service @Validated public class CollectedDatasetService extends DatasetService { @@ -410,10 +408,8 @@ public List processVariablesForStudyDataset(StudyDataset datase } @Override - @NotNull - protected RestValueTable getTable(@NotNull StudyDataset dataset) throws NoSuchValueTableException { - StudyTable studyTable = dataset.getSafeStudyTable(); - return execute(studyTable, datasource -> (RestValueTable) datasource.getValueTable(studyTable.getTable())); + protected ValueTable getValueTable(@NotNull StudyDataset dataset) throws NoSuchValueTableException { + return getStudyTableSource(dataset.getSafeStudyTable()).getValueTable(); } @Override @@ -428,25 +424,19 @@ public Iterable getDatasetVariables(StudyDataset dataset) { @Override public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getVariableValueSource(dataset, variableName).getVariable()); + return new DatasetVariable(dataset, getValueTable(dataset).getVariable(variableName)); } @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName") public SummaryStatisticsWrapper getVariableSummary(@NotNull StudyDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { log.info("Caching variable summary {} {}", dataset.getId(), variableName); - return new SummaryStatisticsWrapper(getVariableValueSource(dataset, variableName).getSummary()); - } - - public Search.QueryResultDto getVariableFacet(@NotNull StudyDataset dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - log.debug("Getting variable facet {} {}", dataset.getId(), variableName); - return getVariableValueSource(dataset, variableName).getFacet(); + return new SummaryStatisticsWrapper(getStudyTableSource(dataset.getSafeStudyTable()).getVariableSummary(variableName)); } public Search.QueryResultDto getFacets(@NotNull StudyDataset dataset, Search.QueryTermsDto query) throws NoSuchValueTableException, NoSuchVariableException { - return getTable(dataset).getFacets(query); + return getStudyTableSource(dataset.getSafeStudyTable()).getFacets(query); } public Search.QueryResultDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, @@ -515,18 +505,6 @@ protected EventBus getEventBus() { // Private methods // - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param studyTable - * @param callback - * @param - * @return - */ - private T execute(StudyTable studyTable, DatasourceCallback callback) { - return execute(getDatasource(studyTable), callback); - } - private void saveInternal(StudyDataset dataset, String comment) { if (!Strings.isNullOrEmpty(dataset.getId()) && micaConfigService.getConfig().isCommentsRequiredOnDocumentSave() && Strings.isNullOrEmpty(comment)) { throw new MissingCommentException("Due to the server configuration, comments are required when saving this document."); diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 52543c8e40..f061a9dcfc 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -10,42 +10,34 @@ package org.obiba.mica.dataset.service; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.Serializable; -import java.util.List; - -import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; - +import com.google.common.base.Strings; +import com.google.common.eventbus.EventBus; +import com.google.protobuf.GeneratedMessage; import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf; - -import org.obiba.magma.MagmaRuntimeException; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; -import org.obiba.magma.Variable; +import org.obiba.magma.*; +import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.EntityState; -import org.obiba.mica.core.domain.HarmonizationStudyTable; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.core.domain.OpalTable; -import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.core.service.AbstractGitPersistableService; +import org.obiba.mica.core.service.StudyTableSourceServiceRegistry; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.dataset.StudyTableSource; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestDatasource; -import org.obiba.opal.rest.client.magma.RestValueTable; -import org.obiba.opal.web.model.Magma; import org.obiba.opal.web.model.Math; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Strings; -import com.google.common.eventbus.EventBus; -import com.google.protobuf.GeneratedMessage; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.List; /** * {@link org.obiba.mica.dataset.domain.Dataset} management service. @@ -55,6 +47,9 @@ public abstract class DatasetService private static final Logger log = LoggerFactory.getLogger(DatasetService.class); + @Inject + private StudyTableSourceServiceRegistry studyTableSourceServiceRegistry; + /** * Get all {@link org.obiba.mica.dataset.domain.DatasetVariable}s from a {@link org.obiba.mica.dataset.domain.Dataset}. * @@ -73,18 +68,7 @@ public abstract class DatasetService public abstract DatasetVariable getDatasetVariable(T dataset, String name) throws NoSuchValueTableException, NoSuchVariableException; - /** - * Get the {@link org.obiba.opal.web.model.Magma.TableDto} of the {@link org.obiba.mica.dataset.domain.Dataset} identified by its id. - * - * @param dataset - * @return - */ - @NotNull - protected abstract RestValueTable getTable(@NotNull T dataset) throws NoSuchValueTableException; - - protected RestValueTable getTable(@NotNull String project, @NotNull String table) throws NoSuchValueTableException { - return execute(getDatasource(project), datasource -> (RestValueTable) datasource.getValueTable(table)); - } + protected abstract ValueTable getValueTable(@NotNull T dataset) throws NoSuchValueTableException; protected abstract StudyService getStudyService(); @@ -150,71 +134,11 @@ private void ensureAcronym(@NotNull T dataset) { */ protected Iterable getVariables(@NotNull T dataset) throws NoSuchDatasetException, NoSuchValueTableException { - return getTable(dataset).getVariables(); - } - - /** - * Get the {@link org.obiba.magma.VariableValueSource} (proxy to the {@link org.obiba.magma.Variable} of - * the {@link org.obiba.mica.dataset.domain.Dataset} identified by its id. - * - * @param dataset - * @param variableName - * @return - * @throws NoSuchDatasetException - */ - protected RestValueTable.RestVariableValueSource getVariableValueSource(@NotNull T dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(dataset).getVariableValueSource(variableName); - } - - protected RestValueTable.RestVariableValueSource getVariableValueSource(String project, String table, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(project, table).getVariableValueSource(variableName); - } - - public Magma.TableDto getTableDto(@NotNull T dataset) { - return getTable(dataset).getTableDto(); - } - - public Magma.VariableDto getVariable(@NotNull T dataset, String variableName) { - return getVariableValueSource(dataset, variableName).getVariableDto(); - } - - /** - * Callback that can be used to make any operations on a {@link org.obiba.opal.rest.client.magma.RestDatasource} - * - * @param - */ - public interface DatasourceCallback { - R doWithDatasource(RestDatasource datasource); - } - - /** - * Execute the callback on the given datasource. - * - * @param datasource - * @param callback - * @param - * @return - */ - protected R execute(RestDatasource datasource, DatasourceCallback callback) { - return callback.doWithDatasource(datasource); - } - - protected RestDatasource getDatasource(@NotNull OpalTable opalTable) { - String opalUrl = null; - - if (opalTable instanceof StudyTable) { - opalUrl = getStudyService().findDraft(((StudyTable) opalTable).getStudyId()).getOpal(); - } else if (opalTable instanceof HarmonizationStudyTable) { - opalUrl = getStudyService().findDraft(((HarmonizationStudyTable) opalTable).getStudyId()).getOpal(); - } - - return getOpalService().getDatasource(opalUrl, opalTable.getProject()); + return getValueTable(dataset).getVariables(); } - protected RestDatasource getDatasource(@NotNull String project) { - return getOpalService().getDatasource(project); + protected StudyTableSource getStudyTableSource(@NotNull BaseStudyTable studyTable) { + return studyTableSourceServiceRegistry.makeSource(getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSourceURN()); } protected Iterable wrappedGetDatasetVariables(T dataset) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index 7eeeee0226..d104912ff7 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -10,33 +10,14 @@ package org.obiba.mica.dataset.service; -import static java.util.stream.Collectors.toList; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.validation.Valid; -import javax.validation.constraints.NotNull; - -import org.obiba.magma.MagmaRuntimeException; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; -import org.obiba.magma.ValueTable; -import org.obiba.magma.Variable; +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import org.obiba.magma.*; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.BaseStudyTable; -import org.obiba.mica.core.domain.OpalTable; import org.obiba.mica.core.domain.PublishCascadingScope; import org.obiba.mica.core.repository.EntityStateRepository; import org.obiba.mica.core.service.MissingCommentException; @@ -62,7 +43,6 @@ import org.obiba.mica.study.service.HarmonizationStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestValueTable; import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,11 +56,19 @@ import org.springframework.util.Assert; import org.springframework.validation.annotation.Validated; -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.common.eventbus.EventBus; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static java.util.stream.Collectors.toList; @Service @Validated @@ -313,8 +301,7 @@ public void publish(@NotNull String id, boolean published, PublishCascadingScope private void checkIsPublishable(HarmonizationDataset dataset) { if (dataset == null || dataset.getHarmonizationTable() == null - || dataset.getHarmonizationTable().getProject() == null - || dataset.getHarmonizationTable().getTable() == null + || dataset.getHarmonizationTable().getSourceURN() == null || dataset.getHarmonizationTable().getStudyId() == null) { throw new IllegalArgumentException("dataset.harmonization.missing-attributes"); } @@ -357,10 +344,8 @@ public void delete(String id) { } @Override - @NotNull - protected RestValueTable getTable(@NotNull HarmonizationDataset dataset) throws NoSuchValueTableException { - return execute(dataset.getSafeHarmonizationTable().getProject(), - datasource -> (RestValueTable) datasource.getValueTable(dataset.getSafeHarmonizationTable().getTable())); + protected ValueTable getValueTable(@NotNull HarmonizationDataset dataset) throws NoSuchValueTableException { + return getStudyTableSource(dataset.getSafeHarmonizationTable()).getValueTable(); } @Override @@ -372,52 +357,48 @@ public Iterable getDatasetVariables(HarmonizationDataset datase @Override public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getVariableValueSource(dataset, variableName).getVariable()); + return new DatasetVariable(dataset, getStudyTableSource(dataset.getSafeHarmonizationTable()).getValueTable().getVariable(variableName)); } - public Iterable getDatasetVariables(HarmonizationDataset dataset, OpalTable opalTable) + public Iterable getDatasetVariables(HarmonizationDataset dataset, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException { - return StreamSupport.stream(getVariables(opalTable).spliterator(), false) - .map(input -> new DatasetVariable(dataset, input, opalTable)).collect(toList()); + return StreamSupport.stream(getVariables(studyTable).spliterator(), false) + .map(input -> new DatasetVariable(dataset, input, studyTable)).collect(toList()); } - public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, OpalTable opalTable) + public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getTable(opalTable).getVariableValueSource(variableName).getVariable()); + return new DatasetVariable(dataset, getStudyTableSource(studyTable).getValueTable().getVariable(variableName)); } public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, String studyId, - String project, String table) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { + String sourceURN) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { return new DatasetVariable(dataset, - getTable(dataset, studyId, project, table).getVariableValueSource(variableName).getVariable()); + getTable(dataset, studyId, sourceURN).getVariableValueSource(variableName).getVariable()); } @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", - key = "#variableName + ':' + #studyId + ':' + #project + ':' + #table") - public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, - String studyId, String project, String table) + key = "#variableName + ':' + #studyId + ':' + #sourceURN") + public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String sourceURN) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, project, table); - - return new SummaryStatisticsWrapper( - getVariableValueSource(dataset, variableName, studyId, project, table).getSummary()); - } + log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, sourceURN); + for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if(baseTable.isFor(studyId, sourceURN)) { + return new SummaryStatisticsWrapper(getStudyTableSource(baseTable).getVariableSummary(variableName)); + } + } - public Search.QueryResultDto getVariableFacet(@NotNull HarmonizationDataset dataset, String variableName, - String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.debug("Getting variable facet {} {}", dataset.getId(), variableName); - return getVariableValueSource(dataset, variableName, studyId, project, table).getFacet(); + throw NoSuchStudyException.withId(studyId); } - public Search.QueryResultDto getFacets(Search.QueryTermsDto query, OpalTable opalTable) + public Search.QueryResultDto getFacets(Search.QueryTermsDto query, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException { - return getTable(opalTable).getFacets(query); + return getStudyTableSource(studyTable).getFacets(query); } - public Search.QueryResultDto getContingencyTable(@NotNull OpalTable opalTable, DatasetVariable variable, + public Search.QueryResultDto getContingencyTable(@NotNull BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { - return getFacets(QueryTermsUtil.getContingencyQuery(variable, crossVariable), opalTable); + return getFacets(QueryTermsUtil.getContingencyQuery(variable, crossVariable), studyTable); } @Override @@ -487,71 +468,23 @@ protected HarmonizationDataset prepareSave(HarmonizationDataset dataset) { } } - private Iterable getVariables(OpalTable opalTable) + private Iterable getVariables(BaseStudyTable studyTable) throws NoSuchDatasetException, NoSuchStudyException, NoSuchValueTableException { - return getTable(opalTable).getVariables(); + return getStudyTableSource(studyTable).getValueTable().getVariables(); } - private RestValueTable getTable(@NotNull OpalTable opalTable) + private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String sourceURN) throws NoSuchStudyException, NoSuchValueTableException { - return execute(opalTable, ds -> (RestValueTable) ds.getValueTable(opalTable.getTable())); - } - - private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException { - - for(BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if(opalTable.isFor(opalTableId, project, table)) { - return getTable(opalTable); - } - } - - throw NoSuchStudyException.withId(studyId); - } - private RestValueTable.RestVariableValueSource getVariableValueSource(@NotNull HarmonizationDataset dataset, - String variableName, String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - for(BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if(opalTable.isFor(opalTableId, project, table)) { - return getVariableValueSource(variableName, opalTable); + for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if(baseTable.isFor(studyId, sourceURN)) { + return getStudyTableSource(baseTable).getValueTable(); } } throw NoSuchStudyException.withId(studyId); } - private RestValueTable.RestVariableValueSource getVariableValueSource(String variableName, OpalTable opalTable) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(opalTable).getVariableValueSource(variableName); - } - - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param project - * @param callback - * @param - * @return - */ - private T execute(String project, DatasourceCallback callback) { - return execute(getDatasource(project), callback); - } - - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param opalTable - * @param callback - * @param - * @return - */ - private T execute(OpalTable opalTable, DatasourceCallback callback) { - return execute(getDatasource(opalTable), callback); - } - @Override protected EntityStateRepository getEntityStateRepository() { return harmonizationDatasetStateRepository; @@ -609,7 +542,7 @@ public void asyncBuildDatasetVariablesCache(HarmonizationDataset dataset, dataset.getBaseStudyTables().forEach(st -> harmonizationVariables.forEach((k, v) -> v.forEach(var -> { try { String studyId = st.getStudyId(); - service.getVariableSummary(dataset, var.getName(), studyId, st.getProject(), st.getTable()); + service.getVariableSummary(dataset, var.getName(), studyId, st.getSourceURN()); } catch(Exception e) { //ignoring } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java index f10ae60e22..a46d0c0fbc 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java @@ -13,23 +13,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.googlecode.protobuf.format.JsonFormat; -import java.io.IOException; -import java.io.OutputStream; -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -import org.obiba.mica.core.domain.Attributes; -import org.obiba.mica.core.domain.DocumentSet; -import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.core.domain.OpalTable; +import org.obiba.mica.core.domain.*; import org.obiba.mica.core.service.DocumentSetService; -import org.obiba.mica.dataset.domain.Dataset; -import org.obiba.mica.dataset.domain.DatasetCategory; -import org.obiba.mica.dataset.domain.DatasetVariable; -import org.obiba.mica.dataset.domain.HarmonizationDataset; -import org.obiba.mica.dataset.domain.StudyDataset; +import org.obiba.mica.dataset.domain.*; import org.obiba.mica.micaConfig.domain.MicaConfig; import org.obiba.mica.study.service.PublishedDatasetVariableService; import org.obiba.opal.web.model.Magma; @@ -37,6 +23,13 @@ import org.springframework.validation.annotation.Validated; import javax.inject.Inject; +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Service @Validated @@ -320,16 +313,16 @@ private List toCategoryDtoList(List categori private List toOpalTableFullName(Dataset dataset) { if (dataset instanceof StudyDataset) { - OpalTable opalTable = ((StudyDataset) dataset).getSafeStudyTable(); - return Lists.newArrayList(opalTable.getProject() + "." + opalTable.getTable()); + StudyTable studyTable = ((StudyDataset) dataset).getSafeStudyTable(); + return Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSourceURN())); } else { HarmonizationDataset harmoDataset = (HarmonizationDataset) dataset; // one for each study and harmo tables List tableNames = Lists.newArrayList(); tableNames.addAll(harmoDataset.getStudyTables().stream() - .map(st -> st.getProject() + "." + st.getTable()).collect(Collectors.toList())); + .map(st -> OpalTableSource.toTableName(st.getSourceURN())).collect(Collectors.toList())); tableNames.addAll(harmoDataset.getHarmonizationTables().stream() - .map(ht -> ht.getProject() + "." + ht.getTable()).collect(Collectors.toList())); + .map(ht -> OpalTableSource.toTableName(ht.getSourceURN())).collect(Collectors.toList())); return tableNames; } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index 22bbacab51..b40ca985c7 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -108,7 +108,7 @@ public void buildDatasetVariablesCache() { String studyId = st.getStudyId(); try { harmonizedDatasetService - .getVariableSummary(dataset, v.getName(), studyId, st.getProject(), st.getTable()); + .getVariableSummary(dataset, v.getName(), studyId, st.getSourceURN()); } catch(NoSuchVariableException ex) { //ignore } catch(Exception e) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index f06ba2d76a..a42198fe64 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -16,6 +16,8 @@ import com.google.common.io.Files; import org.obiba.core.util.FileUtil; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; +import org.obiba.mica.spi.dataset.StudyTableSourceService; +import org.obiba.mica.spi.dataset.StudyTableSourceServiceLoader; import org.obiba.mica.spi.search.ConfigurationProvider; import org.obiba.mica.spi.search.SearchEngineService; import org.obiba.mica.spi.search.SearchEngineServiceLoader; @@ -86,12 +88,17 @@ public SearchEngineService getSearchEngineService() { return (SearchEngineService) getServicePlugins(SearchEngineService.class).iterator().next(); } + public Collection getStudyTableSourceServices() { + return getServicePlugins(StudyTableSourceService.class).stream() + .map(service -> (StudyTableSourceService)service) + .collect(Collectors.toList()); + } + // // Private methods // private Collection getServicePlugins(Class clazz) { - //init(); return servicePlugins.stream().filter(s -> clazz.isAssignableFrom(s.getClass())).collect(Collectors.toList()); } @@ -155,6 +162,13 @@ private void initSearchEngineServicePlugin(PluginResources plugin) { servicePlugins.add(service); } + private void initStudyTableSourceServicePlugins(PluginResources plugin) { + StudyTableSourceServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { + service.start(); + servicePlugins.add(service); + }); + } + private synchronized Collection getPlugins(boolean extract) { Map pluginsMap = Maps.newLinkedHashMap(); // make sure plugins directory exists diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java index 45565580fa..9a1974315f 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java @@ -10,29 +10,15 @@ package org.obiba.mica.web.model; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.validation.constraints.NotNull; - +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.obiba.magma.type.BooleanType; import org.obiba.mica.JSONUtils; import org.obiba.mica.core.domain.*; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.StudyDatasetStateRepository; -import org.obiba.mica.dataset.domain.Dataset; -import org.obiba.mica.dataset.domain.DatasetCategory; -import org.obiba.mica.dataset.domain.DatasetVariable; -import org.obiba.mica.dataset.domain.HarmonizationDataset; -import org.obiba.mica.dataset.domain.HarmonizationDatasetState; -import org.obiba.mica.dataset.domain.StudyDataset; -import org.obiba.mica.dataset.domain.StudyDatasetState; +import org.obiba.mica.dataset.domain.*; import org.obiba.mica.micaConfig.domain.MicaConfig; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.security.service.SubjectAclService; @@ -45,9 +31,11 @@ import org.springframework.stereotype.Component; import org.springframework.util.Assert; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.*; +import java.util.stream.Collectors; @Component class DatasetDtos { @@ -194,11 +182,13 @@ Mica.DatasetVariableResolverDto.Builder asDto(@NotNull DatasetVariable.IdResolve if(resolver.hasStudyId()) { builder.setStudyId(resolver.getStudyId()); } - if(resolver.hasProject()) { - builder.setProject(resolver.getProject()); - } - if(resolver.hasTable()) { - builder.setTable(resolver.getTable()); + if(resolver.hasSourceURN()) { + builder.setSourceURN(resolver.getSourceURN()); + if (OpalTableSource.isFor(resolver.getSourceURN())) { + OpalTableSource source = OpalTableSource.fromURN(resolver.getSourceURN()); + builder.setProject(source.getProject()); + builder.setTable(source.getTable()); + } } return builder; @@ -314,8 +304,8 @@ Mica.DatasetVariableDto asDto(@NotNull DatasetVariable variable, @NotNull List builder.addAttributes(attributeDtos.asDto(attribute))); } - if(opalTable instanceof StudyTable) builder.setStudyTable(asDto((StudyTable) opalTable, includeSummaries)); - else if(opalTable instanceof HarmonizationStudyTable) { - builder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable, includeSummaries)); + if(studyTable instanceof StudyTable) builder.setStudyTable(asDto((StudyTable) studyTable, includeSummaries)); + else if(studyTable instanceof HarmonizationStudyTable) { + builder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable, includeSummaries)); } return builder.build(); @@ -411,13 +401,18 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable) { } public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable, boolean includeSummary) { - Mica.DatasetDto.StudyTableDto.Builder sbuilder = Mica.DatasetDto.StudyTableDto.newBuilder() // - .setProject(studyTable.getProject())// - .setTable(studyTable.getTable()) // - .setWeight(studyTable.getWeight()) // - .setStudyId(studyTable.getStudyId()) // + Mica.DatasetDto.StudyTableDto.Builder sbuilder = Mica.DatasetDto.StudyTableDto.newBuilder() + .setSourceURN(studyTable.getSourceURN()) + .setWeight(studyTable.getWeight()) + .setStudyId(studyTable.getStudyId()) .setDceId(studyTable.getDataCollectionEventUId()); + if (OpalTableSource.isFor(studyTable.getSourceURN())) { + OpalTableSource source = OpalTableSource.fromURN(studyTable.getSourceURN()); + sbuilder.setProject(source.getProject()); + sbuilder.setTable(source.getTable()); + } + String populationId = studyTable.getPopulationId(); if (!Strings.isNullOrEmpty(populationId)) { sbuilder.setPopulationId(populationId); @@ -444,11 +439,16 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTable harmonizationTable, boolean includeSummary) { Mica.DatasetDto.HarmonizationTableDto.Builder hBuilder = Mica.DatasetDto.HarmonizationTableDto.newBuilder() - .setProject(harmonizationTable.getProject()) - .setTable(harmonizationTable.getTable()) + .setSourceURN(harmonizationTable.getSourceURN()) .setWeight(harmonizationTable.getWeight()) .setStudyId(harmonizationTable.getStudyId()); + if (OpalTableSource.isFor(harmonizationTable.getSourceURN())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationTable.getSourceURN()); + hBuilder.setProject(source.getProject()); + hBuilder.setTable(source.getTable()); + } + if(includeSummary) hBuilder.setStudySummary(studySummaryDtos.asDto(harmonizationTable.getStudyId())); hBuilder.addAllName(localizedStringDtos.asDto(harmonizationTable.getName())); @@ -458,16 +458,16 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab return hBuilder; } - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull OpalTable opalTable, + public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); aggDto = simpleAggregationDto(aggDto, summary); - if(opalTable instanceof StudyTable) - aggDto.setStudyTable(asDto((StudyTable) opalTable, withStudySummary)); - else if (opalTable instanceof HarmonizationStudyTable) - aggDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable, withStudySummary)); + if(studyTable instanceof StudyTable) + aggDto.setStudyTable(asDto((StudyTable) studyTable, withStudySummary)); + else if (studyTable instanceof HarmonizationStudyTable) + aggDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable, withStudySummary)); return aggDto; } @@ -493,14 +493,14 @@ public Mica.DatasetVariableAggregationDto.Builder simpleAggregationDto(@NotNull return aggDto; } - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull OpalTable opalTable, + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { Mica.DatasetVariableContingencyDto.Builder crossDto = Mica.DatasetVariableContingencyDto.newBuilder(); - if(opalTable instanceof StudyTable) - crossDto.setStudyTable(asDto((StudyTable) opalTable, true)); - else if (opalTable instanceof HarmonizationStudyTable) - crossDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable)); + if(studyTable instanceof StudyTable) + crossDto.setStudyTable(asDto((StudyTable) studyTable, true)); + else if (studyTable instanceof HarmonizationStudyTable) + crossDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable)); Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); @@ -783,8 +783,12 @@ private Dataset fromDto(@NotNull Mica.HarmonizedDatasetDto dto) { if(dto.hasHarmonizationTable()) { HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); - harmonizationLink.setProject(dto.getHarmonizationTable().getProject()); - harmonizationLink.setTable(dto.getHarmonizationTable().getTable()); + // legacy + if (dto.getHarmonizationTable().hasProject() && dto.getHarmonizationTable().hasTable()) { + harmonizationLink.setSourceURN(makeSourceURN(dto.getHarmonizationTable().getProject(), dto.getHarmonizationTable().getTable())); + } else { + harmonizationLink.setSourceURN(dto.getHarmonizationTable().getSourceURN()); + } harmonizationLink.setStudyId(dto.getHarmonizationTable().getStudyId()); harmonizationDataset.setHarmonizationTable(harmonizationLink); } @@ -807,41 +811,56 @@ private StudyTable fromDto(Mica.DatasetDto.StudyTableDto dto) { if (dto.hasDataCollectionEventId()) { table.setDataCollectionEventId(dto.getDataCollectionEventId()); } - table.setProject(dto.getProject()); - table.setTable(dto.getTable()); table.setWeight(dto.getWeight()); - table.setName(localizedStringDtos.fromDto(dto.getNameList())); table.setDescription(localizedStringDtos.fromDto(dto.getDescriptionList())); table.setAdditionalInformation(localizedStringDtos.fromDto(dto.getAdditionalInformationList())); + if (dto.hasProject() && dto.hasTable()) { + table.setSourceURN(makeSourceURN(dto.getProject(), dto.getTable())); + } else { + table.setSourceURN(dto.getSourceURN()); + } + return table; } private HarmonizationStudyTable fromDto(Mica.DatasetDto.HarmonizationTableDto dto) { HarmonizationStudyTable table = new HarmonizationStudyTable(); table.setStudyId(dto.getStudyId()); - table.setProject(dto.getProject()); - table.setTable(dto.getTable()); table.setWeight(dto.getWeight()); - table.setName(localizedStringDtos.fromDto(dto.getNameList())); table.setDescription(localizedStringDtos.fromDto(dto.getDescriptionList())); table.setAdditionalInformation(localizedStringDtos.fromDto(dto.getAdditionalInformationList())); + // legacy + if (dto.hasProject() && dto.hasTable()) { + table.setSourceURN(makeSourceURN(dto.getProject(), dto.getTable())); + } else { + table.setSourceURN(dto.getSourceURN()); + } + return table; } + // legacy + private String makeSourceURN(String project, String table) { + return OpalTableSource.newSource(project, table).getURN(); + } + private Mica.DatasetDto.HarmonizationTableDto.Builder createHarmonizationLinkDtoFromHarmonizationTable( HarmonizationStudyTable harmonizationLink, boolean asDraft) { Mica.DatasetDto.HarmonizationTableDto.Builder harmonizationLinkBuilder = Mica.DatasetDto.HarmonizationTableDto .newBuilder(); - if(!Strings.isNullOrEmpty(harmonizationLink.getProject())) - harmonizationLinkBuilder.setProject(harmonizationLink.getProject()); - - if(!Strings.isNullOrEmpty(harmonizationLink.getTable())) - harmonizationLinkBuilder.setTable(harmonizationLink.getTable()); + if(!Strings.isNullOrEmpty(harmonizationLink.getSourceURN())) { + harmonizationLinkBuilder.setSourceURN(harmonizationLink.getSourceURN()); + if (OpalTableSource.isFor(harmonizationLink.getSourceURN())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationLink.getSourceURN()); + harmonizationLinkBuilder.setProject(source.getProject()); + harmonizationLinkBuilder.setTable(source.getTable()); + } + } String studyId = harmonizationLink.getStudyId(); diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java index efecd0583b..cb7a8d03f4 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java @@ -397,8 +397,8 @@ public Mica.DatasetHarmonizedVariableSummaryDto asHarmonizedSummaryDto(@NotNull } @NotNull - public Mica.DatasetVariableSummaryDto asSummaryDto(@NotNull DatasetVariable variable, OpalTable opalTable, boolean includeSummaries) { - return datasetDtos.asSummaryDto(variable, opalTable, includeSummaries); + public Mica.DatasetVariableSummaryDto asSummaryDto(@NotNull DatasetVariable variable, BaseStudyTable studyTable, boolean includeSummaries) { + return datasetDtos.asSummaryDto(variable, studyTable, includeSummaries); } @NotNull @@ -427,9 +427,9 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(@NotNull StudyTable studyTabl } @NotNull - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull OpalTable opalTable, + public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { - return datasetDtos.asDto(opalTable, summary, withStudySummary); + return datasetDtos.asDto(studyTable, summary, withStudySummary); } @NotNull @@ -438,9 +438,9 @@ public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull Math.SummarySta } @NotNull - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(OpalTable opalTable, DatasetVariable variable, + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { - return datasetDtos.asContingencyDto(opalTable, variable, crossVariable, results); + return datasetDtos.asContingencyDto(studyTable, variable, crossVariable, results); } @NotNull diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java index f3247ef05a..7290609736 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java @@ -19,6 +19,7 @@ import org.mockito.MockitoAnnotations; import org.obiba.magma.MagmaRuntimeException; import org.obiba.mica.core.domain.LocalizedString; +import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.core.service.GitService; import org.obiba.mica.dataset.StudyDatasetRepository; @@ -98,8 +99,7 @@ public void testDatasourceConnectionErrorIsIgnoredForDraft() { private StudyDataset buildStudyDataset() { StudyDataset ds = new StudyDataset(); StudyTable st = new StudyTable(); - st.setProject("proj"); - st.setTable("tab"); + st.setSourceURN(OpalTableSource.newSource("proj", "tab").getURN()); st.setPopulationId("1"); st.setDataCollectionEventId("1"); ds.setStudyTable(st); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java index a7ef5e54cf..9ff32e5db6 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java @@ -24,6 +24,7 @@ import org.mockito.Spy; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.magma.Variable; import org.obiba.magma.type.BooleanType; @@ -98,8 +99,7 @@ private HarmonizationDataset buildHarmonizationDataset(String id, StudyTable... private StudyTable buildStudyTable(String project, String table, String studyId) { StudyTable st = new StudyTable(); - st.setProject(project); - st.setTable(table); + st.setSourceURN(OpalTableSource.newSource(project, table).getURN()); st.setStudyId(studyId); st.setPopulationId("pop"); st.setDataCollectionEventId("ev"); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java index 619a01fbe9..2736b5acdb 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java @@ -19,6 +19,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.HarmonizationStudyTable; +import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import static org.obiba.mica.assertj.Assertions.assertThat; @@ -107,8 +108,9 @@ private StudyTable createStudyTable(String studyId) { private void initTable(String studyId, BaseStudyTable table) { table.setStudyId(studyId); - table.setProject(RandomStringUtils.random(10, true, false)); - table.setTable(RandomStringUtils.random(10, true, false)); + String project = RandomStringUtils.random(10, true, false); + String tbl = RandomStringUtils.random(10, true, false); + table.setSourceURN(OpalTableSource.newSource(project, tbl).getURN()); table.setPopulationId(RandomStringUtils.random(10, true, true)); } } diff --git a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java index 3e43c7e942..3a31dd978f 100644 --- a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java +++ b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java @@ -18,6 +18,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.HarmonizationStudyTable; +import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.domain.HarmonizationDatasetState; @@ -108,8 +109,7 @@ private HarmonizationDataset createHarmonizedDataset() { HarmonizationDataset harmonizationDataset = new HarmonizationDataset(); harmonizationDataset.setId("123"); HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); - harmonizationLink.setProject("project123"); - harmonizationLink.setTable("table123"); + harmonizationLink.setSourceURN(OpalTableSource.newSource("project123", "table123").getURN()); harmonizationLink.setStudyId("study123"); harmonizationLink.setPopulationId("population123"); harmonizationDataset.setHarmonizationTable(harmonizationLink); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java index 0b99919e8a..1c4e033b4d 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java @@ -17,6 +17,7 @@ import javax.ws.rs.PathParam; import org.apache.shiro.SecurityUtils; +import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.domain.DatasetVariable; @@ -88,8 +89,7 @@ private StudyDataset alternativeStudyDataset(String id, String project, String t if (!(dataset instanceof StudyDataset)) throw NoSuchDatasetException.withId(id); StudyDataset asStudyDataset = (StudyDataset) dataset; - asStudyDataset.getStudyTable().setProject(project); - asStudyDataset.getStudyTable().setTable(table); + asStudyDataset.getStudyTable().setSourceURN(OpalTableSource.newSource(project, table).getURN()); return asStudyDataset; } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java index f35a36937d..1e3f65d751 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java @@ -132,13 +132,6 @@ public Response unPublish() { return Response.noContent().build(); } - @GET - @Path("/table") - public Magma.TableDto getTable() { - checkPermission("/draft/collected-dataset", "VIEW"); - return datasetService.getTableDto(getDataset()); - } - @GET @Path("/variables") public List getVariables() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java index 48085456d0..7d8b74aa1f 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java @@ -49,12 +49,6 @@ public Math.SummaryStatisticsDto getVariableSummary() { return datasetService.getVariableSummary(getDataset(), variableName).getWrappedDto(); } - @GET - @Path("/facet") - public Search.QueryResultDto getVariableFacet() { - return datasetService.getVariableFacet(getDataset(), variableName); - } - private StudyDataset getDataset() { return datasetService.findById(datasetId); } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java index 13520f0078..01ae1090c0 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java @@ -182,8 +182,7 @@ private RQLFieldReferences parseField(String path) { } else if (DatasetVariable.Type.Harmonized.equals(resolver.getType())) { HarmonizationDataset ds = harmonizedDatasetService.findById(resolver.getDatasetId()); Optional studyTable = ds.getBaseStudyTables().stream().filter(st -> st.getStudyId().equals(resolver.getStudyId()) - && st.getProject().equals(resolver.getProject()) - && st.getTable().equals(resolver.getTable())).findFirst(); + && st.getSourceURN().equals(resolver.getSourceURN())).findFirst(); if (!studyTable.isPresent()) throw new IllegalArgumentException("Not a valid variable: " + path); BaseStudy study = studyService.findStudy(studyTable.get().getStudyId()); return new RQLFieldReferences(path, ds, studyTable.get(), study, getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, path)); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java index b3a7cc7a11..6c59af7c68 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java @@ -89,6 +89,6 @@ public LocalizedString getStudyTableName() { * @return */ private String getOpalTablePath(BaseStudyTable studyTable) { - return studyTable.getProject() + "." + studyTable.getTable(); + return studyTable.getSourceURN().replace("urn:opal:", ""); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java index 59f08ddd36..e704dadfd7 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java @@ -58,7 +58,7 @@ public List getVariableSummaries() { try { String studyId = table.getStudyId(); builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) + .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) .getWrappedDto()); } catch(NoSuchVariableException | NoSuchValueTableException e) { // ignore (case the study has not implemented this dataschema variable) @@ -67,22 +67,6 @@ public List getVariableSummaries() { return builder.build(); } - @GET - @Path("/facet") - public List getVariableFacets() { - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService.getVariableFacet(dataset, variableName, studyId, table.getProject(), table.getTable())); - } catch(NoSuchVariableException | NoSuchValueTableException e) { - // ignore (case the study has not implemented this dataschema variable) - } - }); - return builder.build(); - } - @GET @Path("/harmonizations") public List getHarmonizedVariables() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java index 9cf772160c..72bb06857c 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java @@ -137,13 +137,6 @@ public Response unPublish() { return Response.noContent().build(); } - @GET - @Path("/table") - public Magma.TableDto getTable() { - checkPermission("/draft/harmonized-dataset", "VIEW"); - return datasetService.getTableDto(getDataset()); - } - @GET @Path("/variables") public List getVariables() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java index 6e50046272..eb13792215 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java @@ -39,37 +39,25 @@ public class DraftHarmonizedDatasetVariableResource implements DatasetVariableRe private String studyId; - private String project; - - private String table; + private String sourceURN; @GET public Mica.DatasetVariableDto getVariable() { - return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, project, table)); + return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, sourceURN)); } @GET @Path("/summary") public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName, studyId, project, table).getWrappedDto(); - } - - @GET - @Path("/facet") - public Search.QueryResultDto getVariableFacet() { - return datasetService.getVariableFacet(getDataset(), variableName, studyId, project, table); + return datasetService.getVariableSummary(getDataset(), variableName, studyId, sourceURN).getWrappedDto(); } public void setStudyId(String studyId) { this.studyId = studyId; } - public void setProject(String project) { - this.project = project; - } - - public void setTable(String table) { - this.table = table; + public void setSourceURN(String sourceURN) { + this.sourceURN = sourceURN; } private HarmonizationDataset getDataset() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java index fdd222a13b..589a2b88f3 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java @@ -52,8 +52,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id) { subjectAclService.isPermitted("/draft/harmonized-dataset", "VIEW", resolver.getDatasetId()); resource = applicationContext.getBean(DraftHarmonizedDatasetVariableResource.class); ((DraftHarmonizedDatasetVariableResource) resource).setStudyId(resolver.getStudyId()); - ((DraftHarmonizedDatasetVariableResource) resource).setProject(resolver.getProject()); - ((DraftHarmonizedDatasetVariableResource) resource).setTable(resolver.getTable()); + ((DraftHarmonizedDatasetVariableResource) resource).setSourceURN(resolver.getSourceURN()); break; } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java index 933ff76eac..e4e1ca09a7 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java @@ -10,23 +10,11 @@ package org.obiba.mica.dataset.search.rest; -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.validation.constraints.NotNull; -import javax.ws.rs.ForbiddenException; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; import org.apache.shiro.SecurityUtils; import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.core.domain.BaseStudyTable; -import org.obiba.mica.core.domain.OpalTable; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.Dataset; @@ -44,8 +32,17 @@ import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.ForbiddenException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; /** * Retrieve the {@link org.obiba.mica.dataset.domain.Dataset} from the published dataset index. @@ -211,22 +208,21 @@ protected Mica.DatasetVariableHarmonizationSummaryDto getVariableHarmonizationSu } protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, OpalTable opalTable) { + DatasetVariable.Type variableType, BaseStudyTable studyTable) { - if (opalTable != null) { + if (studyTable != null) { return getDatasetVariable(datasetId, variableName, variableType, - opalTable instanceof BaseStudyTable ? ((BaseStudyTable) opalTable).getStudyId() : null, - opalTable.getProject(), - opalTable.getTable(), - opalTable instanceof StudyTable + studyTable.getStudyId(), + studyTable.getSourceURN(), + studyTable instanceof StudyTable ? DatasetVariable.OPAL_STUDY_TABLE_PREFIX : DatasetVariable.OPAL_HARMONIZATION_TABLE_PREFIX); } - return getDatasetVariable(datasetId, variableName, variableType, null, null, null, null); + return getDatasetVariable(datasetId, variableName, variableType, null, null, null); } /** @@ -236,18 +232,18 @@ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull * @param datasetId * @param variableName * @param studyId - * @param project - * @param table + * @param sourceURN + * @param tableType * @return * @throws NoSuchVariableException */ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String project, @Nullable String table, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String sourceURN, @Nullable String tableType) throws NoSuchVariableException { String variableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, variableType, studyId, project, table, tableType); + .encode(datasetId, variableName, variableType, studyId, sourceURN, tableType); if (variableType.equals(DatasetVariable.Type.Harmonized)) { return getHarmonizedDatasetVariable(datasetId, variableId, variableName); @@ -262,23 +258,23 @@ protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetI } protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { + DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { return dtos - .asDto(getDatasetVariable(datasetId, variableName, variableType, opalTable), getTaxonomies(), + .asDto(getDatasetVariable(datasetId, variableName, variableType, studyTable), getTaxonomies(), getLocale()); } protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String project, @Nullable String table, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String sourceURN, @Nullable String tableType) { - return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, project, table, tableType), + return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, sourceURN, tableType), getTaxonomies(), getLocale()); } protected Mica.DatasetHarmonizedVariableSummaryDto getDatasetHarmonizedVariableSummaryDto(@NotNull String datasetId, - @NotNull String variableName, DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { + @NotNull String variableName, DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { try { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); return dtos.asHarmonizedSummaryDto(variable); } catch (NoSuchVariableException e) { return Mica.DatasetHarmonizedVariableSummaryDto.newBuilder().setStatus("").build(); @@ -287,18 +283,18 @@ protected Mica.DatasetHarmonizedVariableSummaryDto getDatasetHarmonizedVariableS protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull String datasetId, - @NotNull String variableName, DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); - return dtos.asSummaryDto(variable, opalTable, true); + @NotNull String variableName, DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); + return dtos.asSummaryDto(variable, studyTable, true); } protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull String datasetId, @NotNull String variableName, DatasetVariable.Type variableType, - @Nullable OpalTable opalTable, + @Nullable BaseStudyTable studyTable, boolean includeSummaries) { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); - return dtos.asSummaryDto(variable, opalTable, includeSummaries); + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); + return dtos.asSummaryDto(variable, studyTable, includeSummaries); } @NotNull @@ -324,7 +320,7 @@ protected void checkVariableSummaryAccess() { private DatasetVariable getHarmonizedDatasetVariable(String datasetId, String variableId, String variableName) { String dataSchemaVariableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, DatasetVariable.Type.Dataschema, null, null, null, null); + .encode(datasetId, variableName, DatasetVariable.Type.Dataschema); DatasetVariable harmonizedDatasetVariable = getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, variableId, variableName); DatasetVariable dataSchemaVariable = getDatasetVariableInternal(Indexer.PUBLISHED_VARIABLE_INDEX, Indexer.VARIABLE_TYPE, diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java index f6af8217b8..6cdbf514cf 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java @@ -63,15 +63,6 @@ public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { return datasetService.getVariableSummary(getDataset(StudyDataset.class, datasetId), variableName).getWrappedDto(); } - @GET - @Path("/facet") - @Timed - public Search.QueryResultDto getVariableFacet() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService.getVariableFacet(getDataset(StudyDataset.class, datasetId), variableName); - } - @GET @Path("/aggregation") @Timed diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java index 5b922ada9b..8d4aa9e559 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java @@ -84,10 +84,9 @@ private void writeBody(CSVWriter writer, Mica.DatasetVariableContingencyDto dto) private void writeHeaders(CSVWriter writer, Mica.DatasetVariableContingencyDto c, List terms) { if(c.hasStudyTable()) writer.writeNext(new String[] { String - .format("%s - %s - %s", c.getStudyTable().getProject(), c.getStudyTable().getTable(), - c.getStudyTable().getDceId()) }); + .format("%s - %s", c.getStudyTable().getSourceURN(), c.getStudyTable().getDceId()) }); else if(c.hasHarmonizationStudyTable()) writer.writeNext(new String[] { String - .format("%s - %s", c.getHarmonizationStudyTable().getProject(), c.getHarmonizationStudyTable().getTable())}); + .format("%s", c.getHarmonizationStudyTable().getSourceURN())}); writer.writeNext(concat(concat(Stream.of(""), terms.stream()), Stream.of("Total")).toArray(String[]::new)); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java index b4197e64bb..02c1df3913 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java @@ -83,11 +83,10 @@ private void writeBody(CSVWriter writer, HarmonizationDataset dataset, final boolean[] found = { false }; variableHarmonization.getDatasetVariableSummariesList().forEach( summary -> { - String id = table instanceof StudyTable ? ((StudyTable) table).getStudyId() : ((HarmonizationStudyTable) table).getStudyId(); + String id = table.getStudyId(); Mica.DatasetVariableResolverDto resolver = summary.getResolver(); if ((resolver.getStudyId().equals(id) - && resolver.getProject().equals(table.getProject()) - && resolver.getTable().equals(table.getTable()))) { + && resolver.getSourceURN().equals(table.getSourceURN()))) { String statusDetail = getAttributeByName(summary, "status_detail", locale); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index 45bbf17570..ab922b0e45 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -20,9 +20,6 @@ import org.obiba.magma.NoSuchValueTableException; import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.core.domain.BaseStudyTable; -import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTable; -import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.HarmonizationDataset; @@ -85,7 +82,7 @@ public List getVariableSummaries() { try { String studyId = table.getStudyId(); builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) + .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) .getWrappedDto()); } catch (NoSuchVariableException | NoSuchValueTableException e) { // case the study has not implemented this dataschema variable @@ -95,27 +92,6 @@ public List getVariableSummaries() { return builder.build(); } - @GET - @Path("/facet") - @Timed - public List getVariableFacets() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableFacet(dataset, variableName, studyId, table.getProject(), table.getTable())); - } catch (NoSuchVariableException | NoSuchValueTableException e) { - // case the study has not implemented this dataschema variable - builder.add(Search.QueryResultDto.newBuilder().setTotalHits(0).build()); - } - }); - return builder.build(); - } - @GET @Path("/aggregation") @Timed @@ -288,18 +264,11 @@ public static class Helper { @Async protected Future getVariableFacet(HarmonizationDataset dataset, String variableName, - OpalTable table) { + BaseStudyTable table) { try { - String studyId = null; - - if (table instanceof StudyTable) { - studyId = ((StudyTable) table).getStudyId(); - } else if (table instanceof HarmonizationStudyTable) { - studyId = ((HarmonizationStudyTable) table).getStudyId(); - } - + String studyId = table.getStudyId(); return new AsyncResult<>(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) + .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) .getWrappedDto()); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); @@ -309,9 +278,9 @@ protected Future getVariableFacet(HarmonizationDatase @Async protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, - DatasetVariable crossVar, OpalTable table) { + DatasetVariable crossVar, BaseStudyTable studyTable) { try { - return new AsyncResult<>(datasetService.getContingencyTable(table, var, crossVar)); + return new AsyncResult<>(datasetService.getContingencyTable(studyTable, var, crossVar)); } catch (Exception e) { log.warn("Unable to retrieve contingency statistics: " + e.getMessage(), e); return new AsyncResult<>(null); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java index d315837740..a9cfa14e22 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java @@ -215,16 +215,17 @@ private Mica.DatasetVariablesHarmonizationSummaryDto getVariableHarmonizationsSu // harmonized variables, extract one column per study table List> harmonizedVariables = Lists.newArrayList(); builder.setTotal(variablesDto.getTotal()).setLimit(variablesDto.getLimit()).setFrom(variablesDto.getFrom()); - String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(project,%s),eq(table,%s))"; + + String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(sourceURN,%s))"; dataset.getStudyTables().forEach(st -> { builder.addStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getProject(), st.getTable()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getSourceURN()); List datasetVariablesInternal = getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true); harmonizedVariables.add(datasetVariablesInternal.stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); dataset.getHarmonizationTables().forEach(st -> { builder.addHarmonizationStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getProject(), st.getTable()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getSourceURN()); harmonizedVariables.add(getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true).stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index 3e246b9539..b07f5961f9 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -21,7 +21,6 @@ import org.obiba.mica.dataset.search.rest.AbstractPublishedDatasetResource; import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -49,9 +48,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe private String studyId; - private String project; - - private String table; + private String sourceURN; private String tableType; @@ -62,8 +59,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe @Timed public Mica.DatasetVariableDto getVariable() { checkDatasetAccess(); - return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, project, table, - tableType); + return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); } @GET @@ -73,20 +69,10 @@ public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { checkDatasetAccess(); checkVariableSummaryAccess(); return datasetService - .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, project, table) + .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, sourceURN) .getWrappedDto(); } - @GET - @Path("/facet") - @Timed - public Search.QueryResultDto getVariableFacet() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService - .getVariableFacet(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, project, table); - } - @GET @Path("/aggregation") @Timed @@ -94,20 +80,19 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s checkDatasetAccess(); checkVariableSummaryAccess(); HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - for (BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if (opalTable.isFor(opalTableId, project, table)) { + for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if (baseTable.isFor(studyId, sourceURN)) { try { - return dtos.asDto(opalTable, - datasetService.getVariableSummary(dataset, variableName, studyId, project, table).getWrappedDto(), withStudySummary).build(); + return dtos.asDto(baseTable, + datasetService.getVariableSummary(dataset, variableName, studyId, sourceURN).getWrappedDto(), withStudySummary).build(); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); - return dtos.asDto(opalTable, null, withStudySummary).build(); + return dtos.asDto(baseTable, null, withStudySummary).build(); } } } - throw new NoSuchValueTableException(project, table); + throw new NoSuchValueTableException(sourceURN); } @GET @@ -123,20 +108,19 @@ public Mica.DatasetVariableContingencyDto getContingency(@QueryParam("by") Strin private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var, DatasetVariable crossVar) { HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - for (BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if (opalTable.isFor(opalTableId, project, table)) { + for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if (baseTable.isFor(studyId, sourceURN)) { try { - return dtos.asContingencyDto(opalTable, var, crossVar, - datasetService.getContingencyTable(opalTable, var, crossVar)).build(); + return dtos.asContingencyDto(baseTable, var, crossVar, + datasetService.getContingencyTable(baseTable, var, crossVar)).build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - return dtos.asContingencyDto(opalTable, var, crossVar, null).build(); + return dtos.asContingencyDto(baseTable, var, crossVar, null).build(); } } } - throw new NoSuchValueTableException(project, table); + throw new NoSuchValueTableException(sourceURN); } @GET @@ -183,12 +167,8 @@ public void setStudyId(String studyId) { this.studyId = studyId; } - public void setProject(String project) { - this.project = project; - } - - public void setTable(String table) { - this.table = table; + public void setSourceURN(String sourceURN) { + this.sourceURN = sourceURN; } public void setTableType(String tableType) { @@ -199,10 +179,8 @@ private Pair getContingencyVariables(String cr if (Strings.isNullOrEmpty(crossVariable)) throw new BadRequestException("Cross variable name is required for the contingency table"); - DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, project, - table, tableType); - DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, - project, table, tableType); + DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); + DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); return Pair.create(var, crossVar); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java index 0a590df80f..ae337460a2 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java @@ -64,8 +64,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id, if (!harmonizedDatasetService.isPublished(resolver.getDatasetId())) throw NoSuchDatasetException.withId(resolver.getDatasetId()); resource = applicationContext.getBean(PublishedHarmonizedDatasetVariableResource.class); ((PublishedHarmonizedDatasetVariableResource)resource).setStudyId(resolver.getStudyId()); - ((PublishedHarmonizedDatasetVariableResource)resource).setProject(resolver.getProject()); - ((PublishedHarmonizedDatasetVariableResource)resource).setTable(resolver.getTable()); + ((PublishedHarmonizedDatasetVariableResource)resource).setSourceURN(resolver.getSourceURN()); ((PublishedHarmonizedDatasetVariableResource)resource).setTableType(resolver.getTableType()); ((PublishedHarmonizedDatasetVariableResource)resource).setLocale(locale); break; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java new file mode 100644 index 0000000000..88aedb999c --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.dataset; + +import org.obiba.magma.ValueTable; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +/** + * Describes the parameters to establish a connection with a Datasource + * and retrieve a ValueTable containing the dataset's variables. + */ +public interface StudyTableSource { + + /** + * Get the {@link ValueTable} implementing the data dictionary and the data values. + * + * @return + */ + ValueTable getValueTable(); + + /** + * Make a facet search, to retrieve contingency table results. + * + * @param query + * @return + */ + Search.QueryResultDto getFacets(Search.QueryTermsDto query); + + /** + * Get a variable summary statistics. + * + * @param variableName + * @return + */ + Math.SummaryStatisticsDto getVariableSummary(String variableName); + + /** + * URN representation of a value table source, indicates the identifier of the value table, in the namespace of the source. + * + * @return + */ + String getURN(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java new file mode 100644 index 0000000000..6d4901c3fc --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.dataset; + +import org.obiba.plugins.spi.ServicePlugin; + +/** + * Makes a {@link StudyTableSource} from a URN. + */ +public interface StudyTableSourceService extends ServicePlugin { + + boolean isFor(String sourceURN); + + StudyTableSource makeSource(String sourceURN); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java new file mode 100644 index 0000000000..f458a6f4c0 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.dataset; + +import com.google.common.collect.Lists; + +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.ServiceLoader; + +/** + * {@link StudyTableSourceService} loader. + */ +public class StudyTableSourceServiceLoader { + + public static Collection get(URLClassLoader classLoader) { + return Lists.newArrayList(ServiceLoader.load(StudyTableSourceService.class, classLoader).iterator()); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java b/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java index fe42507de8..c2479efd9a 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java @@ -15,7 +15,7 @@ public interface SearchEngineService extends ServicePlugin { /** - * Provides some Mica configurations usefull for the search engine. + * Provides some Mica configurations useful for the search engine. * * @param configurationProvider */ diff --git a/mica-web-model/src/main/protobuf/Mica.proto b/mica-web-model/src/main/protobuf/Mica.proto index 4bcd968704..c480d0f2ca 100644 --- a/mica-web-model/src/main/protobuf/Mica.proto +++ b/mica-web-model/src/main/protobuf/Mica.proto @@ -542,8 +542,8 @@ message DatasetDto { extensions 1000 to max; message StudyTableDto { - required string project = 1; - required string table = 2; + optional string project = 1; + optional string table = 2; repeated LocalizedStringDto name = 3; repeated LocalizedStringDto description = 4; @@ -554,11 +554,12 @@ message DatasetDto { optional StudySummaryDto studySummary = 9; optional int32 weight = 10; repeated LocalizedStringDto additionalInformation = 11; + optional string sourceURN = 12; } message HarmonizationTableDto { - required string project = 1; - required string table = 2; + optional string project = 1; + optional string table = 2; repeated LocalizedStringDto name = 3; repeated LocalizedStringDto description = 4; @@ -566,6 +567,7 @@ message DatasetDto { optional StudySummaryDto studySummary = 7; optional int32 weight = 8; repeated LocalizedStringDto additionalInformation = 9; + optional string sourceURN = 10; } } @@ -665,6 +667,7 @@ message DatasetVariableResolverDto { repeated LocalizedStringDto populationName = 29; repeated LocalizedStringDto dceName = 30; optional string entityType = 31; + optional string sourceURN = 32; } message DatasetVariableSummaryDto { diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java index 3c7805e9f0..b4a6b7d4f9 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java @@ -83,7 +83,17 @@ public ModelAndView variable(@PathVariable String id) { break; } - DatasetVariable variable = resolver.getType().equals(DatasetVariable.Type.Harmonized) ? getHarmonizedDatasetVariable(resolver.getDatasetId(), id, variableName) : getDatasetVariable(id, variableName); + DatasetVariable variable; + if (resolver.getType().equals(DatasetVariable.Type.Harmonized)) { + try { + variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), resolver.getId(), variableName); + } catch (NoSuchVariableException e) { + // legacy variable id format + variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), id, variableName); + } + } else + variable = getDatasetVariable(resolver.getId(), variableName); + params.put("variable", variable); params.put("type", resolver.getType().toString()); @@ -163,10 +173,11 @@ private DatasetVariable getDatasetVariableInternal(String indexName, String inde } private DatasetVariable getHarmonizedDatasetVariable(String datasetId, String variableId, String variableName) { - String dataSchemaVariableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, DatasetVariable.Type.Dataschema, null, null, null, null); DatasetVariable harmonizedDatasetVariable = getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, variableId, variableName); + + String dataSchemaVariableId = DatasetVariable.IdResolver + .encode(datasetId, variableName, DatasetVariable.Type.Dataschema); DatasetVariable dataSchemaVariable = getDatasetVariableInternal(Indexer.PUBLISHED_VARIABLE_INDEX, Indexer.VARIABLE_TYPE, dataSchemaVariableId, variableName); @@ -208,13 +219,13 @@ private void addStudyTableParameters(Map params, DatasetVariable if (DatasetVariable.OpalTableType.Study.equals(variable.getOpalTableType())) { Optional studyTable = dataset.getStudyTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getProject().equals(st.getProject()) && variable.getTable().equals(st.getTable())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSourceURN().equals(st.getSourceURN())) .findFirst(); if (studyTable.isPresent()) params.put("opalTable", studyTable.get()); } else { Optional harmoStudyTable = dataset.getHarmonizationTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getProject().equals(st.getProject()) && variable.getTable().equals(st.getTable())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSourceURN().equals(st.getSourceURN())) .findFirst(); if (harmoStudyTable.isPresent()) params.put("opalTable", harmoStudyTable.get()); From bb6095623f878aee24341bca9648d3eab04727c4 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 1 Dec 2022 09:52:51 +0100 Subject: [PATCH 02/44] sourceURN property/getter/setter renamed to source --- .../mica/core/domain/BaseStudyTable.java | 20 ++++----- .../mica/core/domain/OpalTableSource.java | 16 +++---- .../obiba/mica/core/domain/StudyTable.java | 2 +- .../StudyTableSourceServiceRegistry.java | 17 ++++---- .../mica/dataset/domain/DatasetVariable.java | 38 ++++++++--------- .../mica/dataset/service/DatasetService.java | 2 +- .../service/HarmonizedDatasetService.java | 20 ++++----- .../dataset/service/VariableSetService.java | 6 +-- .../mica/micaConfig/service/CacheService.java | 2 +- .../org/obiba/mica/web/model/DatasetDtos.java | 42 +++++++++---------- .../service/CollectedDatasetServiceTest.java | 2 +- .../service/HarmonizedDatasetServiceTest.java | 3 +- .../support/HarmonizedDatasetHelperTest.java | 2 +- .../obiba/mica/web/model/DatasetDtosTest.java | 2 +- .../rest/PublishedDatasetResource.java | 2 +- .../entity/rql/RQLCriteriaOpalConverter.java | 3 +- .../rest/entity/rql/RQLFieldReferences.java | 2 +- ...raftDataschemaDatasetVariableResource.java | 3 +- ...raftHarmonizedDatasetVariableResource.java | 11 +++-- .../DraftDatasetVariableResource.java | 2 +- .../AbstractPublishedDatasetResource.java | 12 +++--- .../harmonization/CsvContingencyWriter.java | 4 +- .../CsvHarmonizationVariablesWriter.java | 2 +- ...shedDataschemaDatasetVariableResource.java | 4 +- .../PublishedHarmonizedDatasetResource.java | 6 +-- ...shedHarmonizedDatasetVariableResource.java | 24 +++++------ .../PublishedDatasetVariableResource.java | 3 +- .../spi/dataset/StudyTableSourceService.java | 4 +- mica-web-model/src/main/protobuf/Mica.proto | 6 +-- .../web/controller/VariableController.java | 4 +- 30 files changed, 130 insertions(+), 136 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java index 15ebe196d1..4fe35c9e10 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java @@ -29,7 +29,7 @@ public class BaseStudyTable { private int weight; - private String sourceURN; + private String source; // legacy private String project; @@ -86,7 +86,7 @@ public void setPopulationWeight(int populationWeight) { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("source", getSourceURN()) + return MoreObjects.toStringHelper(this).add("source", getSource()) .add("studyId", getStudyId()).add("populationId", getPopulationId()) .toString(); } @@ -123,20 +123,20 @@ public void setWeight(int weight) { this.weight = weight; } - public boolean isFor(String studyId, String sourceURN) { - return this.studyId.equals(studyId) && getSourceURN().equals(sourceURN); + public boolean isFor(String studyId, String source) { + return this.studyId.equals(studyId) && getSource().equals(source); } - public void setSourceURN(String sourceURN) { - this.sourceURN = sourceURN; + public void setSource(String source) { + this.source = source; } - public String getSourceURN() { + public String getSource() { // legacy - if (Strings.isNullOrEmpty(sourceURN)) { - this.sourceURN = OpalTableSource.newSource(project, table).getURN(); + if (Strings.isNullOrEmpty(source)) { + this.source = OpalTableSource.newSource(project, table).getURN(); } - return sourceURN; + return source; } @Deprecated diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java index fe19508893..ee13fa4ee9 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java @@ -72,8 +72,8 @@ public String getURN() { return String.format("urn:opal:%s.%s", project, table); } - public static boolean isFor(String sourceURN) { - return !Strings.isNullOrEmpty(sourceURN) && sourceURN.startsWith("urn:opal:"); + public static boolean isFor(String source) { + return !Strings.isNullOrEmpty(source) && source.startsWith("urn:opal:"); } public static OpalTableSource newSource(String project, String table) { @@ -83,19 +83,19 @@ public static OpalTableSource newSource(String project, String table) { return source; } - public static OpalTableSource fromURN(String sourceURN) { - if (Strings.isNullOrEmpty(sourceURN) || !sourceURN.startsWith("urn:opal:")) - throw new IllegalArgumentException("Not a valid Opal table source URN: " + sourceURN); + public static OpalTableSource fromURN(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:opal:")) + throw new IllegalArgumentException("Not a valid Opal table source URN: " + source); - String fullName = toTableName(sourceURN); + String fullName = toTableName(source); int sep = fullName.indexOf("."); String project = fullName.substring(0, sep); String table = fullName.substring(sep + 1); return OpalTableSource.newSource(project, table); } - public static String toTableName(String sourceURN) { - return sourceURN.replace("urn:opal:", ""); + public static String toTableName(String source) { + return source.replace("urn:opal:", ""); } public void init(OpalService opalService, String opalUrl) { diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java index e3036a78dc..2da6510811 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java @@ -63,7 +63,7 @@ public boolean appliesTo(String studyId, String populationId, String dataCollect @Override public String toString() { - return MoreObjects.toStringHelper(this).add("source", getSourceURN()) + return MoreObjects.toStringHelper(this).add("source", getSource()) .add("dceId", getDataCollectionEventUId()).toString(); } diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 7ad425a408..383edc360c 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -10,7 +10,6 @@ package org.obiba.mica.core.service; -import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.OpalTableSource; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.micaConfig.service.PluginsService; @@ -32,19 +31,19 @@ public class StudyTableSourceServiceRegistry { @Inject private OpalService opalService; - public StudyTableSource makeSource(BaseStudy study, String sourceURN) { - if (OpalTableSource.isFor(sourceURN)) { - OpalTableSource source = OpalTableSource.fromURN(sourceURN); - source.init(opalService, study.getOpal()); - return source; + public StudyTableSource makeStudyTableSource(BaseStudy study, String source) { + if (OpalTableSource.isFor(source)) { + OpalTableSource tableSource = OpalTableSource.fromURN(source); + tableSource.init(opalService, study.getOpal()); + return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() - .filter(service -> service.isFor(sourceURN)).findFirst(); + .filter(service -> service.isFor(source)).findFirst(); if (serviceOptional.isPresent()) { // TODO add a context to the study table source - return serviceOptional.get().makeSource(sourceURN); + return serviceOptional.get().makeSource(source); } - throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + sourceURN); + throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index 6618a77d98..bcd1ed33c2 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -89,7 +89,7 @@ public enum OpalTableType { private int index; - private String sourceURN; + private String source; private OpalTableType opalTableType; @@ -140,7 +140,7 @@ public DatasetVariable(HarmonizationDataset dataset, Variable variable, BaseStud populationId = studyTable.getPopulationUId(); dceId = studyTable.getDataCollectionEventUId(); - sourceURN = studyTable.getSourceURN(); + source = studyTable.getSource(); } private DatasetVariable(Dataset dataset, Type type, Variable variable) { @@ -177,7 +177,7 @@ public String getId() { if (Type.Harmonized == variableType) { String entityId = studyId; String tableType = opalTableType == OpalTableType.Study ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + sourceURN; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + source; } return id; @@ -192,7 +192,7 @@ public void setId(String id) { opalTableType = Strings.isNullOrEmpty(tableType) ? null : OpalTableType.valueOf(tableType); if (resolver.hasStudyId()) studyId = resolver.getStudyId(); - if (resolver.hasSourceURN()) sourceURN = resolver.getSourceURN(); + if (resolver.hasSource()) source = resolver.getSource(); } public String getDatasetId() { @@ -316,8 +316,8 @@ public int getIndex() { return index; } - public String getSourceURN() { - return sourceURN; + public String getSource() { + return source; } public OpalTableType getOpalTableType() { @@ -480,7 +480,7 @@ public static class IdResolver { private final String tableType; - private final String sourceURN; + private final String source; public static IdResolver from(String id) { return new IdResolver(id); @@ -495,14 +495,14 @@ public static String encode(String datasetId, String variableName, Type variable } public static String encode(String datasetId, String variableName, Type variableType, String studyId, - String sourceURN, String tableType) { + String source, String tableType) { String id = datasetId + ID_SEPARATOR + IdEncoderDecoder.encode(variableName) + ID_SEPARATOR + variableType; String entityId; if (Type.Harmonized == variableType) { entityId = studyId; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + sourceURN; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + source; } return id; @@ -516,7 +516,7 @@ public static String encode(String datasetId, String variableName, Type variable variableName, variableType, studyTable.getStudyId(), - studyTable.getSourceURN(), + studyTable.getSource(), tableType); } @@ -538,14 +538,14 @@ private IdResolver(String id) { studyId = tokens.length > 4 ? tokens[4] : null; if (parts.length>1) { - sourceURN = "urn:" + parts[1]; + source = "urn:" + parts[1]; } else if (tokens.length > 6) { // legacy - sourceURN = String.format("urn:opal:%s.%s", tokens[5], tokens[6]); + source = String.format("urn:opal:%s.%s", tokens[5], tokens[6]); // need to rewrite id - this.id = IdEncoderDecoder.encode(encode(datasetId, name, type,studyId, sourceURN, tableType)); + this.id = IdEncoderDecoder.encode(encode(datasetId, name, type,studyId, source, tableType)); } else { - sourceURN = null; + source = null; } } @@ -573,18 +573,18 @@ public boolean hasStudyId() { return !Strings.isNullOrEmpty(studyId); } - public boolean hasSourceURN() { - return !Strings.isNullOrEmpty(sourceURN); + public boolean hasSource() { + return !Strings.isNullOrEmpty(source); } - public String getSourceURN() { - return sourceURN; + public String getSource() { + return source; } @Override public String toString() { String tableType = type == Type.Dataschema ? OPAL_HARMONIZATION_TABLE_PREFIX : OPAL_STUDY_TABLE_PREFIX; - return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + sourceURN + "]"; + return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + source + "]"; } public String getTableType() { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index f061a9dcfc..304dae29e0 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -138,7 +138,7 @@ protected Iterable getVariables(@NotNull T dataset) } protected StudyTableSource getStudyTableSource(@NotNull BaseStudyTable studyTable) { - return studyTableSourceServiceRegistry.makeSource(getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSourceURN()); + return studyTableSourceServiceRegistry.makeStudyTableSource(getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSource()); } protected Iterable wrappedGetDatasetVariables(T dataset) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index d104912ff7..7a3ea9d3df 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -301,7 +301,7 @@ public void publish(@NotNull String id, boolean published, PublishCascadingScope private void checkIsPublishable(HarmonizationDataset dataset) { if (dataset == null || dataset.getHarmonizationTable() == null - || dataset.getHarmonizationTable().getSourceURN() == null + || dataset.getHarmonizationTable().getSource() == null || dataset.getHarmonizationTable().getStudyId() == null) { throw new IllegalArgumentException("dataset.harmonization.missing-attributes"); } @@ -372,18 +372,18 @@ public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String v } public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, String studyId, - String sourceURN) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { + String source) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { return new DatasetVariable(dataset, - getTable(dataset, studyId, sourceURN).getVariableValueSource(variableName).getVariable()); + getTable(dataset, studyId, source).getVariableValueSource(variableName).getVariable()); } @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", - key = "#variableName + ':' + #studyId + ':' + #sourceURN") - public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String sourceURN) + key = "#variableName + ':' + #studyId + ':' + #source") + public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String source) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, sourceURN); + log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, source); for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { - if(baseTable.isFor(studyId, sourceURN)) { + if(baseTable.isFor(studyId, source)) { return new SummaryStatisticsWrapper(getStudyTableSource(baseTable).getVariableSummary(variableName)); } } @@ -473,11 +473,11 @@ private Iterable getVariables(BaseStudyTable studyTable) return getStudyTableSource(studyTable).getValueTable().getVariables(); } - private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String sourceURN) + private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String source) throws NoSuchStudyException, NoSuchValueTableException { for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { - if(baseTable.isFor(studyId, sourceURN)) { + if(baseTable.isFor(studyId, source)) { return getStudyTableSource(baseTable).getValueTable(); } } @@ -542,7 +542,7 @@ public void asyncBuildDatasetVariablesCache(HarmonizationDataset dataset, dataset.getBaseStudyTables().forEach(st -> harmonizationVariables.forEach((k, v) -> v.forEach(var -> { try { String studyId = st.getStudyId(); - service.getVariableSummary(dataset, var.getName(), studyId, st.getSourceURN()); + service.getVariableSummary(dataset, var.getName(), studyId, st.getSource()); } catch(Exception e) { //ignoring } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java index a46d0c0fbc..cb4b7324ae 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java @@ -314,15 +314,15 @@ private List toCategoryDtoList(List categori private List toOpalTableFullName(Dataset dataset) { if (dataset instanceof StudyDataset) { StudyTable studyTable = ((StudyDataset) dataset).getSafeStudyTable(); - return Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSourceURN())); + return Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSource())); } else { HarmonizationDataset harmoDataset = (HarmonizationDataset) dataset; // one for each study and harmo tables List tableNames = Lists.newArrayList(); tableNames.addAll(harmoDataset.getStudyTables().stream() - .map(st -> OpalTableSource.toTableName(st.getSourceURN())).collect(Collectors.toList())); + .map(st -> OpalTableSource.toTableName(st.getSource())).collect(Collectors.toList())); tableNames.addAll(harmoDataset.getHarmonizationTables().stream() - .map(ht -> OpalTableSource.toTableName(ht.getSourceURN())).collect(Collectors.toList())); + .map(ht -> OpalTableSource.toTableName(ht.getSource())).collect(Collectors.toList())); return tableNames; } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index b40ca985c7..4e38583b5a 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -108,7 +108,7 @@ public void buildDatasetVariablesCache() { String studyId = st.getStudyId(); try { harmonizedDatasetService - .getVariableSummary(dataset, v.getName(), studyId, st.getSourceURN()); + .getVariableSummary(dataset, v.getName(), studyId, st.getSource()); } catch(NoSuchVariableException ex) { //ignore } catch(Exception e) { diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java index 9a1974315f..f9c5d1372d 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java @@ -182,10 +182,10 @@ Mica.DatasetVariableResolverDto.Builder asDto(@NotNull DatasetVariable.IdResolve if(resolver.hasStudyId()) { builder.setStudyId(resolver.getStudyId()); } - if(resolver.hasSourceURN()) { - builder.setSourceURN(resolver.getSourceURN()); - if (OpalTableSource.isFor(resolver.getSourceURN())) { - OpalTableSource source = OpalTableSource.fromURN(resolver.getSourceURN()); + if(resolver.hasSource()) { + builder.setSource(resolver.getSource()); + if (OpalTableSource.isFor(resolver.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(resolver.getSource()); builder.setProject(source.getProject()); builder.setTable(source.getTable()); } @@ -402,13 +402,13 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable) { public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable, boolean includeSummary) { Mica.DatasetDto.StudyTableDto.Builder sbuilder = Mica.DatasetDto.StudyTableDto.newBuilder() - .setSourceURN(studyTable.getSourceURN()) + .setSource(studyTable.getSource()) .setWeight(studyTable.getWeight()) .setStudyId(studyTable.getStudyId()) .setDceId(studyTable.getDataCollectionEventUId()); - if (OpalTableSource.isFor(studyTable.getSourceURN())) { - OpalTableSource source = OpalTableSource.fromURN(studyTable.getSourceURN()); + if (OpalTableSource.isFor(studyTable.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(studyTable.getSource()); sbuilder.setProject(source.getProject()); sbuilder.setTable(source.getTable()); } @@ -439,12 +439,12 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTable harmonizationTable, boolean includeSummary) { Mica.DatasetDto.HarmonizationTableDto.Builder hBuilder = Mica.DatasetDto.HarmonizationTableDto.newBuilder() - .setSourceURN(harmonizationTable.getSourceURN()) + .setSource(harmonizationTable.getSource()) .setWeight(harmonizationTable.getWeight()) .setStudyId(harmonizationTable.getStudyId()); - if (OpalTableSource.isFor(harmonizationTable.getSourceURN())) { - OpalTableSource source = OpalTableSource.fromURN(harmonizationTable.getSourceURN()); + if (OpalTableSource.isFor(harmonizationTable.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationTable.getSource()); hBuilder.setProject(source.getProject()); hBuilder.setTable(source.getTable()); } @@ -785,9 +785,9 @@ private Dataset fromDto(@NotNull Mica.HarmonizedDatasetDto dto) { HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); // legacy if (dto.getHarmonizationTable().hasProject() && dto.getHarmonizationTable().hasTable()) { - harmonizationLink.setSourceURN(makeSourceURN(dto.getHarmonizationTable().getProject(), dto.getHarmonizationTable().getTable())); + harmonizationLink.setSource(makesource(dto.getHarmonizationTable().getProject(), dto.getHarmonizationTable().getTable())); } else { - harmonizationLink.setSourceURN(dto.getHarmonizationTable().getSourceURN()); + harmonizationLink.setSource(dto.getHarmonizationTable().getSource()); } harmonizationLink.setStudyId(dto.getHarmonizationTable().getStudyId()); harmonizationDataset.setHarmonizationTable(harmonizationLink); @@ -817,9 +817,9 @@ private StudyTable fromDto(Mica.DatasetDto.StudyTableDto dto) { table.setAdditionalInformation(localizedStringDtos.fromDto(dto.getAdditionalInformationList())); if (dto.hasProject() && dto.hasTable()) { - table.setSourceURN(makeSourceURN(dto.getProject(), dto.getTable())); + table.setSource(makesource(dto.getProject(), dto.getTable())); } else { - table.setSourceURN(dto.getSourceURN()); + table.setSource(dto.getSource()); } return table; @@ -835,16 +835,16 @@ private HarmonizationStudyTable fromDto(Mica.DatasetDto.HarmonizationTableDto dt // legacy if (dto.hasProject() && dto.hasTable()) { - table.setSourceURN(makeSourceURN(dto.getProject(), dto.getTable())); + table.setSource(makesource(dto.getProject(), dto.getTable())); } else { - table.setSourceURN(dto.getSourceURN()); + table.setSource(dto.getSource()); } return table; } // legacy - private String makeSourceURN(String project, String table) { + private String makesource(String project, String table) { return OpalTableSource.newSource(project, table).getURN(); } @@ -853,10 +853,10 @@ private Mica.DatasetDto.HarmonizationTableDto.Builder createHarmonizationLinkDto Mica.DatasetDto.HarmonizationTableDto.Builder harmonizationLinkBuilder = Mica.DatasetDto.HarmonizationTableDto .newBuilder(); - if(!Strings.isNullOrEmpty(harmonizationLink.getSourceURN())) { - harmonizationLinkBuilder.setSourceURN(harmonizationLink.getSourceURN()); - if (OpalTableSource.isFor(harmonizationLink.getSourceURN())) { - OpalTableSource source = OpalTableSource.fromURN(harmonizationLink.getSourceURN()); + if(!Strings.isNullOrEmpty(harmonizationLink.getSource())) { + harmonizationLinkBuilder.setSource(harmonizationLink.getSource()); + if (OpalTableSource.isFor(harmonizationLink.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationLink.getSource()); harmonizationLinkBuilder.setProject(source.getProject()); harmonizationLinkBuilder.setTable(source.getTable()); } diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java index 7290609736..dd033f9521 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java @@ -99,7 +99,7 @@ public void testDatasourceConnectionErrorIsIgnoredForDraft() { private StudyDataset buildStudyDataset() { StudyDataset ds = new StudyDataset(); StudyTable st = new StudyTable(); - st.setSourceURN(OpalTableSource.newSource("proj", "tab").getURN()); + st.setSource(OpalTableSource.newSource("proj", "tab").getURN()); st.setPopulationId("1"); st.setDataCollectionEventId("1"); ds.setStudyTable(st); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java index 9ff32e5db6..26d879f9d7 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java @@ -13,7 +13,6 @@ import java.util.Locale; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.Supplier; import org.junit.Before; @@ -99,7 +98,7 @@ private HarmonizationDataset buildHarmonizationDataset(String id, StudyTable... private StudyTable buildStudyTable(String project, String table, String studyId) { StudyTable st = new StudyTable(); - st.setSourceURN(OpalTableSource.newSource(project, table).getURN()); + st.setSource(OpalTableSource.newSource(project, table).getURN()); st.setStudyId(studyId); st.setPopulationId("pop"); st.setDataCollectionEventId("ev"); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java index 2736b5acdb..8aa3fcb745 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java @@ -110,7 +110,7 @@ private void initTable(String studyId, BaseStudyTable table) { table.setStudyId(studyId); String project = RandomStringUtils.random(10, true, false); String tbl = RandomStringUtils.random(10, true, false); - table.setSourceURN(OpalTableSource.newSource(project, tbl).getURN()); + table.setSource(OpalTableSource.newSource(project, tbl).getURN()); table.setPopulationId(RandomStringUtils.random(10, true, true)); } } diff --git a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java index 3a31dd978f..88b7c40a0f 100644 --- a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java +++ b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java @@ -109,7 +109,7 @@ private HarmonizationDataset createHarmonizedDataset() { HarmonizationDataset harmonizationDataset = new HarmonizationDataset(); harmonizationDataset.setId("123"); HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); - harmonizationLink.setSourceURN(OpalTableSource.newSource("project123", "table123").getURN()); + harmonizationLink.setSource(OpalTableSource.newSource("project123", "table123").getURN()); harmonizationLink.setStudyId("study123"); harmonizationLink.setPopulationId("population123"); harmonizationDataset.setHarmonizationTable(harmonizationLink); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java index 1c4e033b4d..080543a4a6 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java @@ -89,7 +89,7 @@ private StudyDataset alternativeStudyDataset(String id, String project, String t if (!(dataset instanceof StudyDataset)) throw NoSuchDatasetException.withId(id); StudyDataset asStudyDataset = (StudyDataset) dataset; - asStudyDataset.getStudyTable().setSourceURN(OpalTableSource.newSource(project, table).getURN()); + asStudyDataset.getStudyTable().setSource(OpalTableSource.newSource(project, table).getURN()); return asStudyDataset; } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java index 01ae1090c0..e51c3c4b9a 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java @@ -30,7 +30,6 @@ import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.study.domain.BaseStudy; -import org.obiba.mica.study.domain.HarmonizationStudy; import org.obiba.mica.study.service.StudyService; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -182,7 +181,7 @@ private RQLFieldReferences parseField(String path) { } else if (DatasetVariable.Type.Harmonized.equals(resolver.getType())) { HarmonizationDataset ds = harmonizedDatasetService.findById(resolver.getDatasetId()); Optional studyTable = ds.getBaseStudyTables().stream().filter(st -> st.getStudyId().equals(resolver.getStudyId()) - && st.getSourceURN().equals(resolver.getSourceURN())).findFirst(); + && st.getSource().equals(resolver.getSource())).findFirst(); if (!studyTable.isPresent()) throw new IllegalArgumentException("Not a valid variable: " + path); BaseStudy study = studyService.findStudy(studyTable.get().getStudyId()); return new RQLFieldReferences(path, ds, studyTable.get(), study, getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, path)); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java index 6c59af7c68..bda2f47579 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java @@ -89,6 +89,6 @@ public LocalizedString getStudyTableName() { * @return */ private String getOpalTablePath(BaseStudyTable studyTable) { - return studyTable.getSourceURN().replace("urn:opal:", ""); + return studyTable.getSource().replace("urn:opal:", ""); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java index e704dadfd7..93d6a0c355 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java @@ -24,7 +24,6 @@ import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -58,7 +57,7 @@ public List getVariableSummaries() { try { String studyId = table.getStudyId(); builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) + .getVariableSummary(dataset, variableName, studyId, table.getSource()) .getWrappedDto()); } catch(NoSuchVariableException | NoSuchValueTableException e) { // ignore (case the study has not implemented this dataschema variable) diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java index eb13792215..d335eda792 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java @@ -19,7 +19,6 @@ import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -39,25 +38,25 @@ public class DraftHarmonizedDatasetVariableResource implements DatasetVariableRe private String studyId; - private String sourceURN; + private String source; @GET public Mica.DatasetVariableDto getVariable() { - return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, sourceURN)); + return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, source)); } @GET @Path("/summary") public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName, studyId, sourceURN).getWrappedDto(); + return datasetService.getVariableSummary(getDataset(), variableName, studyId, source).getWrappedDto(); } public void setStudyId(String studyId) { this.studyId = studyId; } - public void setSourceURN(String sourceURN) { - this.sourceURN = sourceURN; + public void setSource(String source) { + this.source = source; } private HarmonizationDataset getDataset() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java index 589a2b88f3..40fc090bac 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java @@ -52,7 +52,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id) { subjectAclService.isPermitted("/draft/harmonized-dataset", "VIEW", resolver.getDatasetId()); resource = applicationContext.getBean(DraftHarmonizedDatasetVariableResource.class); ((DraftHarmonizedDatasetVariableResource) resource).setStudyId(resolver.getStudyId()); - ((DraftHarmonizedDatasetVariableResource) resource).setSourceURN(resolver.getSourceURN()); + ((DraftHarmonizedDatasetVariableResource) resource).setSource(resolver.getSource()); break; } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java index e4e1ca09a7..ce8c28ba2f 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java @@ -215,7 +215,7 @@ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull variableName, variableType, studyTable.getStudyId(), - studyTable.getSourceURN(), + studyTable.getSource(), studyTable instanceof StudyTable ? DatasetVariable.OPAL_STUDY_TABLE_PREFIX : DatasetVariable.OPAL_HARMONIZATION_TABLE_PREFIX); @@ -232,18 +232,18 @@ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull * @param datasetId * @param variableName * @param studyId - * @param sourceURN + * @param source * @param tableType * @return * @throws NoSuchVariableException */ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String sourceURN, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String source, @Nullable String tableType) throws NoSuchVariableException { String variableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, variableType, studyId, sourceURN, tableType); + .encode(datasetId, variableName, variableType, studyId, source, tableType); if (variableType.equals(DatasetVariable.Type.Harmonized)) { return getHarmonizedDatasetVariable(datasetId, variableId, variableName); @@ -265,9 +265,9 @@ protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetI } protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String sourceURN, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String source, @Nullable String tableType) { - return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, sourceURN, tableType), + return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, source, tableType), getTaxonomies(), getLocale()); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java index 8d4aa9e559..aa2651989f 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java @@ -84,9 +84,9 @@ private void writeBody(CSVWriter writer, Mica.DatasetVariableContingencyDto dto) private void writeHeaders(CSVWriter writer, Mica.DatasetVariableContingencyDto c, List terms) { if(c.hasStudyTable()) writer.writeNext(new String[] { String - .format("%s - %s", c.getStudyTable().getSourceURN(), c.getStudyTable().getDceId()) }); + .format("%s - %s", c.getStudyTable().getSource(), c.getStudyTable().getDceId()) }); else if(c.hasHarmonizationStudyTable()) writer.writeNext(new String[] { String - .format("%s", c.getHarmonizationStudyTable().getSourceURN())}); + .format("%s", c.getHarmonizationStudyTable().getSource())}); writer.writeNext(concat(concat(Stream.of(""), terms.stream()), Stream.of("Total")).toArray(String[]::new)); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java index 02c1df3913..c0adeb06ab 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java @@ -86,7 +86,7 @@ private void writeBody(CSVWriter writer, HarmonizationDataset dataset, String id = table.getStudyId(); Mica.DatasetVariableResolverDto resolver = summary.getResolver(); if ((resolver.getStudyId().equals(id) - && resolver.getSourceURN().equals(table.getSourceURN()))) { + && resolver.getSource().equals(table.getSource()))) { String statusDetail = getAttributeByName(summary, "status_detail", locale); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index ab922b0e45..4bb7565a9b 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -82,7 +82,7 @@ public List getVariableSummaries() { try { String studyId = table.getStudyId(); builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) + .getVariableSummary(dataset, variableName, studyId, table.getSource()) .getWrappedDto()); } catch (NoSuchVariableException | NoSuchValueTableException e) { // case the study has not implemented this dataschema variable @@ -268,7 +268,7 @@ protected Future getVariableFacet(HarmonizationDatase try { String studyId = table.getStudyId(); return new AsyncResult<>(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSourceURN()) + .getVariableSummary(dataset, variableName, studyId, table.getSource()) .getWrappedDto()); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java index a9cfa14e22..1f89d56c96 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java @@ -216,16 +216,16 @@ private Mica.DatasetVariablesHarmonizationSummaryDto getVariableHarmonizationsSu List> harmonizedVariables = Lists.newArrayList(); builder.setTotal(variablesDto.getTotal()).setLimit(variablesDto.getLimit()).setFrom(variablesDto.getFrom()); - String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(sourceURN,%s))"; + String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(source,%s))"; dataset.getStudyTables().forEach(st -> { builder.addStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getSourceURN()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getSource()); List datasetVariablesInternal = getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true); harmonizedVariables.add(datasetVariablesInternal.stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); dataset.getHarmonizationTables().forEach(st -> { builder.addHarmonizationStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getSourceURN()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getSource()); harmonizedVariables.add(getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true).stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index b07f5961f9..4ccddb1ebc 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -48,7 +48,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe private String studyId; - private String sourceURN; + private String source; private String tableType; @@ -59,7 +59,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe @Timed public Mica.DatasetVariableDto getVariable() { checkDatasetAccess(); - return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); + return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, source, tableType); } @GET @@ -69,7 +69,7 @@ public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { checkDatasetAccess(); checkVariableSummaryAccess(); return datasetService - .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, sourceURN) + .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, source) .getWrappedDto(); } @@ -81,10 +81,10 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s checkVariableSummaryAccess(); HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { - if (baseTable.isFor(studyId, sourceURN)) { + if (baseTable.isFor(studyId, source)) { try { return dtos.asDto(baseTable, - datasetService.getVariableSummary(dataset, variableName, studyId, sourceURN).getWrappedDto(), withStudySummary).build(); + datasetService.getVariableSummary(dataset, variableName, studyId, source).getWrappedDto(), withStudySummary).build(); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); return dtos.asDto(baseTable, null, withStudySummary).build(); @@ -92,7 +92,7 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s } } - throw new NoSuchValueTableException(sourceURN); + throw new NoSuchValueTableException(source); } @GET @@ -109,7 +109,7 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { - if (baseTable.isFor(studyId, sourceURN)) { + if (baseTable.isFor(studyId, source)) { try { return dtos.asContingencyDto(baseTable, var, crossVar, datasetService.getContingencyTable(baseTable, var, crossVar)).build(); @@ -120,7 +120,7 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var } } - throw new NoSuchValueTableException(sourceURN); + throw new NoSuchValueTableException(source); } @GET @@ -167,8 +167,8 @@ public void setStudyId(String studyId) { this.studyId = studyId; } - public void setSourceURN(String sourceURN) { - this.sourceURN = sourceURN; + public void setSource(String source) { + this.source = source; } public void setTableType(String tableType) { @@ -179,8 +179,8 @@ private Pair getContingencyVariables(String cr if (Strings.isNullOrEmpty(crossVariable)) throw new BadRequestException("Cross variable name is required for the contingency table"); - DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); - DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, sourceURN, tableType); + DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, source, tableType); + DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, source, tableType); return Pair.create(var, crossVar); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java index ae337460a2..e3523ff049 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java @@ -16,7 +16,6 @@ import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; -import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.DatasetVariable; @@ -64,7 +63,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id, if (!harmonizedDatasetService.isPublished(resolver.getDatasetId())) throw NoSuchDatasetException.withId(resolver.getDatasetId()); resource = applicationContext.getBean(PublishedHarmonizedDatasetVariableResource.class); ((PublishedHarmonizedDatasetVariableResource)resource).setStudyId(resolver.getStudyId()); - ((PublishedHarmonizedDatasetVariableResource)resource).setSourceURN(resolver.getSourceURN()); + ((PublishedHarmonizedDatasetVariableResource)resource).setSource(resolver.getSource()); ((PublishedHarmonizedDatasetVariableResource)resource).setTableType(resolver.getTableType()); ((PublishedHarmonizedDatasetVariableResource)resource).setLocale(locale); break; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java index 6d4901c3fc..277dd7f97c 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java @@ -17,8 +17,8 @@ */ public interface StudyTableSourceService extends ServicePlugin { - boolean isFor(String sourceURN); + boolean isFor(String source); - StudyTableSource makeSource(String sourceURN); + StudyTableSource makeSource(String source); } diff --git a/mica-web-model/src/main/protobuf/Mica.proto b/mica-web-model/src/main/protobuf/Mica.proto index c480d0f2ca..5f1a4dc319 100644 --- a/mica-web-model/src/main/protobuf/Mica.proto +++ b/mica-web-model/src/main/protobuf/Mica.proto @@ -554,7 +554,7 @@ message DatasetDto { optional StudySummaryDto studySummary = 9; optional int32 weight = 10; repeated LocalizedStringDto additionalInformation = 11; - optional string sourceURN = 12; + optional string source = 12; } message HarmonizationTableDto { @@ -567,7 +567,7 @@ message DatasetDto { optional StudySummaryDto studySummary = 7; optional int32 weight = 8; repeated LocalizedStringDto additionalInformation = 9; - optional string sourceURN = 10; + optional string source = 10; } } @@ -667,7 +667,7 @@ message DatasetVariableResolverDto { repeated LocalizedStringDto populationName = 29; repeated LocalizedStringDto dceName = 30; optional string entityType = 31; - optional string sourceURN = 32; + optional string source = 32; } message DatasetVariableSummaryDto { diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java index b4a6b7d4f9..681e8a8c97 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java @@ -219,13 +219,13 @@ private void addStudyTableParameters(Map params, DatasetVariable if (DatasetVariable.OpalTableType.Study.equals(variable.getOpalTableType())) { Optional studyTable = dataset.getStudyTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getSourceURN().equals(st.getSourceURN())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSource().equals(st.getSource())) .findFirst(); if (studyTable.isPresent()) params.put("opalTable", studyTable.get()); } else { Optional harmoStudyTable = dataset.getHarmonizationTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getSourceURN().equals(st.getSourceURN())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSource().equals(st.getSource())) .findFirst(); if (harmoStudyTable.isPresent()) params.put("opalTable", harmoStudyTable.get()); From 0a03355fac8002792b5b8df168ec9d3a7ec57e6e Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 1 Dec 2022 10:20:15 +0100 Subject: [PATCH 03/44] source package added, with unit tests --- .../mica/core/domain/BaseStudyTable.java | 1 + .../StudyTableSourceServiceRegistry.java | 6 ++--- .../{domain => source}/OpalTableSource.java | 4 +-- .../mica/dataset/service/DatasetService.java | 2 +- .../dataset/service/VariableSetService.java | 1 + .../micaConfig/service/PluginsService.java | 4 +-- .../org/obiba/mica/web/model/DatasetDtos.java | 1 + .../mica/core/source/OpalTableSourceTest.java | 25 +++++++++++++++++++ .../service/CollectedDatasetServiceTest.java | 2 +- .../service/HarmonizedDatasetServiceTest.java | 2 +- .../support/HarmonizedDatasetHelperTest.java | 2 +- .../obiba/mica/web/model/DatasetDtosTest.java | 2 +- .../rest/PublishedDatasetResource.java | 2 +- .../{dataset => source}/StudyTableSource.java | 2 +- .../StudyTableSourceService.java | 2 +- .../StudyTableSourceServiceLoader.java | 2 +- 16 files changed, 44 insertions(+), 16 deletions(-) rename mica-core/src/main/java/org/obiba/mica/core/{domain => source}/OpalTableSource.java (97%) create mode 100644 mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java rename mica-spi/src/main/java/org/obiba/mica/spi/{dataset => source}/StudyTableSource.java (97%) rename mica-spi/src/main/java/org/obiba/mica/spi/{dataset => source}/StudyTableSourceService.java (94%) rename mica-spi/src/main/java/org/obiba/mica/spi/{dataset => source}/StudyTableSourceServiceLoader.java (95%) diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java index 4fe35c9e10..51b9cb5169 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java @@ -12,6 +12,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Strings; +import org.obiba.mica.core.source.OpalTableSource; public class BaseStudyTable { diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 383edc360c..3b06a109a2 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -10,11 +10,11 @@ package org.obiba.mica.core.service; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.micaConfig.service.PluginsService; -import org.obiba.mica.spi.dataset.StudyTableSource; -import org.obiba.mica.spi.dataset.StudyTableSourceService; +import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.spi.source.StudyTableSourceService; import org.obiba.mica.study.domain.BaseStudy; import org.springframework.stereotype.Service; diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java similarity index 97% rename from mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java rename to mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index ee13fa4ee9..3dfa63925d 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -8,12 +8,12 @@ * along with this program. If not, see . */ -package org.obiba.mica.core.domain; +package org.obiba.mica.core.source; import com.google.common.base.Strings; import org.obiba.magma.ValueTable; import org.obiba.mica.micaConfig.service.OpalService; -import org.obiba.mica.spi.dataset.StudyTableSource; +import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestValueTable; import org.obiba.opal.web.model.Math; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 304dae29e0..4b2c836015 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -25,7 +25,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; -import org.obiba.mica.spi.dataset.StudyTableSource; +import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.mica.study.service.StudyService; import org.obiba.opal.web.model.Math; import org.slf4j.Logger; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java index cb4b7324ae..f0da0ed3a1 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java @@ -15,6 +15,7 @@ import com.googlecode.protobuf.format.JsonFormat; import org.obiba.mica.core.domain.*; import org.obiba.mica.core.service.DocumentSetService; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.domain.*; import org.obiba.mica.micaConfig.domain.MicaConfig; import org.obiba.mica.study.service.PublishedDatasetVariableService; diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index a42198fe64..cb734f1605 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -16,8 +16,8 @@ import com.google.common.io.Files; import org.obiba.core.util.FileUtil; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; -import org.obiba.mica.spi.dataset.StudyTableSourceService; -import org.obiba.mica.spi.dataset.StudyTableSourceServiceLoader; +import org.obiba.mica.spi.source.StudyTableSourceService; +import org.obiba.mica.spi.source.StudyTableSourceServiceLoader; import org.obiba.mica.spi.search.ConfigurationProvider; import org.obiba.mica.spi.search.SearchEngineService; import org.obiba.mica.spi.search.SearchEngineServiceLoader; diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java index f9c5d1372d..602b805943 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java @@ -16,6 +16,7 @@ import org.obiba.magma.type.BooleanType; import org.obiba.mica.JSONUtils; import org.obiba.mica.core.domain.*; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.StudyDatasetStateRepository; import org.obiba.mica.dataset.domain.*; diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java new file mode 100644 index 0000000000..b64cf1999b --- /dev/null +++ b/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OpalTableSourceTest { + + @Test + public void test_urn_parse() { + OpalTableSource source = OpalTableSource.fromURN("urn:opal:MyProject.MyTable"); + assertThat(source.getProject()).isEqualTo("MyProject"); + assertThat(source.getTable()).isEqualTo("MyTable"); + } +} diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java index dd033f9521..c359a8c396 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java @@ -19,7 +19,7 @@ import org.mockito.MockitoAnnotations; import org.obiba.magma.MagmaRuntimeException; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.core.service.GitService; import org.obiba.mica.dataset.StudyDatasetRepository; diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java index 26d879f9d7..4e70e0fa99 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java @@ -23,7 +23,7 @@ import org.mockito.Spy; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.magma.Variable; import org.obiba.magma.type.BooleanType; diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java index 8aa3fcb745..ccb26edad6 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java @@ -19,7 +19,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import static org.obiba.mica.assertj.Assertions.assertThat; diff --git a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java index 88b7c40a0f..7a4010b262 100644 --- a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java +++ b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java @@ -18,7 +18,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.domain.HarmonizationDatasetState; diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java index 080543a4a6..886332f79d 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java @@ -17,7 +17,7 @@ import javax.ws.rs.PathParam; import org.apache.shiro.SecurityUtils; -import org.obiba.mica.core.domain.OpalTableSource; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.domain.DatasetVariable; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java similarity index 97% rename from mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java rename to mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java index 88aedb999c..59eea7b57e 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.dataset; +package org.obiba.mica.spi.source; import org.obiba.magma.ValueTable; import org.obiba.opal.web.model.Math; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java similarity index 94% rename from mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java rename to mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java index 277dd7f97c..94727b4659 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.dataset; +package org.obiba.mica.spi.source; import org.obiba.plugins.spi.ServicePlugin; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java similarity index 95% rename from mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java rename to mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java index f458a6f4c0..865ab098fd 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/dataset/StudyTableSourceServiceLoader.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.dataset; +package org.obiba.mica.spi.source; import com.google.common.collect.Lists; From 36bac703f9551bbfa4f12a5049217d922724e3ef Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 1 Dec 2022 15:12:44 +0100 Subject: [PATCH 04/44] Excel table source added, reads excel from Mica's file system --- mica-core/pom.xml | 4 + .../StudyTableSourceServiceRegistry.java | 37 ++++++- .../mica/core/source/ExcelTableSource.java | 96 +++++++++++++++++++ .../mica/core/source/OpalTableSource.java | 3 +- .../core/source/ExcelTableSourceTest.java | 37 +++++++ .../mica/spi/source/StudyTableFileSource.java | 25 +++++ .../mica/spi/source/StudyTableSource.java | 7 ++ .../spi/source/StudyTableStreamSource.java | 27 ++++++ pom.xml | 7 ++ 9 files changed, 240 insertions(+), 3 deletions(-) create mode 100644 mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java create mode 100644 mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java diff --git a/mica-core/pom.xml b/mica-core/pom.xml index 81239b7076..c49c90b33a 100644 --- a/mica-core/pom.xml +++ b/mica-core/pom.xml @@ -34,6 +34,10 @@ org.obiba.mica mica-web-model + + org.obiba.magma + magma-datasource-excel + com.codahale.metrics metrics-core diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 3b06a109a2..c3b4038983 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -10,40 +10,73 @@ package org.obiba.mica.core.service; +import org.apache.commons.math3.util.Pair; +import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; +import org.obiba.mica.file.AttachmentState; +import org.obiba.mica.file.FileStoreService; +import org.obiba.mica.file.service.FileSystemService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.spi.source.StudyTableFileSource; import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.mica.spi.source.StudyTableSourceService; import org.obiba.mica.study.domain.BaseStudy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; +import java.io.InputStream; import java.util.NoSuchElementException; import java.util.Optional; @Service public class StudyTableSourceServiceRegistry { + private static final Logger log = LoggerFactory.getLogger(StudyTableSourceServiceRegistry.class); + @Inject private PluginsService pluginsService; @Inject private OpalService opalService; + @Inject + protected FileSystemService fileSystemService; + + @Inject + private FileStoreService fileStoreService; + public StudyTableSource makeStudyTableSource(BaseStudy study, String source) { if (OpalTableSource.isFor(source)) { OpalTableSource tableSource = OpalTableSource.fromURN(source); - tableSource.init(opalService, study.getOpal()); + tableSource.initialise(opalService, study.getOpal()); + return tableSource; + } + if (ExcelTableSource.isFor(source)) { + ExcelTableSource tableSource = ExcelTableSource.fromURN(source); + tableSource.initialise(getFileInputStream(tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() .filter(service -> service.isFor(source)).findFirst(); if (serviceOptional.isPresent()) { // TODO add a context to the study table source - return serviceOptional.get().makeSource(source); + StudyTableSource tableSource = serviceOptional.get().makeSource(source); + if (tableSource instanceof StudyTableFileSource) { + StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; + fileSource.initialise(getFileInputStream(fileSource.getPath())); + } + return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } + private InputStream getFileInputStream(String path) { + Pair pathName = FileSystemService.extractPathName(path); + AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); + return fileStoreService.getFile(state.getAttachment().getFileReference()); + } + } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java new file mode 100644 index 0000000000..b8ebcf181c --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import org.obiba.magma.ValueTable; +import org.obiba.magma.datasource.excel.ExcelDatasource; +import org.obiba.magma.support.Initialisables; +import org.obiba.mica.spi.source.StudyTableFileSource; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +import javax.validation.constraints.NotNull; +import java.io.InputStream; +import java.util.List; + +public class ExcelTableSource implements StudyTableFileSource { + + @NotNull + private String path; + + @NotNull + private String table; + + private ExcelDatasource excelDatasource; + + public static boolean isFor(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) + return false; + List tokens = Splitter.on(":").splitToList(source); + return tokens.size() == 4 && tokens.get(2).toLowerCase().endsWith(".xlsx"); + } + + public static ExcelTableSource fromURN(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) + throw new IllegalArgumentException("Not a valid Excel table source URN: " + source); + + String fullName = source.replace("urn:file:", ""); + int sep = fullName.lastIndexOf(":"); + String file = fullName.substring(0, sep); + String table = fullName.substring(sep + 1); + return ExcelTableSource.newSource(file, table); + } + + private static ExcelTableSource newSource(String path, String table) { + ExcelTableSource source = new ExcelTableSource(); + source.path = path; + source.table = table; + return source; + } + + @Override + public String getPath() { + return path; + } + + @Override + public String getTable() { + return table; + } + + @Override + public ValueTable getValueTable() { + return excelDatasource.getValueTable(table); + } + + @Override + public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { + throw new UnsupportedOperationException("Facet search not available from an Excel file"); + } + + @Override + public Math.SummaryStatisticsDto getVariableSummary(String variableName) { + throw new UnsupportedOperationException("Summary statistics not available from an Excel file"); + } + + @Override + public String getURN() { + return String.format("urn:file:%s:%s", path, table); + } + + @Override + public void initialise(InputStream in) { + excelDatasource = new ExcelDatasource(path, in); + Initialisables.initialise(excelDatasource); + } +} diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index 3dfa63925d..e5b497a659 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -44,6 +44,7 @@ public void setProject(String project) { this.project = project; } + @Override public String getTable() { return table; } @@ -98,7 +99,7 @@ public static String toTableName(String source) { return source.replace("urn:opal:", ""); } - public void init(OpalService opalService, String opalUrl) { + public void initialise(OpalService opalService, String opalUrl) { this.opalService = opalService; this.opalUrl = opalUrl; } diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java new file mode 100644 index 0000000000..fcb9991c4b --- /dev/null +++ b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExcelTableSourceTest { + + @Test + public void test_is_for() { + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xlsx:MyTable")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:/path/to/MyProject.xlsx:MyTable")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xls")).isFalse(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.spss")).isFalse(); + } + + @Test + public void test_urn_parse() { + ExcelTableSource source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx:MyTable"); + assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); + assertThat(source.getTable()).isEqualTo("MyTable"); + + source = ExcelTableSource.fromURN("urn:file:/path/to/MyProject.xlsx:MyTable"); + assertThat(source.getPath()).isEqualTo("/path/to/MyProject.xlsx"); + assertThat(source.getTable()).isEqualTo("MyTable"); + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java new file mode 100644 index 0000000000..8d06cb9fd6 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +/** + * Study table is to be extracted from a file which path applies to the Mica's internal file system. + */ +public interface StudyTableFileSource extends StudyTableStreamSource { + + /** + * The path to the file in Mica's file system. + * + * @return + */ + String getPath(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java index 59eea7b57e..3803f824e1 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java @@ -20,6 +20,13 @@ */ public interface StudyTableSource { + /** + * Get the associated table name. + * + * @return + */ + String getTable(); + /** * Get the {@link ValueTable} implementing the data dictionary and the data values. * diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java new file mode 100644 index 0000000000..a78177e8a9 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import java.io.InputStream; + +/** + * Study table is to be extracted from an {@link InputStream}. + */ +public interface StudyTableStreamSource extends StudyTableSource { + + /** + * Initialise the source from the given input stream. + * + * @param in + */ + void initialise(InputStream in); + +} diff --git a/pom.xml b/pom.xml index 77dd43480c..0f6ac2edce 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 3.0.0 1.2.9 6.6 + 4.0.1 3.0.0 4.7.2 0.7.9 @@ -422,6 +423,12 @@ cli + + org.obiba.magma + magma-datasource-excel + ${magma.version} + + org.obiba.opal opal-web-model From ce06962fa706405cf23d6aa290d649ffcd78fadc Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 2 Dec 2022 14:12:22 +0100 Subject: [PATCH 05/44] Edit/create study tables with opal or file source. WIP. --- .../StudyTableSourceServiceRegistry.java | 42 +++++++-- .../mica/core/source/ExcelTableSource.java | 12 +-- .../core/source/ExcelTableSourceTest.java | 5 ++ .../mica/spi/source/StudyTableFileSource.java | 13 ++- .../spi/source/StudyTableStreamSource.java | 27 ------ mica-webapp/src/main/resources/i18n/en.json | 21 ++++- mica-webapp/src/main/resources/i18n/fr.json | 18 +++- .../webapp/app/dataset/dataset-controller.js | 71 +++++++++++---- .../webapp/app/dataset/dataset-service.js | 54 +++++++++++ .../views/collected-dataset-view-details.html | 56 +++++++++--- .../views/dataset-study-table-form.html | 4 +- .../views/harmonization-dataset-form.html | 90 +++++++++++++------ .../harmonization-study-tables-form.html | 4 +- .../harmonized-dataset-view-details.html | 60 +++++++++---- .../dataset/views/opal-table-modal-form.html | 84 +++++++++++------ .../main/webapp/assets/js/mica-variable.js | 2 +- 16 files changed, 413 insertions(+), 150 deletions(-) delete mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index c3b4038983..5ecf52d903 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -10,7 +10,10 @@ package org.obiba.mica.core.service; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.commons.math3.util.Pair; +import org.obiba.magma.NoSuchValueTableException; import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.file.AttachmentState; @@ -22,14 +25,18 @@ import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.mica.spi.source.StudyTableSourceService; import org.obiba.mica.study.domain.BaseStudy; +import org.obiba.mica.study.domain.Study; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; import java.io.InputStream; +import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; @Service public class StudyTableSourceServiceRegistry { @@ -48,7 +55,17 @@ public class StudyTableSourceServiceRegistry { @Inject private FileStoreService fileStoreService; - public StudyTableSource makeStudyTableSource(BaseStudy study, String source) { + private Cache sourcesCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(1, TimeUnit.MINUTES).build(); + + public synchronized StudyTableSource makeStudyTableSource(BaseStudy study, String source) { + String cacheKey = String.format("%s::%s", study.getId(), source); + try { + return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(study, source)); + } catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); + } + } + private StudyTableSource makeStudyTableSourceInternal(BaseStudy study, String source) { if (OpalTableSource.isFor(source)) { OpalTableSource tableSource = OpalTableSource.fromURN(source); tableSource.initialise(opalService, study.getOpal()); @@ -56,7 +73,7 @@ public StudyTableSource makeStudyTableSource(BaseStudy study, String source) { } if (ExcelTableSource.isFor(source)) { ExcelTableSource tableSource = ExcelTableSource.fromURN(source); - tableSource.initialise(getFileInputStream(tableSource.getPath())); + tableSource.initialise(getFileInputStream(study, tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() @@ -66,17 +83,28 @@ public StudyTableSource makeStudyTableSource(BaseStudy study, String source) { StudyTableSource tableSource = serviceOptional.get().makeSource(source); if (tableSource instanceof StudyTableFileSource) { StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; - fileSource.initialise(getFileInputStream(fileSource.getPath())); + fileSource.initialise(getFileInputStream(study, fileSource.getPath())); } return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } - private InputStream getFileInputStream(String path) { - Pair pathName = FileSystemService.extractPathName(path); - AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); - return fileStoreService.getFile(state.getAttachment().getFileReference()); + private InputStream getFileInputStream(BaseStudy study, String path) { + String fullPath = path; + if (!fullPath.startsWith("/")) { + // not a full path, then it is relative to the study's folder + fullPath = String.format("/%s-study/%s/%s", (study instanceof Study ? "individual" : "harmonization"), study.getId(), path); + } + log.info("Reading study table from file: {}", fullPath); + Pair pathName = FileSystemService.extractPathName(fullPath); + try { + AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); + return fileStoreService.getFile(state.getAttachment().getFileReference()); + } catch (Exception e) { + log.error("Cannot read study table file: {}", fullPath , e); + throw new NoSuchValueTableException("No value table at " + fullPath); + } } } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index b8ebcf181c..9b8b624c5a 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -37,7 +37,7 @@ public static boolean isFor(String source) { if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) return false; List tokens = Splitter.on(":").splitToList(source); - return tokens.size() == 4 && tokens.get(2).toLowerCase().endsWith(".xlsx"); + return tokens.size() > 2 && tokens.get(2).toLowerCase().endsWith(".xlsx"); } public static ExcelTableSource fromURN(String source) { @@ -46,8 +46,8 @@ public static ExcelTableSource fromURN(String source) { String fullName = source.replace("urn:file:", ""); int sep = fullName.lastIndexOf(":"); - String file = fullName.substring(0, sep); - String table = fullName.substring(sep + 1); + String file = sep > 0 ? fullName.substring(0, sep) : fullName; + String table = sep > 0 ? fullName.substring(sep + 1) : null; return ExcelTableSource.newSource(file, table); } @@ -70,7 +70,9 @@ public String getTable() { @Override public ValueTable getValueTable() { - return excelDatasource.getValueTable(table); + return Strings.isNullOrEmpty(table) ? + excelDatasource.getValueTables().stream().findFirst().get() : + excelDatasource.getValueTable(table); } @Override @@ -85,7 +87,7 @@ public Math.SummaryStatisticsDto getVariableSummary(String variableName) { @Override public String getURN() { - return String.format("urn:file:%s:%s", path, table); + return Strings.isNullOrEmpty(table) ? String.format("urn:file:%s", path) : String.format("urn:file:%s:%s", path, table); } @Override diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java index fcb9991c4b..08cbc329ef 100644 --- a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java @@ -19,6 +19,7 @@ public class ExcelTableSourceTest { @Test public void test_is_for() { assertThat(ExcelTableSource.isFor("urn:file:MyProject.xlsx:MyTable")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xlsx")).isTrue(); assertThat(ExcelTableSource.isFor("urn:file:/path/to/MyProject.xlsx:MyTable")).isTrue(); assertThat(ExcelTableSource.isFor("urn:file:MyProject.xls")).isFalse(); assertThat(ExcelTableSource.isFor("urn:file:MyProject.spss")).isFalse(); @@ -30,6 +31,10 @@ public void test_urn_parse() { assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); assertThat(source.getTable()).isEqualTo("MyTable"); + source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx"); + assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); + assertThat(source.getTable()).isNullOrEmpty(); + source = ExcelTableSource.fromURN("urn:file:/path/to/MyProject.xlsx:MyTable"); assertThat(source.getPath()).isEqualTo("/path/to/MyProject.xlsx"); assertThat(source.getTable()).isEqualTo("MyTable"); diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java index 8d06cb9fd6..527ebdca71 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java @@ -10,16 +10,25 @@ package org.obiba.mica.spi.source; +import java.io.InputStream; + /** * Study table is to be extracted from a file which path applies to the Mica's internal file system. */ -public interface StudyTableFileSource extends StudyTableStreamSource { +public interface StudyTableFileSource extends StudyTableSource { /** - * The path to the file in Mica's file system. + * The path to the file in Mica's file system, that can be absolute or relative the study's folder considered. * * @return */ String getPath(); + /** + * Initialise the source from the given input stream that represents the file content. + * + * @param in + */ + void initialise(InputStream in); + } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java deleted file mode 100644 index a78177e8a9..0000000000 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableStreamSource.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2022 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.spi.source; - -import java.io.InputStream; - -/** - * Study table is to be extracted from an {@link InputStream}. - */ -public interface StudyTableStreamSource extends StudyTableSource { - - /** - * Initialise the source from the given input stream. - * - * @param in - */ - void initialise(InputStream in); - -} diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index f0c57295eb..373db00456 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -955,7 +955,18 @@ }, "datasource": { "title": "Data Source", - "info": "Reference to the Opal table from which variables and data summaries will be extracted from." + "info": "Source of the data dictionary and summary statistics.", + "info-schema": "Source of the data dictionary.", + "opal": { + "title": "Opal Source", + "info": "Reference to the Opal table from which variables and data summaries will be extracted from.", + "info-schema": "Reference to the Opal table from which variables will be extracted from." + }, + "file": { + "title": "File Source", + "info": "Path to the file from which table's variables will be extracted from. The path may be relative to the associated study's folder. The table name is optional if there is only one described in the file.", + "info-schema": "Path to the file from which table's variables will be extracted from. The path may be relative to the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." + } }, "delete-dialog": { "title": "Delete Dataset", @@ -976,7 +987,11 @@ }, "study-table": { "title": "Study Table", - "info": "A study table defines which study data collection event it applies to and which Opal table the variables and data summaries are to be extracted from." + "info": "A study table defines which study data collection event it applies to and which Opal table the variables and data summaries are to be extracted from.", + "source": { + "opal": "Opal source", + "file": "File source" + } }, "harmonization-study-table": { "title": "Harmonization Initiative Table", @@ -1006,7 +1021,7 @@ "table-additional-information": "Additional Information", "data-schema": { "title": "Harmonization Protocol", - "info": "The variables of the harmonization protocol are the reference variables. This is a simple data dictionary: no data summaries will be extracted from the Opal table." + "info": "The variables of the harmonization protocol are the reference variables. This is a simple data dictionary: no data summary statistics will be extracted." }, "index": "Index", "add-study-table": "Add Table", diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index afa8787016..6a0ce7dd1e 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -955,7 +955,15 @@ }, "datasource": { "title": "Source de données", - "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites." + "info": "Source du dictionnaire de données et des statistiques sommaires.", + "opal": { + "title": "Source Opal", + "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites." + }, + "file": { + "title": "Fichier source", + "info": "Fichier duquel les variables de la table seront extraites. Le chemin du fichier peut-être relatif au dossier de l'étude associée. Le nom de la table est optionel si le fichier ne décrit qu'une table." + } }, "delete-dialog": { "title": "Supprimer l'ensemble de données", @@ -976,7 +984,11 @@ }, "study-table": { "title": "Table d'étude", - "info": "La table d'étude définie à quel événement de collecte elle s'applique et à partir de quelle table Opal il faut extraire les variables et les données sommaires." + "info": "La table d'étude définie à quel événement de collecte elle s'applique et à partir de quelle table Opal il faut extraire les variables et les données sommaires.", + "source": { + "opal": "Opal source", + "file": "Fichier source" + } }, "harmonization-study-table": { "title": "Table d'initiative d'harmonisation", @@ -1006,7 +1018,7 @@ "table-additional-information": "Information additionnelle", "data-schema": { "title": "Protocole d'harmonisation", - "info": "Les variables du protocole d'harmonisation sont les variables de référence. Il s'agit d'un simple dictionnaire de données: aucun résumé de données ne sera extrait de la table Opal." + "info": "Les variables du protocole d'harmonisation sont les variables de référence. Il s'agit d'un simple dictionnaire de données: aucune statistique sommaires ne seront extraites." }, "index": "Index", "add-study-table": "Ajouter une table d'étude", diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js index 410224a71e..5078e21188 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js @@ -317,20 +317,33 @@ mica.dataset return p.name === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project; }).pop(); - if ($scope.selected.project) { - $scope.selected.project.table = $scope.selected.project.datasource.table.filter(function (t) { - return t === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table; - }).pop(); + if (!$scope.selected.project) { + $scope.selected.project = $scope.projects.pop(); + } + + $scope.selected.project.table = $scope.selected.project.datasource.table.filter(function (t) { + return t === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table; + }).pop(); + + if (!$scope.selected.project.table) { + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); } }); }); } else { - getOpalProjects(); $scope.dataset = { published: false, - 'obiba.mica.HarmonizedDatasetDto.type': {}, + 'obiba.mica.HarmonizedDatasetDto.type': { + harmonizationTable: { + namespace: 'opal' + } + }, model: {} }; + getOpalProjects().then(function() { + $scope.selected.project = $scope.projects.pop(); + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); + }); } $scope.save = function () { @@ -343,8 +356,10 @@ mica.dataset } $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}; - $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project = $scope.selected.project.name; - $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table = $scope.selected.project.table; + if ($scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.namespace === 'opal') { + $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project = $scope.selected.project.name; + $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table = $scope.selected.project.table; + } $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.studyId = $scope.selected.study ? $scope.selected.study.id : null; if ($scope.dataset.id) { @@ -387,6 +402,11 @@ mica.dataset initializeForm(); FormDirtyStateObserver.observe($scope); + + $scope.NAMESPACES = [ + 'opal', + 'file' + ]; }]) .controller('DatasetViewController', ['$rootScope', @@ -861,10 +881,13 @@ mica.dataset $scope.type = tableType; $scope.selected.isHarmonizationTable = tableType === mica.dataset.OPAL_TABLE_TYPES.HARMONIZATION_TABLE; $scope.table = $.extend(true, {}, table); + if (!$scope.table.namespace) { + $scope.table.namespace = 'opal'; + } $scope.table.model = { name: LocalizedValues.arrayToObject(table.name), description: LocalizedValues.arrayToObject(table.description), - additionalInformation: LocalizedValues.arrayToObject(table.additionalInformation) + additionalInformation: LocalizedValues.arrayToObject(table.additionalInformation), }; MicaConfigResource.get(function (micaConfig) { @@ -941,6 +964,9 @@ mica.dataset if (selectedTable) { $scope.selected.project.table = selectedTable; } + } else { + $scope.selected.project = $scope.projects.pop(); + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); } }); @@ -956,13 +982,21 @@ mica.dataset !$scope.selected.study.population || !$scope.selected.study.population.dataCollectionEvent ? null : $scope.selected.study.population.dataCollectionEvent.id; - angular.extend($scope.table, { - studyId: $scope.selected.study.id, - populationId: populationId, - dataCollectionEventId: dceId, - project: $scope.selected.project.name, - table: $scope.selected.project.table - }); + if ($scope.table.namespace === 'file') { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId + }); + } else { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId, + project: $scope.selected.project.name, + table: $scope.selected.project.table + }); + } $scope.table.name = LocalizedValues.objectToArray($scope.table.model.name); $scope.table.description = LocalizedValues.objectToArray($scope.table.model.description); @@ -981,4 +1015,9 @@ mica.dataset $uibModalInstance.dismiss('cancel'); }; + $scope.NAMESPACES = [ + 'opal', + 'file' + ]; + }]); diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js index e795af4c36..4f4da3916d 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js @@ -372,6 +372,14 @@ mica.dataset serializeOpalTableForRestoringFields(datasetCopy); } + if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(serializeTableSource); + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].studyTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables || []).map(serializeTableSource); + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = serializeTableSource(dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}); + } else if (typeof dataset['obiba.mica.CollectedDatasetDto.type'] === 'object') { + datasetCopy['obiba.mica.CollectedDatasetDto.type'].studyTable = serializeTableSource(dataset['obiba.mica.CollectedDatasetDto.type'].studyTable || {}); + } + datasetCopy.content = datasetCopy.model ? angular.toJson(datasetCopy.model) : null; delete datasetCopy.model; // NOTICE: must be removed to avoid protobuf exception in dto. return angular.toJson(datasetCopy); @@ -394,6 +402,14 @@ mica.dataset deserializeOpalTableForRestoringFields(dataset); } + if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { + dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(deserializeTableSource); + dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables || []).map(deserializeTableSource); + dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = deserializeTableSource(dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}); + } else if (typeof dataset['obiba.mica.CollectedDatasetDto.type'] === 'object') { + dataset['obiba.mica.CollectedDatasetDto.type'].studyTable = deserializeTableSource(dataset['obiba.mica.CollectedDatasetDto.type'].studyTable || {}); + } + return dataset; } @@ -416,6 +432,22 @@ mica.dataset } } + function serializeTableSource(table) { + const result = JSON.parse(JSON.stringify(table)); + + if (result.namespace === 'file') { + result.source = 'urn:file:' + result.path + (result.table ? ':' + result.table : ''); + delete result.path; + } else { + result.source = 'urn:opal:' + result.project + '.' + result.table; + delete result.project; + } + delete result.table; + delete result.namespace; + + return result; + } + function deserializeOpalTableForRestoringFields(dataset) { if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { dataset.harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(deserializeTable); @@ -435,5 +467,27 @@ mica.dataset } } + function deserializeTableSource(table) { + const result = JSON.parse(JSON.stringify(table)); + + if (result.source) { + if (result.source.startsWith('urn:opal:')) { + const id = result.source.replace('urn:opal:', ''); + result.namespace = 'opal'; + result.project = id.substring(0, id.indexOf('.')); + result.table = id.substring(id.indexOf('.') + 1); + } else if (result.source.startsWith('urn:file:')) { + const id = result.source.replace('urn:file:', ''); + result.namespace = 'file'; + result.path = id.indexOf(':') > 0 ? id.substring(0, id.indexOf(':')) : id; + result.table = id.indexOf(':') > 0 ? id.substring(id.indexOf(':') + 1) : undefined; + } + } else { + result.namespace = 'opal'; + } + + return result; + } + return this; }]); diff --git a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html index 3f64ebbac5..d86f75fbcc 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html @@ -54,18 +54,6 @@

dataset.study-table.title

{{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.dataCollectionEventId}} - - dataset.project - - {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.project}} - - - - dataset.table - - {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} - - dataset.table-name @@ -101,6 +89,50 @@

dataset.study-table.title

+
+

dataset.datasource.opal.title

+

dataset.datasource.opal.info

+ + + + + + + + + + + + +
dataset.project + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.project}} +
dataset.table + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} +
+
+ +
+

dataset.datasource.file.title

+

dataset.datasource.file.info

+ + + + + + + + + + + + +
path + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.path}} +
dataset.table + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} +
+
+ diff --git a/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html b/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html index 65e927207c..5d9198eac1 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html @@ -57,9 +57,9 @@

dataset.study-dce.title

-

dataset.datasource.title

+

dataset.datasource.opal.title

-

dataset.datasource.info

+

dataset.datasource.opal.info

diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html index b7aef6bb21..fd8e28989c 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html @@ -39,34 +39,72 @@

dataset.data-schema.title

dataset.data-schema.info

-
- - - {{$select.selected.name}} - - - - -

dataset.project-dataschema-help

+

dataset.datasource.info-schema

+ + + +
+ +

dataset.datasource.opal.info-schema

+
+
+
+ + + {{$select.selected.name}} + + + + +

dataset.project-dataschema-help

+
+
+
+
+ + + {{$select.selected}} + + + + +

dataset.table-dataschema-help

+
+
+
+
-
- - - {{$select.selected}} - - - - -

dataset.table-dataschema-help

+
+

dataset.datasource.file.info-schema

+
+
+
+ + +
+
+
+
+ + +
+
+

dataset.harmonization-study.title

diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html index 9a6d7292da..28697ca7fa 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html @@ -21,7 +21,7 @@

dataset.harmonized-tables.title

study.label type - dataset.project + dataset.project / path dataset.table dataset.table-name dataset.table-description @@ -46,7 +46,7 @@

dataset.harmonized-tables.title

{{(wrapper.type === 'studyTable' ? 'search.study.individual' : 'search.study.harmonization') | translate}} - {{wrapper.table.project}} + {{wrapper.table.namespace === 'opal' ? wrapper.table.project : wrapper.table.path}} {{wrapper.table.table}} diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html index 994303fd0b..b890135de3 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html @@ -16,22 +16,50 @@

dataset.data-schema.title

dataset.data-schema.info

- - - - - - - - - - - -
dataset.project - {{datasetProject}} -
dataset.table - {{datasetTable}} -
+ + +
+

dataset.datasource.opal.title

+ + + + + + + + + + + + +
dataset.project + {{datasetProject}} +
dataset.table + {{datasetTable}} +
+
+ +
+

dataset.datasource.file.title

+ + + + + + + + + + + + +
path + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.path}} +
dataset.table + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table}} +
+
+

dataset.harmonization-study.title

dataset.harmonization-study.info

diff --git a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html index 679214b42f..31205b2615 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html @@ -26,38 +26,66 @@

dataset.datasource.title

-

dataset.datasource.info

-
-
-
- - - {{$select.selected.name}} - - - - + + +
+

dataset.datasource.opal.info

+
+
+
+ + + {{$select.selected.name}} + + + + +
+
+
+
+ + + {{$select.selected}} + + + + +
-
-
- - - {{$select.selected}} - - - - +
+ +
+

dataset.datasource.file.info

+
+
+
+ + +
+
+
+
+ + +
diff --git a/mica-webapp/src/main/webapp/assets/js/mica-variable.js b/mica-webapp/src/main/webapp/assets/js/mica-variable.js index c512911758..ac0ba49838 100644 --- a/mica-webapp/src/main/webapp/assets/js/mica-variable.js +++ b/mica-webapp/src/main/webapp/assets/js/mica-variable.js @@ -219,7 +219,7 @@ const makeHarmonizedVariablesTable = function() { dceName = dceName + ' -- ' + localizedString(dce.name); } harmonizedVariablesTableBody.append('' + - '' + harmonizedVariable.resolver.name + '' + + '' + harmonizedVariable.resolver.name + '' + '' + studyAnchor(baseStudyTable.studySummary) + '' + '' + getTableInformation(baseStudyTable, studyTableName) + '' + '' + dceName + '' + From af62535ee8c7d6ed41e41bdd99f808275c002bc8 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 5 Dec 2022 12:17:24 +0100 Subject: [PATCH 06/44] File path can be copied. Study table's file source can be relative to the dataset's folder --- .../StudyTableSourceServiceRegistry.java | 40 ++++++++++++++----- .../service/CollectedDatasetService.java | 6 +-- .../mica/dataset/service/DatasetService.java | 5 ++- .../service/HarmonizedDatasetService.java | 24 +++++------ .../DraftHarmonizedDatasetResource.java | 2 +- ...shedDataschemaDatasetVariableResource.java | 2 +- ...shedHarmonizedDatasetVariableResource.java | 2 +- mica-webapp/src/main/resources/i18n/en.json | 5 ++- mica-webapp/src/main/resources/i18n/fr.json | 8 +++- .../app/file-system/file-system-controller.js | 13 +++++- .../views/documents-table-template.html | 5 +++ .../views/file-system-template.html | 5 +++ 12 files changed, 81 insertions(+), 36 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 5ecf52d903..3db989cde7 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -16,6 +16,8 @@ import org.obiba.magma.NoSuchValueTableException; import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; +import org.obiba.mica.dataset.domain.Dataset; +import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.file.AttachmentState; import org.obiba.mica.file.FileStoreService; import org.obiba.mica.file.service.FileSystemService; @@ -57,15 +59,15 @@ public class StudyTableSourceServiceRegistry { private Cache sourcesCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(1, TimeUnit.MINUTES).build(); - public synchronized StudyTableSource makeStudyTableSource(BaseStudy study, String source) { + public synchronized StudyTableSource makeStudyTableSource(Dataset dataset, BaseStudy study, String source) { String cacheKey = String.format("%s::%s", study.getId(), source); try { - return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(study, source)); + return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(dataset, study, source)); } catch (ExecutionException e) { throw new RuntimeException(e.getCause()); } } - private StudyTableSource makeStudyTableSourceInternal(BaseStudy study, String source) { + private StudyTableSource makeStudyTableSourceInternal(Dataset dataset, BaseStudy study, String source) { if (OpalTableSource.isFor(source)) { OpalTableSource tableSource = OpalTableSource.fromURN(source); tableSource.initialise(opalService, study.getOpal()); @@ -73,7 +75,7 @@ private StudyTableSource makeStudyTableSourceInternal(BaseStudy study, String so } if (ExcelTableSource.isFor(source)) { ExcelTableSource tableSource = ExcelTableSource.fromURN(source); - tableSource.initialise(getFileInputStream(study, tableSource.getPath())); + tableSource.initialise(getFileInputStream(dataset, study, tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() @@ -83,27 +85,43 @@ private StudyTableSource makeStudyTableSourceInternal(BaseStudy study, String so StudyTableSource tableSource = serviceOptional.get().makeSource(source); if (tableSource instanceof StudyTableFileSource) { StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; - fileSource.initialise(getFileInputStream(study, fileSource.getPath())); + fileSource.initialise(getFileInputStream(dataset, study, fileSource.getPath())); } return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } - private InputStream getFileInputStream(BaseStudy study, String path) { + private InputStream getFileInputStream(Dataset dataset, BaseStudy study, String path) { String fullPath = path; + Optional attachmentState = Optional.empty(); if (!fullPath.startsWith("/")) { - // not a full path, then it is relative to the study's folder - fullPath = String.format("/%s-study/%s/%s", (study instanceof Study ? "individual" : "harmonization"), study.getId(), path); + // not a full path, then it may be relative to the dataset's folder + fullPath = String.format("/%s-dataset/%s/%s", (dataset instanceof StudyDataset ? "collected" : "harmonized"), dataset.getId(), path); + attachmentState = getAttachmentState(fullPath); + // not found, then try a path relative to the study's folder + if (!attachmentState.isPresent()) { + fullPath = String.format("/%s-study/%s/%s", (study instanceof Study ? "individual" : "harmonization"), study.getId(), path); + attachmentState = getAttachmentState(fullPath); + } + } else { + attachmentState = getAttachmentState(fullPath); + } + if (attachmentState.isPresent()) { + return fileStoreService.getFile(attachmentState.get().getAttachment().getFileReference()); + } else { + throw new NoSuchValueTableException("No value table at " + fullPath); } + } + + private Optional getAttachmentState(String fullPath) { log.info("Reading study table from file: {}", fullPath); Pair pathName = FileSystemService.extractPathName(fullPath); try { AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); - return fileStoreService.getFile(state.getAttachment().getFileReference()); + return Optional.of(state); } catch (Exception e) { - log.error("Cannot read study table file: {}", fullPath , e); - throw new NoSuchValueTableException("No value table at " + fullPath); + return Optional.empty(); } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 28b01f76b2..8f9e4d6d70 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -409,7 +409,7 @@ public List processVariablesForStudyDataset(StudyDataset datase @Override protected ValueTable getValueTable(@NotNull StudyDataset dataset) throws NoSuchValueTableException { - return getStudyTableSource(dataset.getSafeStudyTable()).getValueTable(); + return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getValueTable(); } @Override @@ -431,12 +431,12 @@ public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableN public SummaryStatisticsWrapper getVariableSummary(@NotNull StudyDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { log.info("Caching variable summary {} {}", dataset.getId(), variableName); - return new SummaryStatisticsWrapper(getStudyTableSource(dataset.getSafeStudyTable()).getVariableSummary(variableName)); + return new SummaryStatisticsWrapper(getStudyTableSource(dataset, dataset.getSafeStudyTable()).getVariableSummary(variableName)); } public Search.QueryResultDto getFacets(@NotNull StudyDataset dataset, Search.QueryTermsDto query) throws NoSuchValueTableException, NoSuchVariableException { - return getStudyTableSource(dataset.getSafeStudyTable()).getFacets(query); + return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getFacets(query); } public Search.QueryResultDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 4b2c836015..1254da1f2c 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -26,6 +26,7 @@ import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.service.StudyService; import org.obiba.opal.web.model.Math; import org.slf4j.Logger; @@ -137,8 +138,8 @@ protected Iterable getVariables(@NotNull T dataset) return getValueTable(dataset).getVariables(); } - protected StudyTableSource getStudyTableSource(@NotNull BaseStudyTable studyTable) { - return studyTableSourceServiceRegistry.makeStudyTableSource(getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSource()); + protected StudyTableSource getStudyTableSource(@NotNull T dataset, @NotNull BaseStudyTable studyTable) { + return studyTableSourceServiceRegistry.makeStudyTableSource(dataset, getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSource()); } protected Iterable wrappedGetDatasetVariables(T dataset) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index 7a3ea9d3df..a014564e29 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -345,7 +345,7 @@ public void delete(String id) { @Override protected ValueTable getValueTable(@NotNull HarmonizationDataset dataset) throws NoSuchValueTableException { - return getStudyTableSource(dataset.getSafeHarmonizationTable()).getValueTable(); + return getStudyTableSource(dataset, dataset.getSafeHarmonizationTable()).getValueTable(); } @Override @@ -357,18 +357,18 @@ public Iterable getDatasetVariables(HarmonizationDataset datase @Override public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getStudyTableSource(dataset.getSafeHarmonizationTable()).getValueTable().getVariable(variableName)); + return new DatasetVariable(dataset, getStudyTableSource(dataset, dataset.getSafeHarmonizationTable()).getValueTable().getVariable(variableName)); } public Iterable getDatasetVariables(HarmonizationDataset dataset, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException { - return StreamSupport.stream(getVariables(studyTable).spliterator(), false) + return StreamSupport.stream(getVariables(dataset, studyTable).spliterator(), false) .map(input -> new DatasetVariable(dataset, input, studyTable)).collect(toList()); } public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getStudyTableSource(studyTable).getValueTable().getVariable(variableName)); + return new DatasetVariable(dataset, getStudyTableSource(dataset, studyTable).getValueTable().getVariable(variableName)); } public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, String studyId, @@ -384,21 +384,21 @@ public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, source); for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { if(baseTable.isFor(studyId, source)) { - return new SummaryStatisticsWrapper(getStudyTableSource(baseTable).getVariableSummary(variableName)); + return new SummaryStatisticsWrapper(getStudyTableSource(dataset, baseTable).getVariableSummary(variableName)); } } throw NoSuchStudyException.withId(studyId); } - public Search.QueryResultDto getFacets(Search.QueryTermsDto query, BaseStudyTable studyTable) + public Search.QueryResultDto getFacets(@NotNull HarmonizationDataset dataset, Search.QueryTermsDto query, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException { - return getStudyTableSource(studyTable).getFacets(query); + return getStudyTableSource(dataset, studyTable).getFacets(query); } - public Search.QueryResultDto getContingencyTable(@NotNull BaseStudyTable studyTable, DatasetVariable variable, + public Search.QueryResultDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { - return getFacets(QueryTermsUtil.getContingencyQuery(variable, crossVariable), studyTable); + return getFacets(dataset, QueryTermsUtil.getContingencyQuery(variable, crossVariable), studyTable); } @Override @@ -468,9 +468,9 @@ protected HarmonizationDataset prepareSave(HarmonizationDataset dataset) { } } - private Iterable getVariables(BaseStudyTable studyTable) + private Iterable getVariables(@NotNull HarmonizationDataset dataset, BaseStudyTable studyTable) throws NoSuchDatasetException, NoSuchStudyException, NoSuchValueTableException { - return getStudyTableSource(studyTable).getValueTable().getVariables(); + return getStudyTableSource(dataset, studyTable).getValueTable().getVariables(); } private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String source) @@ -478,7 +478,7 @@ private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyI for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { if(baseTable.isFor(studyId, source)) { - return getStudyTableSource(baseTable).getValueTable(); + return getStudyTableSource(dataset, baseTable).getValueTable(); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java index 72bb06857c..84c4e1d1f3 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java @@ -172,7 +172,7 @@ public List getFacets(Search.QueryTermsDto query) { ImmutableList.Builder builder = ImmutableList.builder(); HarmonizationDataset dataset = getDataset(); for (BaseStudyTable table : dataset.getBaseStudyTables()) { - builder.add(datasetService.getFacets(query, table)); + builder.add(datasetService.getFacets(dataset, query, table)); } return builder.build(); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index 4bb7565a9b..9b669864a0 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -280,7 +280,7 @@ protected Future getVariableFacet(HarmonizationDatase protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, DatasetVariable crossVar, BaseStudyTable studyTable) { try { - return new AsyncResult<>(datasetService.getContingencyTable(studyTable, var, crossVar)); + return new AsyncResult<>(datasetService.getContingencyTable(dataset, studyTable, var, crossVar)); } catch (Exception e) { log.warn("Unable to retrieve contingency statistics: " + e.getMessage(), e); return new AsyncResult<>(null); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index 4ccddb1ebc..4f23689209 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -112,7 +112,7 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var if (baseTable.isFor(studyId, source)) { try { return dtos.asContingencyDto(baseTable, var, crossVar, - datasetService.getContingencyTable(baseTable, var, crossVar)).build(); + datasetService.getContingencyTable(dataset, baseTable, var, crossVar)).build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); return dtos.asContingencyDto(baseTable, var, crossVar, null).build(); diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 373db00456..7afc7cfd87 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -59,6 +59,7 @@ "update-comment": "Add a comment to describe the changes.", "copied": "Copied", "copy-to-clipboard": "Copy to clipboard", + "copy-path-to-clipboard": "Copy file path to clipboard", "copy-query": "Copy query", "name": "Name", "default": "Default", @@ -964,8 +965,8 @@ }, "file": { "title": "File Source", - "info": "Path to the file from which table's variables will be extracted from. The path may be relative to the associated study's folder. The table name is optional if there is only one described in the file.", - "info-schema": "Path to the file from which table's variables will be extracted from. The path may be relative to the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." + "info": "Path to the file from which table's variables will be extracted from. If the path is relative, it is first looked up in the dataset's folder, then in the associated study's folder. The table name is optional if there is only one described in the file.", + "info-schema": "Path to the file from which table's variables will be extracted from. If the path is relative, it is first looked up in the harmonization protocol's folder, then in the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." } }, "delete-dialog": { diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index 6a0ce7dd1e..16d1972c3e 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -59,6 +59,7 @@ "update-comment": "Ajoutez un commentaire pour décrire les modifications.", "copied": "Copié", "copy-to-clipboard": "Copier dans le presse-papier", + "copy-path-to-clipboard": "Copier le chemin du fichier dans le presse-papier", "copy-query": "Copier la requête", "name": "Nom", "default": "Par défaut", @@ -956,13 +957,16 @@ "datasource": { "title": "Source de données", "info": "Source du dictionnaire de données et des statistiques sommaires.", + "info-schema": "Source du dictionnaire de données.", "opal": { "title": "Source Opal", - "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites." + "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites.", + "info-schema": "Référence vers la table Opal de laquelle les variables seront extraites." }, "file": { "title": "Fichier source", - "info": "Fichier duquel les variables de la table seront extraites. Le chemin du fichier peut-être relatif au dossier de l'étude associée. Le nom de la table est optionel si le fichier ne décrit qu'une table." + "info": "Fichier duquel les variables de la table seront extraites. Si le chemin est relatif, le fichier est d'abord cherché dans le dossier de l'ensemble de données, puis dans celui de l'étude associée. Le nom de la table est optionel si le fichier ne décrit qu'une table.", + "info-schema": "Fichier duquel les variables de la table seront extraites. Le chemin du fichier peut-être relatif, le fichier est d'abord cherché dans le dossier du protocole d'harmonisation, puis dans celui de l'initiative d'harmonisation associée. Le nom de la table est optionel si le fichier ne décrit qu'une table." } }, "delete-dialog": { diff --git a/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js b/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js index 1a431874c3..fd741c323d 100644 --- a/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js +++ b/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js @@ -29,6 +29,7 @@ mica.fileSystem 'DraftFileSystemSearchResource', 'MicaConfigResource', '$q', + '$timeout', function ($rootScope, $scope, @@ -45,7 +46,8 @@ mica.fileSystem DraftFileAccessResource, DraftFileSystemSearchResource, MicaConfigResource, - $q) { + $q, + $timeout) { function buildClipboardCommand(command, origin, items) { return { @@ -499,6 +501,15 @@ mica.fileSystem }); }; + $scope.showCopiedTooltipStatus = false; + + $scope.showCopiedTooltip = function () { + $scope.showCopiedTooltipStatus = true; + $timeout(function () { + $scope.showCopiedTooltipStatus = false; + }, 1000); + }; + $scope.loadPermissions = function (document) { DraftFilePermissionResource.query({path: document.path}, function onSuccess(response) { $scope.data.permissions = response; diff --git a/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html b/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html index d9b57931b3..c05cb4a789 100644 --- a/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html +++ b/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html @@ -77,6 +77,11 @@
    • +
  • + + + +
    • diff --git a/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html b/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html index d5fe8cbb1c..f3fd93a9ca 100644 --- a/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html +++ b/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html @@ -21,6 +21,11 @@ + + + + +
  • From 7e2bf5e65ebf252d9939d745f3a01cb28ff96ce6 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 5 Dec 2022 16:57:53 +0100 Subject: [PATCH 07/44] Study table context added. Opal dtos (aggregations) are not exposed in the source plugin interface. --- .../StudyTableSourceServiceRegistry.java | 36 ++- .../mica/core/source/ExcelTableSource.java | 13 +- .../mica/core/source/OpalTableSource.java | 24 +- .../mica/core/source/support/OpalDtos.java | 280 ++++++++++++++++++ .../source}/support/QueryTermsUtil.java | 28 +- .../obiba/mica/dataset/domain/Dataset.java | 3 +- .../mica/dataset/domain/DatasetCategory.java | 5 +- .../mica/dataset/domain/DatasetVariable.java | 13 +- .../service/CollectedDatasetService.java | 19 +- .../mica/dataset/service/DatasetService.java | 76 ----- .../service/HarmonizedDatasetService.java | 24 +- .../dataset/service/VariableSetService.java | 15 +- .../obiba/mica/study/domain/BaseStudy.java | 4 +- .../org/obiba/mica/web/model/DatasetDtos.java | 267 ++--------------- .../java/org/obiba/mica/web/model/Dtos.java | 12 +- .../rest/PublishedDatasetResource.java | 2 +- .../DraftCollectedDatasetResource.java | 10 - ...DraftCollectedDatasetVariableResource.java | 15 +- ...raftDataschemaDatasetVariableResource.java | 18 -- .../DraftHarmonizedDatasetResource.java | 16 - ...raftHarmonizedDatasetVariableResource.java | 13 +- ...ishedCollectedDatasetVariableResource.java | 17 +- ...shedDataschemaDatasetVariableResource.java | 48 +-- ...shedHarmonizedDatasetVariableResource.java | 18 +- .../spi/source/AbstractStudyTableSource.java | 29 ++ .../org/obiba/mica/spi/source/ICategory.java | 32 ++ .../org/obiba/mica/spi/source/IDataset.java | 29 ++ .../org/obiba/mica/spi/source/IStudy.java | 35 +++ .../org/obiba/mica/spi/source/IVariable.java | 55 ++++ .../mica/spi/source/StudyTableContext.java | 38 +++ .../mica/spi/source/StudyTableSource.java | 14 +- 31 files changed, 653 insertions(+), 555 deletions(-) create mode 100644 mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java rename mica-core/src/main/java/org/obiba/mica/{dataset/service => core/source}/support/QueryTermsUtil.java (77%) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 3db989cde7..dc816651ea 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -16,17 +16,14 @@ import org.obiba.magma.NoSuchValueTableException; import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; -import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.file.AttachmentState; import org.obiba.mica.file.FileStoreService; import org.obiba.mica.file.service.FileSystemService; +import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.micaConfig.service.PluginsService; -import org.obiba.mica.spi.source.StudyTableFileSource; -import org.obiba.mica.spi.source.StudyTableSource; -import org.obiba.mica.spi.source.StudyTableSourceService; -import org.obiba.mica.study.domain.BaseStudy; +import org.obiba.mica.spi.source.*; import org.obiba.mica.study.domain.Study; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +31,6 @@ import javax.inject.Inject; import java.io.InputStream; -import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -45,6 +41,9 @@ public class StudyTableSourceServiceRegistry { private static final Logger log = LoggerFactory.getLogger(StudyTableSourceServiceRegistry.class); + @Inject + private MicaConfigService micaConfigService; + @Inject private PluginsService pluginsService; @@ -59,23 +58,27 @@ public class StudyTableSourceServiceRegistry { private Cache sourcesCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(1, TimeUnit.MINUTES).build(); - public synchronized StudyTableSource makeStudyTableSource(Dataset dataset, BaseStudy study, String source) { + public synchronized StudyTableSource makeStudyTableSource(IDataset dataset, IStudy study, String source) { + StudyTableContext context = new StudyTableContext(dataset, study, micaConfigService.getConfig().getPrivacyThreshold()); + String cacheKey = String.format("%s::%s", study.getId(), source); try { - return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(dataset, study, source)); + return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(context, source)); } catch (ExecutionException e) { throw new RuntimeException(e.getCause()); } } - private StudyTableSource makeStudyTableSourceInternal(Dataset dataset, BaseStudy study, String source) { + private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, String source) { if (OpalTableSource.isFor(source)) { OpalTableSource tableSource = OpalTableSource.fromURN(source); - tableSource.initialise(opalService, study.getOpal()); + tableSource.setStudyTableContext(context); + tableSource.initialise(opalService); return tableSource; } if (ExcelTableSource.isFor(source)) { ExcelTableSource tableSource = ExcelTableSource.fromURN(source); - tableSource.initialise(getFileInputStream(dataset, study, tableSource.getPath())); + tableSource.setStudyTableContext(context); + tableSource.initialise(getFileInputStream(context, tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() @@ -83,25 +86,26 @@ private StudyTableSource makeStudyTableSourceInternal(Dataset dataset, BaseStudy if (serviceOptional.isPresent()) { // TODO add a context to the study table source StudyTableSource tableSource = serviceOptional.get().makeSource(source); + tableSource.setStudyTableContext(context); if (tableSource instanceof StudyTableFileSource) { StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; - fileSource.initialise(getFileInputStream(dataset, study, fileSource.getPath())); + fileSource.initialise(getFileInputStream(context, fileSource.getPath())); } return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } - private InputStream getFileInputStream(Dataset dataset, BaseStudy study, String path) { + private InputStream getFileInputStream(StudyTableContext context, String path) { String fullPath = path; - Optional attachmentState = Optional.empty(); + Optional attachmentState; if (!fullPath.startsWith("/")) { // not a full path, then it may be relative to the dataset's folder - fullPath = String.format("/%s-dataset/%s/%s", (dataset instanceof StudyDataset ? "collected" : "harmonized"), dataset.getId(), path); + fullPath = String.format("/%s-dataset/%s/%s", (context.getDataset() instanceof StudyDataset ? "collected" : "harmonized"), context.getDataset().getId(), path); attachmentState = getAttachmentState(fullPath); // not found, then try a path relative to the study's folder if (!attachmentState.isPresent()) { - fullPath = String.format("/%s-study/%s/%s", (study instanceof Study ? "individual" : "harmonization"), study.getId(), path); + fullPath = String.format("/%s-study/%s/%s", (context.getStudy() instanceof Study ? "individual" : "harmonization"), context.getStudy().getId(), path); attachmentState = getAttachmentState(fullPath); } } else { diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 9b8b624c5a..01796ef89a 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -15,15 +15,16 @@ import org.obiba.magma.ValueTable; import org.obiba.magma.datasource.excel.ExcelDatasource; import org.obiba.magma.support.Initialisables; +import org.obiba.mica.spi.source.AbstractStudyTableSource; +import org.obiba.mica.spi.source.IVariable; import org.obiba.mica.spi.source.StudyTableFileSource; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; import javax.validation.constraints.NotNull; import java.io.InputStream; import java.util.List; -public class ExcelTableSource implements StudyTableFileSource { +public class ExcelTableSource extends AbstractStudyTableSource implements StudyTableFileSource { @NotNull private String path; @@ -76,12 +77,12 @@ public ValueTable getValueTable() { } @Override - public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { - throw new UnsupportedOperationException("Facet search not available from an Excel file"); + public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { + throw new UnsupportedOperationException("Contingency search not available from an Excel file"); } @Override - public Math.SummaryStatisticsDto getVariableSummary(String variableName) { + public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { throw new UnsupportedOperationException("Summary statistics not available from an Excel file"); } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index e5b497a659..e2a64ca932 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -12,8 +12,12 @@ import com.google.common.base.Strings; import org.obiba.magma.ValueTable; +import org.obiba.mica.core.source.support.OpalDtos; +import org.obiba.mica.core.source.support.QueryTermsUtil; import org.obiba.mica.micaConfig.service.OpalService; -import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.spi.source.AbstractStudyTableSource; +import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.web.model.Mica; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestValueTable; import org.obiba.opal.web.model.Math; @@ -24,7 +28,7 @@ /** * Connector to an Opal server, to retrieve value table and summary statistics. */ -public class OpalTableSource implements StudyTableSource { +public class OpalTableSource extends AbstractStudyTableSource { private OpalService opalService; @@ -59,13 +63,17 @@ public ValueTable getValueTable() { } @Override - public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { - return getRestValueTable().getFacets(query); + public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { + Search.QueryTermsDto query = QueryTermsUtil.getContingencyQuery(variable, crossVariable); + Search.QueryResultDto results = getRestValueTable().getFacets(query); + return OpalDtos.asDto(variable, crossVariable, getContext().getPrivacyThreshold(), results); } @Override - public Math.SummaryStatisticsDto getVariableSummary(String variableName) { - return ((RestValueTable.RestVariableValueSource)getRestValueTable().getVariableValueSource(variableName)).getSummary(); + public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { + RestValueTable.RestVariableValueSource variableValueSource = (RestValueTable.RestVariableValueSource) getRestValueTable().getVariableValueSource(variableName); + Math.SummaryStatisticsDto results = variableValueSource.getSummary(); + return OpalDtos.asDto(results); } @Override @@ -99,9 +107,9 @@ public static String toTableName(String source) { return source.replace("urn:opal:", ""); } - public void initialise(OpalService opalService, String opalUrl) { + public void initialise(OpalService opalService) { this.opalService = opalService; - this.opalUrl = opalUrl; + this.opalUrl = getContext().getStudy().getOpal(); } private RestDatasource getDatasource() { diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java new file mode 100644 index 0000000000..d2694ed8e2 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source.support; + +import com.google.common.collect.Lists; +import org.obiba.magma.type.BooleanType; +import org.obiba.mica.spi.source.ICategory; +import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.web.model.Mica; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; + +public class OpalDtos { + + // + // Search.QueryResultDto + // + + public static Mica.DatasetVariableContingencyDto asDto(IVariable variable, IVariable crossVariable, int privacyThreshold, Search.QueryResultDto results) { + Mica.DatasetVariableContingencyDto.Builder crossDto = Mica.DatasetVariableContingencyDto.newBuilder(); + Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + + if (results == null) { + allAggBuilder.setN(0); + allAggBuilder.setTotal(0); + crossDto.setAll(allAggBuilder); + return crossDto.build(); + } + + allAggBuilder.setTotal(results.getTotalHits()); + crossDto.setPrivacyThreshold(privacyThreshold); + boolean privacyChecks = !crossVariable.hasCategories() || validatePrivacyThreshold(results, privacyThreshold); + boolean totalPrivacyChecks = validateTotalPrivacyThreshold(results, privacyThreshold); + + // add facet results in the same order as the variable categories + List catNames = variable.getValueType().equals(BooleanType.get().getName()) ? + Lists.newArrayList("true", "false") : variable.getCategoryNames(); + catNames.forEach(catName -> results.getFacetsList().stream() + .filter(facet -> facet.hasFacet() && catName.equals(facet.getFacet())).forEach(facet -> { + boolean privacyCheck = privacyChecks && checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold); + Mica.DatasetVariableAggregationDto.Builder aggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + aggBuilder.setTotal(totalPrivacyChecks ? results.getTotalHits() : 0); + aggBuilder.setTerm(facet.getFacet()); + ICategory category = variable.getCategory(facet.getFacet()); + aggBuilder.setMissing(category != null && category.isMissing()); + addSummaryStatistics(crossVariable, aggBuilder, facet, privacyCheck, totalPrivacyChecks); + crossDto.addAggregations(aggBuilder); + })); + + // add total facet for all variable categories + results.getFacetsList().stream().filter(facet -> facet.hasFacet() && "_total".equals(facet.getFacet())) + .forEach(facet -> { + boolean privacyCheck = privacyChecks && facet.getFilters(0).getCount() >= privacyThreshold; + addSummaryStatistics(crossVariable, allAggBuilder, facet, privacyCheck, totalPrivacyChecks); + }); + + crossDto.setAll(allAggBuilder); + + return crossDto.build(); + + } + + private static boolean checkPrivacyThreshold(int count, int threshold) { + return count == 0 || count >= threshold; + } + + private static boolean validateTotalPrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { + return results.getFacetsList().stream() + .allMatch(facet -> checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold)); + } + + private static boolean validatePrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { + return results.getFacetsList().stream().map(Search.FacetResultDto::getFrequenciesList).flatMap(Collection::stream) + .allMatch(freq -> checkPrivacyThreshold(freq.getCount(), privacyThreshold)); + } + + private static void addSummaryStatistics(IVariable crossVariable, + Mica.DatasetVariableAggregationDto.Builder aggBuilder, Search.FacetResultDto facet, boolean privacyCheck, + boolean totalPrivacyCheck) { + aggBuilder.setN(totalPrivacyCheck ? facet.getFilters(0).getCount() : -1); + if (!privacyCheck) return; + + List catNames = crossVariable.getValueType().equals(BooleanType.get().getName()) ? + Lists.newArrayList("1", "0") : crossVariable.getCategoryNames(); + // order results as the order of cross variable categories + catNames.forEach(catName -> facet.getFrequenciesList().stream().filter(freq -> catName.equals(freq.getTerm())) + .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq)))); + // observed terms, not described by categories + facet.getFrequenciesList().stream().filter(freq -> !catNames.contains(freq.getTerm())) + .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq))); + + if (facet.hasStatistics()) { + aggBuilder.setStatistics(asDto(facet.getStatistics())); + } + } + + private static Mica.FrequencyDto.Builder asDto(IVariable crossVariable, + Search.FacetResultDto.TermFrequencyResultDto result) { + if (crossVariable.getValueType().equals(BooleanType.get().getName())) { + // for some reason 0/1 is returned instead of false/true + return Mica.FrequencyDto.newBuilder() + .setValue("1".equals(result.getTerm()) ? "true" : "false") + .setCount(result.getCount()) + .setMissing(false); + } else if (crossVariable.getCategory(result.getTerm()) != null) { + ICategory category = crossVariable.getCategory(result.getTerm()); + return Mica.FrequencyDto.newBuilder() + .setValue(result.getTerm()) + .setCount(result.getCount()) + .setMissing(category != null && category.isMissing()); + } else { + // observed value, not described by a category + return Mica.FrequencyDto.newBuilder() + .setValue(result.getTerm()) + .setCount(result.getCount()) + .setMissing(false); + } + } + + private static Mica.StatisticsDto.Builder asDto(Search.FacetResultDto.StatisticalResultDto result) { + return Mica.StatisticsDto.newBuilder() // + .setMin(result.getMin()) // + .setMax(result.getMax()) // + .setMean(result.getMean()) // + .setSum(result.getTotal()) // + .setSumOfSquares(result.getSumOfSquares()) // + .setVariance(result.getVariance()) // + .setStdDeviation(result.getStdDeviation()); + } + + // + // SummaryStatisticsDto methods + // + + public static Mica.DatasetVariableAggregationDto asDto(@Nullable Math.SummaryStatisticsDto summary) { + + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + + if (summary == null) return aggDto.setTotal(0).setN(0).build(); + + if (summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { + aggDto = asDto(summary.getExtension(Math.CategoricalSummaryDto.categorical)); + } else if (summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { + aggDto = asDto(summary.getExtension(Math.ContinuousSummaryDto.continuous)); + } else if (summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { + aggDto = asDto(summary.getExtension(Math.DefaultSummaryDto.defaultSummary)); + } else if (summary.hasExtension(Math.TextSummaryDto.textSummary)) { + aggDto = asDto(summary.getExtension(Math.TextSummaryDto.textSummary)); + } else if (summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { + aggDto = asDto(summary.getExtension(Math.GeoSummaryDto.geoSummary)); + } else if (summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { + aggDto = asDto(summary.getExtension(Math.BinarySummaryDto.binarySummary)); + } + + return aggDto.build(); + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.CategoricalSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList(), + summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.DefaultSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.TextSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList(), + summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.GeoSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.BinarySummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.FrequencyDto.Builder asDto(Math.FrequencyDto freq) { + return Mica.FrequencyDto.newBuilder().setValue(freq.getValue()).setCount(Long.valueOf(freq.getFreq()).intValue()) + .setMissing(freq.getMissing()); + } + + private static Mica.IntervalFrequencyDto.Builder asDto(Math.IntervalFrequencyDto inter) { + return Mica.IntervalFrequencyDto.newBuilder().setCount((int) inter.getFreq()) + .setLower(inter.getLower()).setUpper(inter.getUpper()); + } + + private static void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, + List frequencies) { + addFrequenciesDto(aggDto, frequencies, 0); + } + + private static void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, List frequencies, + int otherFrequency) { + int n = otherFrequency; + if (frequencies != null) { + for (Math.FrequencyDto freq : frequencies) { + aggDto.addFrequencies(asDto(freq)); + if (!freq.getMissing()) n += freq.getFreq(); + } + } + if (otherFrequency > 0) + aggDto.addFrequencies(Mica.FrequencyDto.newBuilder().setValue("???").setCount(otherFrequency) + .setMissing(false)); + aggDto.setN(n); + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.ContinuousSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + Math.DescriptiveStatsDto stats = summary.getSummary(); + + aggDto.setN(Long.valueOf(stats.getN()).intValue()); + + Mica.StatisticsDto.Builder builder = Mica.StatisticsDto.newBuilder(); + + if (stats.hasSum()) builder.setSum(Double.valueOf(stats.getSum()).floatValue()); + if (stats.hasMin() && stats.getMin() != Double.POSITIVE_INFINITY) + builder.setMin(Double.valueOf(stats.getMin()).floatValue()); + if (stats.hasMax() && stats.getMax() != Double.NEGATIVE_INFINITY) + builder.setMax(Double.valueOf(stats.getMax()).floatValue()); + if (stats.hasMean() && !Double.isNaN(stats.getMean())) + builder.setMean(Double.valueOf(stats.getMean()).floatValue()); + if (stats.hasSumsq() && !Double.isNaN(stats.getSumsq())) + builder.setSumOfSquares(Double.valueOf(stats.getSumsq()).floatValue()); + if (stats.hasVariance() && !Double.isNaN(stats.getVariance())) + builder.setVariance(Double.valueOf(stats.getVariance()).floatValue()); + if (stats.hasStdDev() && !Double.isNaN(stats.getStdDev())) + builder.setStdDeviation(Double.valueOf(stats.getStdDev()).floatValue()); + + aggDto.setStatistics(builder); + + if (summary.getFrequenciesCount() > 0) { + summary.getFrequenciesList().forEach(freq -> aggDto.addFrequencies(asDto(freq))); + } + + if (summary.getIntervalFrequencyCount() > 0) { + summary.getIntervalFrequencyList().forEach(inter -> aggDto.addIntervalFrequencies(asDto(inter))); + } + + int total = 0; + if (summary.getFrequenciesCount() > 0) { + for (Math.FrequencyDto freq : summary.getFrequenciesList()) { + total += freq.getFreq(); + } + } + aggDto.setTotal(total); + + return aggDto; + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java similarity index 77% rename from mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java rename to mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java index d9fc80248f..925d700471 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 OBiBa. All rights reserved. + * Copyright (c) 2022 OBiBa. All rights reserved. * * This program and the accompanying materials * are made available under the terms of the GNU Public License v3.0. @@ -8,23 +8,23 @@ * along with this program. If not, see . */ -package org.obiba.mica.dataset.service.support; - -import java.util.List; +package org.obiba.mica.core.source.support; +import com.google.common.collect.Lists; import org.obiba.magma.type.BooleanType; -import org.obiba.mica.dataset.domain.DatasetVariable; +import org.obiba.mica.spi.source.IVariable; import org.obiba.opal.web.model.Search; -import com.google.common.collect.Lists; +import java.util.List; public class QueryTermsUtil { public static final String TOTAL_FACET = "_total"; - private QueryTermsUtil() {} + private QueryTermsUtil() { + } - public static Search.QueryTermsDto getContingencyQuery(DatasetVariable variable, DatasetVariable crossVariable) { + public static Search.QueryTermsDto getContingencyQuery(IVariable variable, IVariable crossVariable) { Search.QueryTermsDto.Builder queries = Search.QueryTermsDto.newBuilder(); // for each category, make a facet query @@ -40,7 +40,7 @@ public static Search.QueryTermsDto getContingencyQuery(DatasetVariable variable, // Private methods // - private static Search.QueryTermDto getQueryTerm(DatasetVariable variable, DatasetVariable crossVariable, String facetName) { + private static Search.QueryTermDto getQueryTerm(IVariable variable, IVariable crossVariable, String facetName) { Search.QueryTermDto.Builder query = Search.QueryTermDto.newBuilder(); query.setFacet(facetName); @@ -51,7 +51,7 @@ private static Search.QueryTermDto getQueryTerm(DatasetVariable variable, Datase return query.build(); } - private static Search.QueryTermDto getTotalTerm(DatasetVariable variable, DatasetVariable crossVariable) { + private static Search.QueryTermDto getTotalTerm(IVariable variable, IVariable crossVariable) { Search.QueryTermDto.Builder query = Search.QueryTermDto.newBuilder(); query.setFacet(TOTAL_FACET); @@ -80,13 +80,13 @@ private static Search.LogicalTermDto getLogicalTermDto(String variableName, List return term.build(); } - private static List getCategories(DatasetVariable variable) { + private static List getCategories(IVariable variable) { List categories = Lists.newArrayList(); - if(variable.getValueType().equals(BooleanType.get().getName())) { + if (variable.getValueType().equals(BooleanType.get().getName())) { categories.add("true"); categories.add("false"); - } else if(variable.hasCategories()) { - variable.getCategories().forEach(c -> categories.add(c.getName())); + } else if (variable.hasCategories()) { + categories = variable.getCategoryNames(); } return categories; } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java index 6e32ab5fb1..b774a5fdd9 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java @@ -24,13 +24,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import org.obiba.mica.spi.source.IDataset; import java.util.Map; /** * Proxy to value tables. */ -public abstract class Dataset extends AbstractModelAware implements AttributeAware, Indexable { +public abstract class Dataset extends AbstractModelAware implements AttributeAware, Indexable, IDataset { private static final long serialVersionUID = -3328963766855899217L; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java index b138f16f4c..e3b6dc7f31 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java @@ -16,8 +16,9 @@ import org.obiba.mica.core.domain.Attribute; import org.obiba.mica.core.domain.AttributeAware; import org.obiba.mica.core.domain.Attributes; +import org.obiba.mica.spi.source.ICategory; -public class DatasetCategory implements AttributeAware { +public class DatasetCategory implements AttributeAware, ICategory { private String name; @@ -37,10 +38,12 @@ public DatasetCategory(Category category) { } } + @Override public String getName() { return name; } + @Override public boolean isMissing() { return missing; } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index bcd1ed33c2..59d8817e16 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -13,10 +13,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import com.google.common.collect.Sets; +import org.apache.commons.compress.utils.Lists; import org.obiba.magma.Variable; import org.obiba.magma.support.VariableNature; import org.obiba.mica.core.domain.*; import org.obiba.mica.spi.search.Indexable; +import org.obiba.mica.spi.source.IVariable; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -29,7 +31,7 @@ import java.util.stream.Stream; -public class DatasetVariable implements Indexable, AttributeAware { +public class DatasetVariable implements Indexable, AttributeAware, IVariable { private static final long serialVersionUID = -141834508275072637L; @@ -239,6 +241,7 @@ public String getUnit() { return unit; } + @Override public String getValueType() { return valueType; } @@ -259,6 +262,7 @@ public String getOccurrenceGroup() { return occurrenceGroup; } + @Override public boolean hasCategories() { return categories != null && !categories.isEmpty(); } @@ -267,6 +271,13 @@ public List getCategories() { return categories; } + @Override + public List getCategoryNames() { + if (!hasCategories()) return Lists.newArrayList(); + return categories.stream().map(DatasetCategory::getName).collect(Collectors.toList()); + } + + @Override public DatasetCategory getCategory(String name) { if (!hasCategories()) return null; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 8f9e4d6d70..91d69adb54 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -35,7 +35,6 @@ import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; import org.obiba.mica.dataset.event.DatasetUpdatedEvent; -import org.obiba.mica.dataset.service.support.QueryTermsUtil; import org.obiba.mica.file.FileUtils; import org.obiba.mica.file.service.FileSystemService; import org.obiba.mica.micaConfig.service.MicaConfigService; @@ -50,7 +49,7 @@ import org.obiba.mica.study.service.IndividualStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -428,20 +427,14 @@ public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableN } @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName") - public SummaryStatisticsWrapper getVariableSummary(@NotNull StudyDataset dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { + public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull StudyDataset dataset, String variableName) { log.info("Caching variable summary {} {}", dataset.getId(), variableName); - return new SummaryStatisticsWrapper(getStudyTableSource(dataset, dataset.getSafeStudyTable()).getVariableSummary(variableName)); - } - - public Search.QueryResultDto getFacets(@NotNull StudyDataset dataset, Search.QueryTermsDto query) - throws NoSuchValueTableException, NoSuchVariableException { - return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getFacets(query); + return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getVariableSummary(variableName); } - public Search.QueryResultDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, - DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { - return getFacets(dataset, QueryTermsUtil.getContingencyQuery(variable, crossVariable)); + public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, + DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { + return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getContingency(variable, crossVariable); } public void delete(String id) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 1254da1f2c..48849e22d0 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -12,8 +12,6 @@ import com.google.common.base.Strings; import com.google.common.eventbus.EventBus; -import com.google.protobuf.GeneratedMessage; -import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf; import org.obiba.magma.*; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.EntityState; @@ -26,18 +24,13 @@ import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; import org.obiba.mica.spi.source.StudyTableSource; -import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.web.model.Math; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.inject.Inject; import javax.validation.constraints.NotNull; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.Serializable; import java.util.List; /** @@ -152,73 +145,4 @@ protected Iterable wrappedGetDatasetVariables(T dataset) { } } - /** - * Helper class to serialize protobuf object extension. - */ - public static class SummaryStatisticsWrapper implements Serializable { - @IgnoreSizeOf - private org.obiba.opal.web.model.Math.SummaryStatisticsDto summary; - - public SummaryStatisticsWrapper(Math.SummaryStatisticsDto summary) { - this.summary = summary; - } - - public Math.SummaryStatisticsDto getWrappedDto() { - return summary; - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - summary = (Math.SummaryStatisticsDto)in.readObject(); - GeneratedMessage ext = (GeneratedMessage)in.readObject(); - - if (ext == null) return; - - Math.SummaryStatisticsDto.Builder builder = summary.toBuilder(); - - if(ext instanceof Math.CategoricalSummaryDto) - builder.setExtension(Math.CategoricalSummaryDto.categorical, (Math.CategoricalSummaryDto) ext); - else if(ext instanceof Math.ContinuousSummaryDto) - builder.setExtension(Math.ContinuousSummaryDto.continuous, (Math.ContinuousSummaryDto) ext); - else if(ext instanceof Math.DefaultSummaryDto) - builder.setExtension(Math.DefaultSummaryDto.defaultSummary, (Math.DefaultSummaryDto) ext); - else if(ext instanceof Math.TextSummaryDto) - builder.setExtension(Math.TextSummaryDto.textSummary, (Math.TextSummaryDto) ext); - else if(ext instanceof Math.GeoSummaryDto) - builder.setExtension(Math.GeoSummaryDto.geoSummary, (Math.GeoSummaryDto) ext); - else if(ext instanceof Math.BinarySummaryDto) - builder.setExtension(Math.BinarySummaryDto.binarySummary, (Math.BinarySummaryDto) ext); - - summary = builder.build(); - } - - private void writeObject(java.io.ObjectOutputStream stream) - throws IOException { - GeneratedMessage ext = null; - - Math.SummaryStatisticsDto.Builder builder = Math.SummaryStatisticsDto.newBuilder(summary); - - if(summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { - ext = summary.getExtension(Math.CategoricalSummaryDto.categorical); - builder.clearExtension(Math.CategoricalSummaryDto.categorical); - } else if(summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { - ext = summary.getExtension(Math.ContinuousSummaryDto.continuous); - builder.clearExtension(Math.ContinuousSummaryDto.continuous); - } else if(summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { - ext = summary.getExtension(Math.DefaultSummaryDto.defaultSummary); - builder.clearExtension(Math.DefaultSummaryDto.defaultSummary); - } else if(summary.hasExtension(Math.TextSummaryDto.textSummary)) { - ext = summary.getExtension(Math.TextSummaryDto.textSummary); - builder.clearExtension(Math.TextSummaryDto.textSummary); - } else if(summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { - ext = summary.getExtension(Math.GeoSummaryDto.geoSummary); - builder.clearExtension(Math.GeoSummaryDto.geoSummary); - } else if(summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { - ext = summary.getExtension(Math.BinarySummaryDto.binarySummary); - builder.clearExtension(Math.BinarySummaryDto.binarySummary); - } - - stream.writeObject(builder.build()); - stream.writeObject(ext); - } - } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index a014564e29..d203715435 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -31,7 +31,6 @@ import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; import org.obiba.mica.dataset.event.DatasetUpdatedEvent; -import org.obiba.mica.dataset.service.support.QueryTermsUtil; import org.obiba.mica.file.FileUtils; import org.obiba.mica.file.service.FileSystemService; import org.obiba.mica.micaConfig.service.MicaConfigService; @@ -43,7 +42,7 @@ import org.obiba.mica.study.service.HarmonizationStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -377,28 +376,21 @@ public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String v getTable(dataset, studyId, source).getVariableValueSource(variableName).getVariable()); } - @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", - key = "#variableName + ':' + #studyId + ':' + #source") - public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String source) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, source); + @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName + ':' + #studyId + ':' + #source") + public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String source) { for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { if(baseTable.isFor(studyId, source)) { - return new SummaryStatisticsWrapper(getStudyTableSource(dataset, baseTable).getVariableSummary(variableName)); + log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, source); + return getStudyTableSource(dataset, baseTable).getVariableSummary(variableName); } } throw NoSuchStudyException.withId(studyId); } - public Search.QueryResultDto getFacets(@NotNull HarmonizationDataset dataset, Search.QueryTermsDto query, BaseStudyTable studyTable) - throws NoSuchStudyException, NoSuchValueTableException { - return getStudyTableSource(dataset, studyTable).getFacets(query); - } - - public Search.QueryResultDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, - DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { - return getFacets(dataset, QueryTermsUtil.getContingencyQuery(variable, crossVariable), studyTable); + public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, + DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { + return getStudyTableSource(dataset, studyTable).getContingency(variable, crossVariable); } @Override diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java index f0da0ed3a1..a85b777ed0 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java @@ -10,6 +10,7 @@ package org.obiba.mica.dataset.service; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.googlecode.protobuf.format.JsonFormat; @@ -256,11 +257,11 @@ private Magma.VariableDto toVariableDto(DatasetVariable datasetVariable) { builder.setName(datasetVariable.getName()); builder.setIndex(datasetVariable.getIndex()); - builder.setReferencedEntityType(datasetVariable.getReferencedEntityType()); + builder.setReferencedEntityType(toNonNullString(datasetVariable.getReferencedEntityType())); builder.setUnit(datasetVariable.getUnit()); - builder.setMimeType(datasetVariable.getMimeType()); + builder.setMimeType(toNonNullString(datasetVariable.getMimeType())); builder.setIsRepeatable(datasetVariable.isRepeatable()); - builder.setOccurrenceGroup(datasetVariable.getOccurrenceGroup()); + builder.setOccurrenceGroup(toNonNullString(datasetVariable.getOccurrenceGroup())); builder.setValueType(datasetVariable.getValueType()); builder.setEntityType(datasetVariable.getEntityType()); @@ -277,6 +278,10 @@ private Magma.VariableDto toVariableDto(DatasetVariable datasetVariable) { return builder.build(); } + private String toNonNullString(String value) { + return value == null ? "" : value; + } + private List toAttributeDtoList(Attributes attributes) { return attributes.asAttributeList().stream().map(attribute -> { Magma.AttributeDto.Builder builder = Magma.AttributeDto.newBuilder(); @@ -315,14 +320,16 @@ private List toCategoryDtoList(List categori private List toOpalTableFullName(Dataset dataset) { if (dataset instanceof StudyDataset) { StudyTable studyTable = ((StudyDataset) dataset).getSafeStudyTable(); - return Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSource())); + return OpalTableSource.isFor(studyTable.getSource()) ? Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSource())) : Lists.newArrayList(); } else { HarmonizationDataset harmoDataset = (HarmonizationDataset) dataset; // one for each study and harmo tables List tableNames = Lists.newArrayList(); tableNames.addAll(harmoDataset.getStudyTables().stream() + .filter(st -> OpalTableSource.isFor(st.getSource())) .map(st -> OpalTableSource.toTableName(st.getSource())).collect(Collectors.toList())); tableNames.addAll(harmoDataset.getHarmonizationTables().stream() + .filter(st -> OpalTableSource.isFor(st.getSource())) .map(ht -> OpalTableSource.toTableName(ht.getSource())).collect(Collectors.toList())); return tableNames; } diff --git a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java index b136120a2c..878bea61e6 100644 --- a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java +++ b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java @@ -16,6 +16,7 @@ import org.obiba.mica.core.domain.*; import org.obiba.mica.file.Attachment; import org.obiba.mica.spi.search.Indexable; +import org.obiba.mica.spi.source.IStudy; import javax.validation.constraints.NotNull; import java.beans.Transient; @@ -26,7 +27,7 @@ /** * Base class for representing all type of studies. */ -public abstract class BaseStudy extends AbstractModelAware implements PersonAware, Indexable { +public abstract class BaseStudy extends AbstractModelAware implements PersonAware, Indexable, IStudy { private Attachment logo; @@ -97,6 +98,7 @@ public void setObjectives(LocalizedString objectives) { this.objectives = objectives; } + @Override public String getOpal() { return opal; } diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java index 602b805943..1ce4740c64 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java @@ -13,29 +13,28 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.obiba.magma.type.BooleanType; import org.obiba.mica.JSONUtils; import org.obiba.mica.core.domain.*; import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.StudyDatasetStateRepository; import org.obiba.mica.dataset.domain.*; -import org.obiba.mica.micaConfig.domain.MicaConfig; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Term; import org.obiba.opal.core.domain.taxonomy.Vocabulary; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import javax.annotation.Nullable; import javax.inject.Inject; import javax.validation.constraints.NotNull; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Component @@ -460,10 +459,12 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab } public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, - @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + @Nullable Mica.DatasetVariableAggregationDto summary, boolean withStudySummary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = summary == null ? Mica.DatasetVariableAggregationDto.newBuilder() : summary.toBuilder(); - aggDto = simpleAggregationDto(aggDto, summary); + if (summary == null) { + aggDto.setTotal(0).setN(0); + } if(studyTable instanceof StudyTable) aggDto.setStudyTable(asDto((StudyTable) studyTable, withStudySummary)); @@ -473,255 +474,23 @@ else if (studyTable instanceof HarmonizationStudyTable) return aggDto; } - public Mica.DatasetVariableAggregationDto.Builder simpleAggregationDto(@NotNull Mica.DatasetVariableAggregationDto.Builder aggDto, @Nullable Math.SummaryStatisticsDto summary) { - - if(summary == null) return aggDto.setTotal(0).setN(0); - - if(summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { - aggDto = asDto(summary.getExtension(Math.CategoricalSummaryDto.categorical)); - } else if(summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { - aggDto = asDto(summary.getExtension(Math.ContinuousSummaryDto.continuous)); - } else if(summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { - aggDto = asDto(summary.getExtension(Math.DefaultSummaryDto.defaultSummary)); - } else if(summary.hasExtension(Math.TextSummaryDto.textSummary)) { - aggDto = asDto(summary.getExtension(Math.TextSummaryDto.textSummary)); - } else if(summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { - aggDto = asDto(summary.getExtension(Math.GeoSummaryDto.geoSummary)); - } else if(summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { - aggDto = asDto(summary.getExtension(Math.BinarySummaryDto.binarySummary)); - } - - return aggDto; - } - - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull BaseStudyTable studyTable, - DatasetVariable variable, DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { - Mica.DatasetVariableContingencyDto.Builder crossDto = Mica.DatasetVariableContingencyDto.newBuilder(); + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull BaseStudyTable studyTable, @Nullable Mica.DatasetVariableContingencyDto crossDto) { + Mica.DatasetVariableContingencyDto.Builder crossDtoBuilder = crossDto == null ? Mica.DatasetVariableContingencyDto.newBuilder() : crossDto.toBuilder(); if(studyTable instanceof StudyTable) - crossDto.setStudyTable(asDto((StudyTable) studyTable, true)); + crossDtoBuilder.setStudyTable(asDto((StudyTable) studyTable, true)); else if (studyTable instanceof HarmonizationStudyTable) - crossDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable)); - - Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + crossDtoBuilder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable)); - if(results == null) { + if(crossDto == null) { + Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); allAggBuilder.setN(0); allAggBuilder.setTotal(0); - crossDto.setAll(allAggBuilder); - return crossDto; - } - - allAggBuilder.setTotal(results.getTotalHits()); - MicaConfig micaConfig = micaConfigService.getConfig(); - int privacyThreshold = micaConfig.getPrivacyThreshold(); - crossDto.setPrivacyThreshold(privacyThreshold); - boolean privacyChecks = !crossVariable.hasCategories() || validatePrivacyThreshold(results, privacyThreshold); - boolean totalPrivacyChecks = validateTotalPrivacyThreshold(results, privacyThreshold); - - // add facet results in the same order as the variable categories - List catNames = variable.getValueType().equals(BooleanType.get().getName()) ? - Lists.newArrayList("true", "false") : variable.getCategories().stream().map(DatasetCategory::getName).collect(Collectors.toList()); - catNames.forEach(catName -> results.getFacetsList().stream() - .filter(facet -> facet.hasFacet() && catName.equals(facet.getFacet())).forEach(facet -> { - boolean privacyCheck = privacyChecks && checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold); - Mica.DatasetVariableAggregationDto.Builder aggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); - aggBuilder.setTotal(totalPrivacyChecks ? results.getTotalHits() : 0); - aggBuilder.setTerm(facet.getFacet()); - DatasetCategory category = variable.getCategory(facet.getFacet()); - aggBuilder.setMissing(category != null && category.isMissing()); - addSummaryStatistics(crossVariable, aggBuilder, facet, privacyCheck, totalPrivacyChecks); - crossDto.addAggregations(aggBuilder); - })); - - // add total facet for all variable categories - results.getFacetsList().stream().filter(facet -> facet.hasFacet() && "_total".equals(facet.getFacet())) - .forEach(facet -> { - boolean privacyCheck = privacyChecks && facet.getFilters(0).getCount() >= micaConfig.getPrivacyThreshold(); - addSummaryStatistics(crossVariable, allAggBuilder, facet, privacyCheck, totalPrivacyChecks); - }); - - crossDto.setAll(allAggBuilder); - - return crossDto; - } - - private boolean checkPrivacyThreshold(int count, int threshold) { - return count == 0 || count >= threshold; - } - - private boolean validateTotalPrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { - return results.getFacetsList().stream() - .allMatch(facet -> checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold)); - } - - private boolean validatePrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { - return results.getFacetsList().stream().map(Search.FacetResultDto::getFrequenciesList).flatMap(Collection::stream) - .allMatch(freq -> checkPrivacyThreshold(freq.getCount(), privacyThreshold)); - } - - private void addSummaryStatistics(DatasetVariable crossVariable, - Mica.DatasetVariableAggregationDto.Builder aggBuilder, Search.FacetResultDto facet, boolean privacyCheck, - boolean totalPrivacyCheck) { - aggBuilder.setN(totalPrivacyCheck ? facet.getFilters(0).getCount() : -1); - if(!privacyCheck) return; - - List catNames = crossVariable.getValueType().equals(BooleanType.get().getName()) ? - Lists.newArrayList("1", "0") : - (crossVariable.hasCategories() ? crossVariable.getCategories().stream().map(DatasetCategory::getName).collect(Collectors.toList()) : Lists.newArrayList()); - // order results as the order of cross variable categories - catNames.forEach(catName -> facet.getFrequenciesList().stream().filter(freq -> catName.equals(freq.getTerm())) - .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq)))); - // observed terms, not described by categories - facet.getFrequenciesList().stream().filter(freq -> !catNames.contains(freq.getTerm())) - .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq))); - - if(facet.hasStatistics()) { - aggBuilder.setStatistics(asDto(facet.getStatistics())); - } - } - - private Mica.FrequencyDto.Builder asDto(DatasetVariable crossVariable, - Search.FacetResultDto.TermFrequencyResultDto result) { - if (crossVariable.getValueType().equals(BooleanType.get().getName())) { - // for some reason 0/1 is returned instead of false/true - return Mica.FrequencyDto.newBuilder() - .setValue("1".equals(result.getTerm()) ? "true" : "false") - .setCount(result.getCount()) - .setMissing(false); - } else if (crossVariable.getCategory(result.getTerm()) != null) { - DatasetCategory category = crossVariable.getCategory(result.getTerm()); - return Mica.FrequencyDto.newBuilder() - .setValue(result.getTerm()) - .setCount(result.getCount()) - .setMissing(category != null && category.isMissing()); - } else { - // observed value, not described by a category - return Mica.FrequencyDto.newBuilder() - .setValue(result.getTerm()) - .setCount(result.getCount()) - .setMissing(false); - } - } - - private Mica.StatisticsDto.Builder asDto(Search.FacetResultDto.StatisticalResultDto result) { - return Mica.StatisticsDto.newBuilder() // - .setMin(result.getMin()) // - .setMax(result.getMax()) // - .setMean(result.getMean()) // - .setSum(result.getTotal()) // - .setSumOfSquares(result.getSumOfSquares()) // - .setVariance(result.getVariance()) // - .setStdDeviation(result.getStdDeviation()); - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.CategoricalSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList(), - summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.DefaultSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.TextSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList(), - summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.GeoSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.BinarySummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.FrequencyDto.Builder asDto(Math.FrequencyDto freq) { - return Mica.FrequencyDto.newBuilder().setValue(freq.getValue()).setCount(Long.valueOf(freq.getFreq()).intValue()) - .setMissing(freq.getMissing()); - } - - private Mica.IntervalFrequencyDto.Builder asDto(Math.IntervalFrequencyDto inter) { - return Mica.IntervalFrequencyDto.newBuilder().setCount((int)inter.getFreq()) - .setLower(inter.getLower()).setUpper(inter.getUpper()); - } - - private void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, - List frequencies) { - addFrequenciesDto(aggDto, frequencies, 0); - } - - private void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, List frequencies, - int otherFrequency) { - int n = otherFrequency; - if(frequencies != null) { - for(Math.FrequencyDto freq : frequencies) { - aggDto.addFrequencies(asDto(freq)); - if(!freq.getMissing()) n += freq.getFreq(); - } + crossDtoBuilder.setAll(allAggBuilder); + return crossDtoBuilder; } - if (otherFrequency>0) - aggDto.addFrequencies(Mica.FrequencyDto.newBuilder().setValue("???").setCount(otherFrequency) - .setMissing(false)); - aggDto.setN(n); - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.ContinuousSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - Math.DescriptiveStatsDto stats = summary.getSummary(); - - aggDto.setN(Long.valueOf(stats.getN()).intValue()); - Mica.StatisticsDto.Builder builder = Mica.StatisticsDto.newBuilder(); - - if(stats.hasSum()) builder.setSum(Double.valueOf(stats.getSum()).floatValue()); - if(stats.hasMin() && stats.getMin() != Double.POSITIVE_INFINITY) - builder.setMin(Double.valueOf(stats.getMin()).floatValue()); - if(stats.hasMax() && stats.getMax() != Double.NEGATIVE_INFINITY) - builder.setMax(Double.valueOf(stats.getMax()).floatValue()); - if(stats.hasMean() && !Double.isNaN(stats.getMean())) builder.setMean(Double.valueOf(stats.getMean()).floatValue()); - if(stats.hasSumsq() && !Double.isNaN(stats.getSumsq())) - builder.setSumOfSquares(Double.valueOf(stats.getSumsq()).floatValue()); - if(stats.hasVariance() && !Double.isNaN(stats.getVariance())) - builder.setVariance(Double.valueOf(stats.getVariance()).floatValue()); - if(stats.hasStdDev() && !Double.isNaN(stats.getStdDev())) - builder.setStdDeviation(Double.valueOf(stats.getStdDev()).floatValue()); - - aggDto.setStatistics(builder); - - if(summary.getFrequenciesCount() > 0) { - summary.getFrequenciesList().forEach(freq -> aggDto.addFrequencies(asDto(freq))); - } - - if (summary.getIntervalFrequencyCount() > 0) { - summary.getIntervalFrequencyList().forEach(inter -> aggDto.addIntervalFrequencies(asDto(inter))); - } - - int total = 0; - if(summary.getFrequenciesCount() > 0) { - for(Math.FrequencyDto freq : summary.getFrequenciesList()) { - total += freq.getFreq(); - } - } - aggDto.setTotal(total); - - return aggDto; + return crossDtoBuilder; } private Mica.DatasetDto.Builder asBuilder(Dataset dataset) { diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java index cb7a8d03f4..0d277e8440 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java @@ -428,19 +428,13 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(@NotNull StudyTable studyTabl @NotNull public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, - @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { + @Nullable Mica.DatasetVariableAggregationDto summary, boolean withStudySummary) { return datasetDtos.asDto(studyTable, summary, withStudySummary); } @NotNull - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull Math.SummaryStatisticsDto summary) { - return datasetDtos.simpleAggregationDto(Mica.DatasetVariableAggregationDto.newBuilder(), summary); - } - - @NotNull - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(BaseStudyTable studyTable, DatasetVariable variable, - DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { - return datasetDtos.asContingencyDto(studyTable, variable, crossVariable, results); + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(BaseStudyTable studyTable, @Nullable Mica.DatasetVariableContingencyDto results) { + return datasetDtos.asContingencyDto(studyTable, results); } @NotNull diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java index 886332f79d..6c8ae17ee2 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/PublishedDatasetResource.java @@ -74,7 +74,7 @@ public DatasetVariable getVariable(@PathParam("id") String id, @PathParam("proje @Path("/collected/{project}/{table}/{variableName}/_summary") public Mica.DatasetVariableAggregationDto getVariableSummary(@PathParam("id") String id, @PathParam("project") String project, @PathParam("table") String table, @PathParam("variableName") String variableName) { checkVariableSummaryAccess(); - return dtos.asDto(collectedDatasetService.getVariableSummary(alternativeStudyDataset(id, project, table), variableName).getWrappedDto()).build(); + return collectedDatasetService.getVariableSummary(alternativeStudyDataset(id, project, table), variableName); } private Dataset getDataset(String id) { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java index 1e3f65d751..c0af7be0a8 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetResource.java @@ -12,7 +12,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; - import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; import org.obiba.mica.core.domain.PublishCascadingScope; @@ -25,8 +24,6 @@ import org.obiba.mica.security.rest.SubjectAclResource; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Magma; -import org.obiba.opal.web.model.Search; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -150,13 +147,6 @@ public DraftCollectedDatasetVariableResource getVariable(@PathParam("variable") return resource; } - @POST - @Path("/facets") - public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { - checkPermission("/draft/collected-dataset", "VIEW"); - return datasetService.getFacets(getDataset(), query); - } - @PUT @Path("/_status") public Response updateStatus(@QueryParam("value") String status) { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java index 7d8b74aa1f..9e9056edca 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java @@ -10,20 +10,17 @@ package org.obiba.mica.dataset.rest.collection; -import javax.inject.Inject; -import javax.ws.rs.GET; -import javax.ws.rs.Path; - import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import javax.inject.Inject; +import javax.ws.rs.GET; + @Component @Scope("request") public class DraftCollectedDatasetVariableResource implements DatasetVariableResource { @@ -43,12 +40,6 @@ public Mica.DatasetVariableDto getVariable() { return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName)); } - @GET - @Path("/summary") - public Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName).getWrappedDto(); - } - private StudyDataset getDataset() { return datasetService.findById(datasetId); } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java index 93d6a0c355..c8165edacf 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java @@ -48,24 +48,6 @@ public Mica.DatasetVariableDto getVariable() { return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName)); } - @GET - @Path("/summary") - public List getVariableSummaries() { - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSource()) - .getWrappedDto()); - } catch(NoSuchVariableException | NoSuchValueTableException e) { - // ignore (case the study has not implemented this dataschema variable) - } - }); - return builder.build(); - } - @GET @Path("/harmonizations") public List getHarmonizedVariables() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java index 84c4e1d1f3..55fb77cbf0 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java @@ -13,10 +13,8 @@ import com.codahale.metrics.annotation.Timed; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; - import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; -import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.PublishCascadingScope; import org.obiba.mica.core.domain.RevisionStatus; import org.obiba.mica.core.service.AbstractGitPersistableService; @@ -27,8 +25,6 @@ import org.obiba.mica.security.rest.SubjectAclResource; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Magma; -import org.obiba.opal.web.model.Search; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -165,18 +161,6 @@ public DraftHarmonizedDatasetVariableResource getVariable(@PathParam("study") St return resource; } - @POST - @Path("/facets") - public List getFacets(Search.QueryTermsDto query) { - checkPermission("/draft/harmonized-dataset", "VIEW"); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - for (BaseStudyTable table : dataset.getBaseStudyTables()) { - builder.add(datasetService.getFacets(dataset, query, table)); - } - return builder.build(); - } - @PUT @Path("/_status") @Timed diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java index d335eda792..e192f32460 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java @@ -10,10 +10,6 @@ package org.obiba.mica.dataset.rest.harmonization; -import javax.inject.Inject; -import javax.ws.rs.GET; -import javax.ws.rs.Path; - import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.service.HarmonizedDatasetService; @@ -22,6 +18,9 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import javax.inject.Inject; +import javax.ws.rs.GET; + @Component @Scope("request") public class DraftHarmonizedDatasetVariableResource implements DatasetVariableResource { @@ -45,12 +44,6 @@ public Mica.DatasetVariableDto getVariable() { return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, source)); } - @GET - @Path("/summary") - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName, studyId, source).getWrappedDto(); - } - public void setStudyId(String studyId) { this.studyId = studyId; } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java index 6cdbf514cf..6c6ecfbdb2 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java @@ -22,7 +22,6 @@ import org.obiba.mica.dataset.search.rest.harmonization.ExcelContingencyWriter; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -54,15 +53,6 @@ public Mica.DatasetVariableDto getVariable() { return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Collected); } - @GET - @Path("/summary") - @Timed - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService.getVariableSummary(getDataset(StudyDataset.class, datasetId), variableName).getWrappedDto(); - } - @GET @Path("/aggregation") @Timed @@ -71,9 +61,8 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s checkVariableSummaryAccess(); StudyDataset dataset = getDataset(StudyDataset.class, datasetId); StudyTable studyTable = dataset.getSafeStudyTable(); - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); try { - return dtos.asDto(studyTable, datasetService.getVariableSummary(dataset, variableName).getWrappedDto(), withStudySummary).build(); + return dtos.asDto(studyTable, datasetService.getVariableSummary(dataset, variableName), withStudySummary).build(); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); return dtos.asDto(studyTable, null, withStudySummary).build(); @@ -97,11 +86,11 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var try { return dtos - .asContingencyDto(studyTable, var, crossVar, datasetService.getContingencyTable(dataset, var, crossVar)) + .asContingencyDto(studyTable, datasetService.getContingencyTable(dataset, var, crossVar)) .build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - return dtos.asContingencyDto(studyTable, var, crossVar, null).build(); + return dtos.asContingencyDto(studyTable, null).build(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index 9b669864a0..9d1a6cdf00 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -17,8 +17,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.commons.math3.util.Pair; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.DatasetVariable; @@ -26,8 +24,6 @@ import org.obiba.mica.dataset.search.rest.AbstractPublishedDatasetResource; import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -70,28 +66,6 @@ public Mica.DatasetVariableDto getVariable() { return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Dataschema); } - @GET - @Path("/summary") - @Timed - public List getVariableSummaries() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSource()) - .getWrappedDto()); - } catch (NoSuchVariableException | NoSuchValueTableException e) { - // case the study has not implemented this dataschema variable - builder.add(Math.SummaryStatisticsDto.newBuilder().setResource(variableName).build()); - } - }); - return builder.build(); - } - @GET @Path("/aggregation") @Timed @@ -102,12 +76,12 @@ public Mica.DatasetVariableAggregationsDto getVariableAggregations(@QueryParam(" HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); Mica.DatasetVariableAggregationsDto.Builder aggDto = Mica.DatasetVariableAggregationsDto.newBuilder(); - List> results = Lists.newArrayList(); + List> results = Lists.newArrayList(); dataset.getBaseStudyTables().forEach(table -> results.add(helper.getVariableFacet(dataset, variableName, table))); for (int i = 0; i < dataset.getBaseStudyTables().size(); i++) { BaseStudyTable opalTable = dataset.getBaseStudyTables().get(i); - Future futureResult = results.get(i); + Future futureResult = results.get(i); try { builder.add(dtos.asDto(opalTable, futureResult.get(), withStudySummary).build()); } catch (Exception e) { @@ -204,25 +178,25 @@ private Mica.DatasetVariableContingenciesDto getDatasetVariableContingenciesDto( HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); Mica.DatasetVariableContingenciesDto.Builder crossDto = Mica.DatasetVariableContingenciesDto.newBuilder(); - List> results = Lists.newArrayList(); + List> results = Lists.newArrayList(); dataset.getBaseStudyTables().forEach(table -> results.add(helper.getContingencyTable(dataset, var, crossVar, table))); Multimap termAggregations = LinkedListMultimap.create(); for (int i = 0; i < dataset.getBaseStudyTables().size(); i++) { - BaseStudyTable opalTable = dataset.getBaseStudyTables().get(i); - Future futureResult = results.get(i); + BaseStudyTable studyTable = dataset.getBaseStudyTables().get(i); + Future futureResult = results.get(i); try { Mica.DatasetVariableContingencyDto studyTableCrossDto = dtos - .asContingencyDto(opalTable, var, crossVar, futureResult.get()).build(); + .asContingencyDto(studyTable, futureResult.get()).build(); termAggregations.put(null, studyTableCrossDto.getAll()); studyTableCrossDto.getAggregationsList() .forEach(termAggDto -> termAggregations.put(termAggDto.getTerm(), termAggDto)); crossDto.addContingencies(studyTableCrossDto); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - crossDto.addContingencies(dtos.asContingencyDto(opalTable, var, crossVar, null)); + crossDto.addContingencies(dtos.asContingencyDto(studyTable, null)); } } @@ -263,13 +237,11 @@ public static class Helper { protected HarmonizedDatasetService datasetService; @Async - protected Future getVariableFacet(HarmonizationDataset dataset, String variableName, + protected Future getVariableFacet(HarmonizationDataset dataset, String variableName, BaseStudyTable table) { try { String studyId = table.getStudyId(); - return new AsyncResult<>(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getSource()) - .getWrappedDto()); + return new AsyncResult<>(datasetService.getVariableSummary(dataset, variableName, studyId, table.getSource())); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); return new AsyncResult<>(null); @@ -277,7 +249,7 @@ protected Future getVariableFacet(HarmonizationDatase } @Async - protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, + protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, DatasetVariable crossVar, BaseStudyTable studyTable) { try { return new AsyncResult<>(datasetService.getContingencyTable(dataset, studyTable, var, crossVar)); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index 4f23689209..9f3950408e 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -62,17 +62,6 @@ public Mica.DatasetVariableDto getVariable() { return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, source, tableType); } - @GET - @Path("/summary") - @Timed - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService - .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, source) - .getWrappedDto(); - } - @GET @Path("/aggregation") @Timed @@ -84,7 +73,7 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s if (baseTable.isFor(studyId, source)) { try { return dtos.asDto(baseTable, - datasetService.getVariableSummary(dataset, variableName, studyId, source).getWrappedDto(), withStudySummary).build(); + datasetService.getVariableSummary(dataset, variableName, studyId, source), withStudySummary).build(); } catch (Exception e) { log.warn("Unable to retrieve statistics: " + e.getMessage(), e); return dtos.asDto(baseTable, null, withStudySummary).build(); @@ -111,11 +100,10 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { if (baseTable.isFor(studyId, source)) { try { - return dtos.asContingencyDto(baseTable, var, crossVar, - datasetService.getContingencyTable(dataset, baseTable, var, crossVar)).build(); + return dtos.asContingencyDto(baseTable, datasetService.getContingencyTable(dataset, baseTable, var, crossVar)).build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - return dtos.asContingencyDto(baseTable, var, crossVar, null).build(); + return dtos.asContingencyDto(baseTable, null).build(); } } } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java new file mode 100644 index 0000000000..20bdbfb0f8 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +/** + * Helper class for implementing {@link StudyTableSource}. + */ +public abstract class AbstractStudyTableSource implements StudyTableSource { + + private StudyTableContext context; + + @Override + public void setStudyTableContext(StudyTableContext context) { + this.context = context; + } + + protected StudyTableContext getContext() { + return context; + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java new file mode 100644 index 0000000000..bdced82663 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +/** + * Category description, for summary statistics. + */ +public interface ICategory { + + /** + * Get the category name. + * + * @return + */ + String getName(); + + /** + * Get whether this category represents a missing value. + * + * @return + */ + boolean isMissing(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java new file mode 100644 index 0000000000..52a1195afb --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import org.obiba.mica.spi.search.Identified; + +import java.util.Map; + +/** + * Dataset as it is exposed to plugins. + */ +public interface IDataset extends Identified { + + /** + * Get the dataset model. + * + * @return + */ + Map getModel(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java new file mode 100644 index 0000000000..0d7de882ae --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import org.obiba.mica.spi.search.Identified; + +import java.util.Map; + +/** + * Study as it is exposed to plugins. + */ +public interface IStudy extends Identified { + + /** + * Get the Opal URL, if any is defined specifically for this study. + * + * @return + */ + String getOpal(); + + /** + * Get the study model. + * + * @return + */ + Map getModel(); +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java new file mode 100644 index 0000000000..cf5f690dee --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import java.util.List; + +/** + * Variable description, for summary statistics. + */ +public interface IVariable { + + /** + * Get the variable name. + * + * @return + */ + String getName(); + + /** + * Get the value type of the variable. + * + * @return + */ + String getValueType(); + + /** + * Whether the variable has categories (may still be categorical without, see boolean value type). + * + * @return + */ + boolean hasCategories(); + + /** + * Helper to get the category names. + * + * @return + */ + List getCategoryNames(); + + /** + * Get a category by its name. + * + * @param name + * @return + */ + ICategory getCategory(String name); +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java new file mode 100644 index 0000000000..dbde7e9793 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +public class StudyTableContext { + + private final IDataset dataset; + + private final IStudy study; + + private final int privacyThreshold; + + public StudyTableContext(IDataset dataset, IStudy study, int privacyThreshold) { + this.dataset = dataset; + this.study = study; + this.privacyThreshold = privacyThreshold; + } + + public IDataset getDataset() { + return dataset; + } + + public IStudy getStudy() { + return study; + } + + public int getPrivacyThreshold() { + return privacyThreshold; + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java index 3803f824e1..b386a9e120 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java @@ -11,8 +11,7 @@ package org.obiba.mica.spi.source; import org.obiba.magma.ValueTable; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; /** * Describes the parameters to establish a connection with a Datasource @@ -35,12 +34,13 @@ public interface StudyTableSource { ValueTable getValueTable(); /** - * Make a facet search, to retrieve contingency table results. + * Make a contingency query and return results. * - * @param query + * @param variable + * @param crossVariable * @return */ - Search.QueryResultDto getFacets(Search.QueryTermsDto query); + Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable); /** * Get a variable summary statistics. @@ -48,7 +48,7 @@ public interface StudyTableSource { * @param variableName * @return */ - Math.SummaryStatisticsDto getVariableSummary(String variableName); + Mica.DatasetVariableAggregationDto getVariableSummary(String variableName); /** * URN representation of a value table source, indicates the identifier of the value table, in the namespace of the source. @@ -57,4 +57,6 @@ public interface StudyTableSource { */ String getURN(); + void setStudyTableContext(StudyTableContext context); + } From d76429ece21aa25e6d9f2553b23baba92f8aee6f Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 5 Dec 2022 21:10:37 +0100 Subject: [PATCH 08/44] Deferred init of excel datasource. Less verbose log messages when study table source operation is not supported --- .../org/obiba/mica/core/source/ExcelTableSource.java | 11 ++++++++++- .../PublishedCollectedDatasetVariableResource.java | 5 ++++- .../PublishedDataschemaDatasetVariableResource.java | 10 ++++++++-- .../PublishedHarmonizedDatasetVariableResource.java | 5 ++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 01796ef89a..d64a2e165d 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -32,6 +32,8 @@ public class ExcelTableSource extends AbstractStudyTableSource implements StudyT @NotNull private String table; + private boolean initialized; + private ExcelDatasource excelDatasource; public static boolean isFor(String source) { @@ -71,6 +73,7 @@ public String getTable() { @Override public ValueTable getValueTable() { + ensureInitialzed(); return Strings.isNullOrEmpty(table) ? excelDatasource.getValueTables().stream().findFirst().get() : excelDatasource.getValueTable(table); @@ -94,6 +97,12 @@ public String getURN() { @Override public void initialise(InputStream in) { excelDatasource = new ExcelDatasource(path, in); - Initialisables.initialise(excelDatasource); + } + + private void ensureInitialzed() { + if (!initialized) { + Initialisables.initialise(excelDatasource); + initialized = true; + } } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java index 6c6ecfbdb2..68b5b66d88 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java @@ -64,7 +64,10 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s try { return dtos.asDto(studyTable, datasetService.getVariableSummary(dataset, variableName), withStudySummary).build(); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); return dtos.asDto(studyTable, null, withStudySummary).build(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index 9d1a6cdf00..2b1e23951a 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -85,7 +85,10 @@ public Mica.DatasetVariableAggregationsDto getVariableAggregations(@QueryParam(" try { builder.add(dtos.asDto(opalTable, futureResult.get(), withStudySummary).build()); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); builder.add(dtos.asDto(opalTable, null, withStudySummary).build()); } } @@ -243,7 +246,10 @@ protected Future getVariableFacet(Harmonizat String studyId = table.getStudyId(); return new AsyncResult<>(datasetService.getVariableSummary(dataset, variableName, studyId, table.getSource())); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); return new AsyncResult<>(null); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index 9f3950408e..88ddd8eba3 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -75,7 +75,10 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s return dtos.asDto(baseTable, datasetService.getVariableSummary(dataset, variableName, studyId, source), withStudySummary).build(); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); return dtos.asDto(baseTable, null, withStudySummary).build(); } } From 49bb303e481f5a3f065854d66f1360f52e029442 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 7 Dec 2022 22:27:23 +0100 Subject: [PATCH 09/44] SPI mica-source testing --- .../StudyTableSourceServiceRegistry.java | 1 - .../mica/core/source/ExcelTableSource.java | 15 +----- .../mica/core/source/OpalTableSource.java | 10 ++++ .../service/CollectedDatasetService.java | 8 +++- .../service/HarmonizedDatasetService.java | 9 ++-- .../micaConfig/service/PluginsService.java | 18 ++++--- ...DatasourceNotAvailableExceptionMapper.java | 2 +- .../spi/source/AbstractStudyTableSource.java | 21 ++++++++ .../AbstractStudyTableSourceService.java | 48 +++++++++++++++++++ .../mica/spi/source/StudyTableSource.java | 19 ++++++++ .../spi/source/StudyTableSourceService.java | 11 +++++ 11 files changed, 136 insertions(+), 26 deletions(-) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index dc816651ea..2fb3907308 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -84,7 +84,6 @@ private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() .filter(service -> service.isFor(source)).findFirst(); if (serviceOptional.isPresent()) { - // TODO add a context to the study table source StudyTableSource tableSource = serviceOptional.get().makeSource(source); tableSource.setStudyTableContext(context); if (tableSource instanceof StudyTableFileSource) { diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index d64a2e165d..76e94e74fd 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -29,7 +29,6 @@ public class ExcelTableSource extends AbstractStudyTableSource implements StudyT @NotNull private String path; - @NotNull private String table; private boolean initialized; @@ -73,22 +72,12 @@ public String getTable() { @Override public ValueTable getValueTable() { - ensureInitialzed(); + ensureInitialized(); return Strings.isNullOrEmpty(table) ? excelDatasource.getValueTables().stream().findFirst().get() : excelDatasource.getValueTable(table); } - @Override - public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { - throw new UnsupportedOperationException("Contingency search not available from an Excel file"); - } - - @Override - public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { - throw new UnsupportedOperationException("Summary statistics not available from an Excel file"); - } - @Override public String getURN() { return Strings.isNullOrEmpty(table) ? String.format("urn:file:%s", path) : String.format("urn:file:%s:%s", path, table); @@ -99,7 +88,7 @@ public void initialise(InputStream in) { excelDatasource = new ExcelDatasource(path, in); } - private void ensureInitialzed() { + private void ensureInitialized() { if (!initialized) { Initialisables.initialise(excelDatasource); initialized = true; diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index e2a64ca932..7c740ead43 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -62,6 +62,11 @@ public ValueTable getValueTable() { return getDatasource().getValueTable(table); } + @Override + public boolean providesContingency() { + return true; + } + @Override public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { Search.QueryTermsDto query = QueryTermsUtil.getContingencyQuery(variable, crossVariable); @@ -69,6 +74,11 @@ public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVa return OpalDtos.asDto(variable, crossVariable, getContext().getPrivacyThreshold(), results); } + @Override + public boolean providesVariableSummary() { + return true; + } + @Override public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { RestValueTable.RestVariableValueSource variableValueSource = (RestValueTable.RestVariableValueSource) getRestValueTable().getVariableValueSource(variableName); diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 91d69adb54..52956a9419 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -40,6 +40,7 @@ import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.DataCollectionEvent; @@ -429,12 +430,15 @@ public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableN @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName") public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull StudyDataset dataset, String variableName) { log.info("Caching variable summary {} {}", dataset.getId(), variableName); - return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getVariableSummary(variableName); + StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); + return tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; } public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { - return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getContingency(variable, crossVariable); + + StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); + return tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; } public void delete(String id) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index d203715435..d73823773b 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -36,6 +36,7 @@ import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.source.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.HarmonizationStudy; @@ -380,8 +381,9 @@ public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String v public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String source) { for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { if(baseTable.isFor(studyId, source)) { - log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, source); - return getStudyTableSource(dataset, baseTable).getVariableSummary(variableName); + log.info("Caching variable summary {} {} {} {}", dataset.getId(), variableName, studyId, source); + StudyTableSource tableSource = getStudyTableSource(dataset, baseTable); + return tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; } } @@ -390,7 +392,8 @@ public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull Harmonizat public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { - return getStudyTableSource(dataset, studyTable).getContingency(variable, crossVariable); + StudyTableSource tableSource = getStudyTableSource(dataset, studyTable); + return tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; } @Override diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index cb734f1605..c3b797f4f2 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -118,16 +118,16 @@ public void init() { */ private void initPlugins() { Collection plugins = getPlugins(true); - String pluginName = environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); + String searchPluginName = environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); try { - String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(pluginName); + String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(searchPluginName); // ensure there is a mica-search plugin installed if (plugins.stream().noneMatch(p -> "mica-search".equals(p.getType())) || plugins.stream() - .filter(plugin -> pluginName.equals(plugin.getName())) + .filter(plugin -> searchPluginName.equals(plugin.getName())) .filter(plugin -> plugin.getVersion().compareTo(new Version(pluginLatestVersion)) >= 0).count() == 0) { - installPlugin(pluginName, null); + installPlugin(searchPluginName, null); // rescan plugins plugins = getPlugins(true); } @@ -137,7 +137,7 @@ private void initPlugins() { boolean micaSearchFound = false; // mica-search plugin is a singleton List filteredPlugins = - plugins.stream().filter(plugin -> pluginName.equals(plugin.getName())) + plugins.stream().filter(plugin -> searchPluginName.equals(plugin.getName()) || "mica-source".equals(plugin.getType())) .sorted(Comparator.comparing(PluginResources::getVersion)) .collect(Collectors.toList()); @@ -145,6 +145,8 @@ private void initPlugins() { if ("mica-search".equals(plugin.getType()) && !micaSearchFound) { initSearchEngineServicePlugin(plugin); micaSearchFound = true; + } else if ("mica-source".equals(plugin.getType())) { + initStudyTableSourceServicePlugin(plugin); } } } @@ -162,8 +164,12 @@ private void initSearchEngineServicePlugin(PluginResources plugin) { servicePlugins.add(service); } - private void initStudyTableSourceServicePlugins(PluginResources plugin) { + private void initStudyTableSourceServicePlugin(PluginResources plugin) { StudyTableSourceServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { + Properties properties = plugin.getProperties(); + properties.setProperty("MICA_HOME", properties.getProperty("OPAL_HOME")); + properties.remove("OPAL_HOME"); + service.configure(properties); service.start(); servicePlugins.add(service); }); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java index fea8745cd4..7e7eec0e01 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java @@ -21,6 +21,6 @@ public class DatasourceNotAvailableExceptionMapper implements ExceptionMapper { @Override public Response toResponse(DatasourceNotAvailableException e) { - return Response.status(Status.SERVICE_UNAVAILABLE).entity("Verify the datasource is available.").build(); + return Response.status(Status.SERVICE_UNAVAILABLE).entity("Verify the datasource is available. " + e.getMessage()).build(); } } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java index 20bdbfb0f8..ffcb50df13 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java @@ -10,6 +10,8 @@ package org.obiba.mica.spi.source; +import org.obiba.mica.web.model.Mica; + /** * Helper class for implementing {@link StudyTableSource}. */ @@ -26,4 +28,23 @@ protected StudyTableContext getContext() { return context; } + @Override + public boolean providesContingency() { + return false; + } + + @Override + public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { + throw new UnsupportedOperationException("Contingency search not provided by: " + getClass().getSimpleName()); + } + + @Override + public boolean providesVariableSummary() { + return false; + } + + @Override + public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { + throw new UnsupportedOperationException("Summary statistics not provided by: " + getClass().getSimpleName()); + } } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java new file mode 100644 index 0000000000..d28260528e --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import java.util.Properties; + +/** + * Helper class. + */ +public abstract class AbstractStudyTableSourceService implements StudyTableSourceService { + + protected Properties properties; + + protected boolean running; + + @Override + public Properties getProperties() { + return properties; + } + + @Override + public void configure(Properties properties) { + this.properties = properties; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java index b386a9e120..e02bf60f30 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java @@ -33,6 +33,13 @@ public interface StudyTableSource { */ ValueTable getValueTable(); + /** + * Whether crossing variables is supported. + * + * @return + */ + boolean providesContingency(); + /** * Make a contingency query and return results. * @@ -42,6 +49,13 @@ public interface StudyTableSource { */ Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable); + /** + * Whether variable summaries are supported. + * + * @return + */ + boolean providesVariableSummary(); + /** * Get a variable summary statistics. * @@ -57,6 +71,11 @@ public interface StudyTableSource { */ String getURN(); + /** + * Set context in which the study table is defined. + * + * @param context + */ void setStudyTableContext(StudyTableContext context); } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java index 94727b4659..c28b2b008d 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java @@ -17,8 +17,19 @@ */ public interface StudyTableSourceService extends ServicePlugin { + /** + * Whether this service plugin can handle the study table source's URN. + * @param source + * @return + */ boolean isFor(String source); + /** + * Make a {@link StudyTableSource} from the source URN. + * + * @param source + * @return + */ StudyTableSource makeSource(String source); } From ce7009b56ad4056a6e3df9b13d2c9daa2eab176f Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 7 Dec 2022 23:17:20 +0100 Subject: [PATCH 10/44] Deferred attachment stream extraction --- .../StudyTableSourceServiceRegistry.java | 69 +++++++++++-------- .../mica/core/source/ExcelTableSource.java | 13 ++-- .../mica/spi/source/StudyTableFileSource.java | 2 +- .../mica/spi/source/StudyTableFileStream.java | 24 +++++++ 4 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 2fb3907308..bc03b656d5 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -78,7 +78,7 @@ private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, if (ExcelTableSource.isFor(source)) { ExcelTableSource tableSource = ExcelTableSource.fromURN(source); tableSource.setStudyTableContext(context); - tableSource.initialise(getFileInputStream(context, tableSource.getPath())); + tableSource.initialise(new AttachmentStream(context, tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() @@ -88,43 +88,56 @@ private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, tableSource.setStudyTableContext(context); if (tableSource instanceof StudyTableFileSource) { StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; - fileSource.initialise(getFileInputStream(context, fileSource.getPath())); + fileSource.initialise(new AttachmentStream(context, fileSource.getPath())); } return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } - private InputStream getFileInputStream(StudyTableContext context, String path) { - String fullPath = path; - Optional attachmentState; - if (!fullPath.startsWith("/")) { - // not a full path, then it may be relative to the dataset's folder - fullPath = String.format("/%s-dataset/%s/%s", (context.getDataset() instanceof StudyDataset ? "collected" : "harmonized"), context.getDataset().getId(), path); - attachmentState = getAttachmentState(fullPath); - // not found, then try a path relative to the study's folder - if (!attachmentState.isPresent()) { - fullPath = String.format("/%s-study/%s/%s", (context.getStudy() instanceof Study ? "individual" : "harmonization"), context.getStudy().getId(), path); + + private class AttachmentStream implements StudyTableFileStream { + + private final StudyTableContext context; + private final String path; + + private AttachmentStream(StudyTableContext context, String path) { + this.context = context; + this.path = path; + } + + @Override + public InputStream getInputStream() { + String fullPath = path; + Optional attachmentState; + if (!fullPath.startsWith("/")) { + // not a full path, then it may be relative to the dataset's folder + fullPath = String.format("/%s-dataset/%s/%s", (context.getDataset() instanceof StudyDataset ? "collected" : "harmonized"), context.getDataset().getId(), path); + attachmentState = getAttachmentState(fullPath); + // not found, then try a path relative to the study's folder + if (!attachmentState.isPresent()) { + fullPath = String.format("/%s-study/%s/%s", (context.getStudy() instanceof Study ? "individual" : "harmonization"), context.getStudy().getId(), path); + attachmentState = getAttachmentState(fullPath); + } + } else { attachmentState = getAttachmentState(fullPath); } - } else { - attachmentState = getAttachmentState(fullPath); - } - if (attachmentState.isPresent()) { - return fileStoreService.getFile(attachmentState.get().getAttachment().getFileReference()); - } else { - throw new NoSuchValueTableException("No value table at " + fullPath); + if (attachmentState.isPresent()) { + return fileStoreService.getFile(attachmentState.get().getAttachment().getFileReference()); + } else { + throw new NoSuchValueTableException("No value table at " + fullPath); + } } - } - private Optional getAttachmentState(String fullPath) { - log.info("Reading study table from file: {}", fullPath); - Pair pathName = FileSystemService.extractPathName(fullPath); - try { - AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); - return Optional.of(state); - } catch (Exception e) { - return Optional.empty(); + private Optional getAttachmentState(String fullPath) { + log.info("Reading study table from file: {}", fullPath); + Pair pathName = FileSystemService.extractPathName(fullPath); + try { + AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); + return Optional.of(state); + } catch (Exception e) { + return Optional.empty(); + } } } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 76e94e74fd..1fcc496b43 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -16,12 +16,10 @@ import org.obiba.magma.datasource.excel.ExcelDatasource; import org.obiba.magma.support.Initialisables; import org.obiba.mica.spi.source.AbstractStudyTableSource; -import org.obiba.mica.spi.source.IVariable; import org.obiba.mica.spi.source.StudyTableFileSource; -import org.obiba.mica.web.model.Mica; +import org.obiba.mica.spi.source.StudyTableFileStream; import javax.validation.constraints.NotNull; -import java.io.InputStream; import java.util.List; public class ExcelTableSource extends AbstractStudyTableSource implements StudyTableFileSource { @@ -35,6 +33,8 @@ public class ExcelTableSource extends AbstractStudyTableSource implements StudyT private ExcelDatasource excelDatasource; + private StudyTableFileStream fileStream; + public static boolean isFor(String source) { if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) return false; @@ -84,12 +84,15 @@ public String getURN() { } @Override - public void initialise(InputStream in) { - excelDatasource = new ExcelDatasource(path, in); + public void initialise(StudyTableFileStream in) { + this.fileStream = in; + // deferred init + this.initialized = false; } private void ensureInitialized() { if (!initialized) { + excelDatasource = new ExcelDatasource(path, fileStream.getInputStream()); Initialisables.initialise(excelDatasource); initialized = true; } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java index 527ebdca71..a1a6d5010b 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java @@ -29,6 +29,6 @@ public interface StudyTableFileSource extends StudyTableSource { * * @param in */ - void initialise(InputStream in); + void initialise(StudyTableFileStream in); } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java new file mode 100644 index 0000000000..0914f2d3f4 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.source; + +import java.io.InputStream; + +public interface StudyTableFileStream { + + /** + * Get the Mica file input stream on demand. + * + * @return + */ + InputStream getInputStream(); + +} From 7444705f7c5e17c2c7f7f5544c703428aa7c902f Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 8 Dec 2022 14:25:15 +0100 Subject: [PATCH 11/44] Handle disposable table sources --- .../StudyTableSourceServiceRegistry.java | 24 +++++++++++++++++-- .../service/CollectedDatasetService.java | 9 +++++-- .../service/HarmonizedDatasetService.java | 9 +++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index bc03b656d5..81b655e428 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -12,8 +12,10 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; import org.apache.commons.math3.util.Pair; import org.obiba.magma.NoSuchValueTableException; +import org.obiba.magma.support.Disposables; import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.domain.StudyDataset; @@ -29,6 +31,8 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import javax.inject.Inject; import java.io.InputStream; import java.util.NoSuchElementException; @@ -56,12 +60,28 @@ public class StudyTableSourceServiceRegistry { @Inject private FileStoreService fileStoreService; - private Cache sourcesCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(1, TimeUnit.MINUTES).build(); + private Cache sourcesCache; + + @PostConstruct + public void initialize() { + sourcesCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.MINUTES) + .removalListener((RemovalListener) notification -> { + Disposables.silentlyDispose(notification.getValue()); + }) + .build(); + } + + @PreDestroy + public void close() { + sourcesCache.cleanUp(); + } public synchronized StudyTableSource makeStudyTableSource(IDataset dataset, IStudy study, String source) { StudyTableContext context = new StudyTableContext(dataset, study, micaConfigService.getConfig().getPrivacyThreshold()); - String cacheKey = String.format("%s::%s", study.getId(), source); + String cacheKey = String.format("%s::%s::%s", dataset.getId(), study.getId(), source); try { return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(context, source)); } catch (ExecutionException e) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 52956a9419..9fa716201c 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -18,6 +18,7 @@ import org.obiba.magma.NoSuchValueTableException; import org.obiba.magma.NoSuchVariableException; import org.obiba.magma.ValueTable; +import org.obiba.magma.support.Disposables; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.AbstractGitPersistable; import org.obiba.mica.core.domain.PublishCascadingScope; @@ -431,14 +432,18 @@ public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableN public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull StudyDataset dataset, String variableName) { log.info("Caching variable summary {} {}", dataset.getId(), variableName); StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); - return tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Mica.DatasetVariableAggregationDto summary = tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Disposables.silentlyDispose(tableSource); + return summary; } public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); - return tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Mica.DatasetVariableContingencyDto results = tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Disposables.silentlyDispose(tableSource); + return results; } public void delete(String id) { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index d73823773b..058d3675c9 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -16,6 +16,7 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import org.obiba.magma.*; +import org.obiba.magma.support.Disposables; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.PublishCascadingScope; @@ -383,7 +384,9 @@ public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull Harmonizat if(baseTable.isFor(studyId, source)) { log.info("Caching variable summary {} {} {} {}", dataset.getId(), variableName, studyId, source); StudyTableSource tableSource = getStudyTableSource(dataset, baseTable); - return tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Mica.DatasetVariableAggregationDto summary = tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Disposables.silentlyDispose(tableSource); + return summary; } } @@ -393,7 +396,9 @@ public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull Harmonizat public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { StudyTableSource tableSource = getStudyTableSource(dataset, studyTable); - return tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Mica.DatasetVariableContingencyDto results = tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Disposables.silentlyDispose(tableSource); + return results; } @Override From f44653f82a4e53577c8f33088184b548a5cf0095 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 9 Dec 2022 10:53:13 +0100 Subject: [PATCH 12/44] Mica source plugin could be an Initialisable --- .../service/StudyTableSourceServiceRegistry.java | 15 ++++++++++----- .../obiba/mica/core/source/ExcelTableSource.java | 10 +++++----- .../obiba/mica/core/source/OpalTableSource.java | 2 +- .../mica/spi/source/StudyTableFileSource.java | 8 +++----- ...eam.java => StudyTableFileStreamProvider.java} | 5 ++++- 5 files changed, 23 insertions(+), 17 deletions(-) rename mica-spi/src/main/java/org/obiba/mica/spi/source/{StudyTableFileStream.java => StudyTableFileStreamProvider.java} (83%) diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 81b655e428..0342e2fff0 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -16,6 +16,7 @@ import org.apache.commons.math3.util.Pair; import org.obiba.magma.NoSuchValueTableException; import org.obiba.magma.support.Disposables; +import org.obiba.magma.support.Initialisables; import org.obiba.mica.core.source.ExcelTableSource; import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.domain.StudyDataset; @@ -92,13 +93,13 @@ private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, if (OpalTableSource.isFor(source)) { OpalTableSource tableSource = OpalTableSource.fromURN(source); tableSource.setStudyTableContext(context); - tableSource.initialise(opalService); + tableSource.setOpalService(opalService); return tableSource; } if (ExcelTableSource.isFor(source)) { ExcelTableSource tableSource = ExcelTableSource.fromURN(source); tableSource.setStudyTableContext(context); - tableSource.initialise(new AttachmentStream(context, tableSource.getPath())); + tableSource.setStudyTableFileStreamProvider(new AttachmentStreamProvider(context, tableSource.getPath())); return tableSource; } Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() @@ -108,20 +109,24 @@ private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, tableSource.setStudyTableContext(context); if (tableSource instanceof StudyTableFileSource) { StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; - fileSource.initialise(new AttachmentStream(context, fileSource.getPath())); + fileSource.setStudyTableFileStreamProvider(new AttachmentStreamProvider(context, fileSource.getPath())); } + Initialisables.initialise(tableSource); return tableSource; } throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); } - private class AttachmentStream implements StudyTableFileStream { + /** + * Get the input stream from an {@link AttachmentState} object. + */ + private class AttachmentStreamProvider implements StudyTableFileStreamProvider { private final StudyTableContext context; private final String path; - private AttachmentStream(StudyTableContext context, String path) { + private AttachmentStreamProvider(StudyTableContext context, String path) { this.context = context; this.path = path; } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 1fcc496b43..0ac4b420c9 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -17,7 +17,7 @@ import org.obiba.magma.support.Initialisables; import org.obiba.mica.spi.source.AbstractStudyTableSource; import org.obiba.mica.spi.source.StudyTableFileSource; -import org.obiba.mica.spi.source.StudyTableFileStream; +import org.obiba.mica.spi.source.StudyTableFileStreamProvider; import javax.validation.constraints.NotNull; import java.util.List; @@ -33,7 +33,7 @@ public class ExcelTableSource extends AbstractStudyTableSource implements StudyT private ExcelDatasource excelDatasource; - private StudyTableFileStream fileStream; + private StudyTableFileStreamProvider fileStreamProvider; public static boolean isFor(String source) { if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) @@ -84,15 +84,15 @@ public String getURN() { } @Override - public void initialise(StudyTableFileStream in) { - this.fileStream = in; + public void setStudyTableFileStreamProvider(StudyTableFileStreamProvider provider) { + this.fileStreamProvider = provider; // deferred init this.initialized = false; } private void ensureInitialized() { if (!initialized) { - excelDatasource = new ExcelDatasource(path, fileStream.getInputStream()); + excelDatasource = new ExcelDatasource(path, fileStreamProvider.getInputStream()); Initialisables.initialise(excelDatasource); initialized = true; } diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index 7c740ead43..271d34ab55 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -117,7 +117,7 @@ public static String toTableName(String source) { return source.replace("urn:opal:", ""); } - public void initialise(OpalService opalService) { + public void setOpalService(OpalService opalService) { this.opalService = opalService; this.opalUrl = getContext().getStudy().getOpal(); } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java index a1a6d5010b..3acafcb42e 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java @@ -10,8 +10,6 @@ package org.obiba.mica.spi.source; -import java.io.InputStream; - /** * Study table is to be extracted from a file which path applies to the Mica's internal file system. */ @@ -25,10 +23,10 @@ public interface StudyTableFileSource extends StudyTableSource { String getPath(); /** - * Initialise the source from the given input stream that represents the file content. + * Set the accessor to the input stream that represents the file content. * - * @param in + * @param provider */ - void initialise(StudyTableFileStream in); + void setStudyTableFileStreamProvider(StudyTableFileStreamProvider provider); } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java similarity index 83% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java rename to mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java index 0914f2d3f4..c84477b7ec 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStream.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java @@ -12,7 +12,10 @@ import java.io.InputStream; -public interface StudyTableFileStream { +/** + * Define an accessor to a file stream. + */ +public interface StudyTableFileStreamProvider { /** * Get the Mica file input stream on demand. From 2ce571d176895c87eec2b11ba7dd2c720d9fdd3d Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 12 Dec 2022 11:07:26 +0100 Subject: [PATCH 13/44] #4381 Variable taxonomies service added to support other taxonomies providers, in addition to the opal one --- .../mica/core/upgrade/Mica510upgrade.java | 42 +++- .../mica/micaConfig/domain/MicaConfig.java | 12 +- .../mica/micaConfig/service/CacheService.java | 6 +- .../service/MicaConfigurationProvider.java | 12 +- .../mica/micaConfig/service/OpalService.java | 203 +++--------------- .../micaConfig/service/PluginsService.java | 12 +- ...omyService.java => TaxonomiesService.java} | 60 ++++-- .../service/VariableTaxonomiesService.java | 176 +++++++++++++++ .../obiba/mica/web/model/MicaConfigDtos.java | 3 +- .../micaConfig/rest/MicaConfigResource.java | 17 +- .../taxonomy/rest/TaxonomiesResource.java | 27 +-- .../mica/taxonomy/rest/TaxonomyResource.java | 26 +-- .../AbstractPublishedDatasetResource.java | 6 +- .../mica/search/CoverageQueryExecutor.java | 7 +- .../obiba/mica/search/JoinQueryExecutor.java | 16 +- ...ConfigurationTaxonomyMetaDataProvider.java | 4 +- .../DatasetTaxonomyMetaDataProvider.java | 2 +- .../NetworkTaxonomyMetaDataProvider.java | 2 +- .../StudyTaxonomyMetaDataProvider.java | 2 +- .../TaxonomyAggregationMetaDataProvider.java | 34 ++- .../VariableTaxonomyMetaDataProvider.java | 2 +- .../search/queries/AbstractDocumentQuery.java | 4 +- .../mica/search/queries/DatasetQuery.java | 4 +- .../mica/search/queries/NetworkQuery.java | 4 +- .../obiba/mica/search/queries/StudyQuery.java | 4 +- .../mica/search/queries/VariableQuery.java | 28 +-- .../obiba/mica/taxonomy/TaxonomyIndexer.java | 26 +-- .../rest/AbstractTaxonomySearchResource.java | 26 +-- .../rest/TaxonomiesSearchResource.java | 6 +- mica-web-model/src/main/protobuf/Mica.proto | 2 +- .../controller/DatasetAnalysisController.java | 16 +- .../web/controller/VariableController.java | 6 +- 32 files changed, 434 insertions(+), 363 deletions(-) rename mica-core/src/main/java/org/obiba/mica/micaConfig/service/{TaxonomyService.java => TaxonomiesService.java} (86%) create mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java index ead2ffbccd..1c0c2ed354 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java @@ -2,11 +2,15 @@ import javax.inject.Inject; +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.json.JSONException; import org.obiba.mica.micaConfig.service.CacheService; import org.obiba.runtime.Version; import org.obiba.runtime.upgrade.UpgradeStep; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Component; @Component @@ -16,14 +20,17 @@ public class Mica510upgrade implements UpgradeStep { private CacheService cacheService; + private final MongoTemplate mongoTemplate; + @Inject - public Mica510upgrade(CacheService cacheService) { + public Mica510upgrade(CacheService cacheService, MongoTemplate mongoTemplate) { this.cacheService = cacheService; + this.mongoTemplate = mongoTemplate; } @Override public String getDescription() { - return "Clear caches for 5.1.0"; + return "Clear caches and updating config for 5.1.0"; } @Override @@ -34,7 +41,36 @@ public Version getAppliesTo() { @Override public void execute(Version currentVersion) { logger.info("Executing Mica upgrade to version 5.1.0"); + + try { + logger.info("Updating 'Mica Config'..."); + updateMicaConfig(); + } catch (JSONException e) { + logger.error("Error occurred while Updating 'Mica Config'"); + } + + logger.info("Clearing all caches..."); cacheService.clearAllCaches(); + + // TODO get updated search plugin and rebuild search index + } + + private void updateMicaConfig() throws JSONException { + Document micaConfig = getDBObjectSafely("micaConfig"); + // delete field anonymousCanCreateCart to reset to default + if (null != micaConfig) { + micaConfig.remove("opal"); + mongoTemplate.save(micaConfig, "micaConfig"); + } } - + + private Document getDBObjectSafely(String collectionName) { + if (mongoTemplate.collectionExists(collectionName)) { + MongoCollection existingCollection = mongoTemplate.getCollection(collectionName); + return existingCollection.find().first(); + } + + return null; + } + } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java index 9127911355..6f12151580 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java @@ -41,10 +41,6 @@ public class MicaConfig extends AbstractAuditableDocument { public static final String DEFAULT_CHARSET = Charsets.UTF_8.toString(); - public static final String DEFAULT_OPAL = "https://localhost:8443"; - - public static final String DEFAULT_PUBLIC_URL = "http://localhost:8082"; - public static final String[] LAYOUT_OPTIONS = {"layout1", "layout2"}; public static final long DEFAULT_MAX_ITEMS_PER_SET = 20000; @@ -64,7 +60,7 @@ public class MicaConfig extends AbstractAuditableDocument { @NotBlank private String defaultCharacterSet = DEFAULT_CHARSET; - private String opal = DEFAULT_OPAL; + private String opal; private List roles = Lists.newArrayList(Membership.CONTACT, Membership.INVESTIGATOR); @@ -221,8 +217,12 @@ public void setDefaultCharacterSet(String defaultCharacterSet) { this.defaultCharacterSet = defaultCharacterSet; } + public boolean hasOpal() { + return !Strings.isNullOrEmpty(opal); + } + public String getOpal() { - return opal == null ? "" : opal; + return opal; } public void setOpal(String opal) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index 4e38583b5a..2cbc639898 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -43,18 +43,18 @@ public class CacheService { private EventBus eventBus; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @CacheEvict(value = "opal-taxonomies", allEntries = true, beforeInvocation = true) public void clearOpalTaxonomiesCache() { log.info("Clearing opal taxonomies cache"); - taxonomyService.getOpalTaxonomies(); + taxonomiesService.getVariableTaxonomies(); } @CacheEvict(value = "micaConfig", allEntries = true) public void clearMicaConfigCache() { log.info("Clearing mica config cache"); - taxonomyService.refresh(); + taxonomiesService.refresh(); } @CacheEvict(value = "aggregations-metadata", allEntries = true) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java index 63efbb76c6..2a5807731a 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java @@ -27,7 +27,7 @@ public class MicaConfigurationProvider implements ConfigurationProvider { @Inject @Lazy - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Override public List getLocales() { @@ -46,26 +46,26 @@ public ObjectMapper getObjectMapper() { @Override public Taxonomy getNetworkTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } @Override public Taxonomy getStudyTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } @Override public Taxonomy getVariableTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } @Override public Taxonomy getDatasetTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } @Override public List getVariableTaxonomies() { - return taxonomyService.getVariableTaxonomies(); + return taxonomiesService.getAllVariableTaxonomies(); } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java index cfb12a6a7a..20e36d3002 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java @@ -11,7 +11,6 @@ package org.obiba.mica.micaConfig.service; import com.google.common.base.Strings; -import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; import org.obiba.magma.support.Initialisables; @@ -19,18 +18,12 @@ import org.obiba.mica.micaConfig.AuthType; import org.obiba.mica.micaConfig.domain.OpalCredential; import org.obiba.mica.micaConfig.service.helper.OpalServiceHelper; -import org.obiba.opal.core.cfg.NoSuchTaxonomyException; -import org.obiba.opal.core.cfg.NoSuchVocabularyException; import org.obiba.opal.core.domain.taxonomy.Taxonomy; -import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; -import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.obiba.opal.rest.client.magma.OpalJavaClient; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestDatasourceFactory; -import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.model.Projects; import org.obiba.opal.web.model.Search; -import org.obiba.opal.web.taxonomy.Dtos; import org.obiba.security.KeyStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +36,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.*; -import java.util.stream.Collectors; @Component public class OpalService implements EnvironmentAware { @@ -110,169 +102,21 @@ public synchronized RestDatasource getDatasource(@Nullable String opalUrl, Strin return datasource; } - private RestDatasource createRestDatasource(OpalCredential opalCredential, String projectUrl, String opalUrl, - String project) { - if (opalCredential.getAuthType() == AuthType.CERTIFICATE) { - KeyStoreManager kms = keyStoreService.getKeyStore(OPAL_KEYSTORE); - - if (!kms.aliasExists(opalCredential.getOpalUrl())) throw new IllegalStateException( - "Trying to use opal certificate credential but could not be found in keystore."); - - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, kms.getKeyStore(), opalUrl, - micaConfigService.getConfig().getSecretKey(), project).create(); - } else if (opalCredential.getAuthType() == AuthType.TOKEN) - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getToken(), project).create(); - - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getUsername(), - opalCredential.getPassword(), project).create(); - } - - /** - * Get a {@link RestDatasource} from the default Opal server. - * - * @param project - * @return - */ - public RestDatasource getDatasource(String project) { - return getDatasource(getDefaultOpal(), project); - } - - private String getOpalProjectUrl(String opalUrl, String project) { - String baseUrl = opalUrl == null ? getDefaultOpal() : opalUrl; - - return String.format("%s/ws/datasource/%s", StringUtils.stripEnd(baseUrl, "/"), project); - } - /** * Get the url of the default Opal server as defined in the configuration. * * @return */ public String getDefaultOpal() { - return environment.getProperty("opal.url"); + String opalConf = micaConfigService.getConfig().getOpal(); + String opalDefault = environment.getProperty("opal.url"); + return Strings.isNullOrEmpty(opalConf) ? opalDefault : opalConf; } // - // Taxonomies + // Opal Project // - public List getTaxonomies() { - Map taxonomies = getTaxonomiesInternal(); - List taxonomyList = Lists.newArrayList(taxonomies.values()); - Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); - return taxonomyList; - } - - public List getTaxonomyDtos() { - return getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); - } - - /** - * Get a summary of all the {@link Taxonomy}s available from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asSummaryDto) - .collect(Collectors.toList()); - - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); - } - - /** - * Get a summary of the {@link Taxonomy} available from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { - return Dtos.asSummaryDto(getTaxonomy(name)); - } - - /** - * Get a summary of all the {@link Taxonomy}s with their - * {@link Vocabulary}s from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) - .collect(Collectors.toList()); - - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); - } - - /** - * Get a summary of the {@link Taxonomy} with its - * {@link Vocabulary}s from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { - return Dtos.asVocabularySummaryDto(getTaxonomy(name)); - } - - /** - * Get a summary of the {@link Vocabulary} from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, - String vocabularyName) { - for (Vocabulary voc : getTaxonomy(name).getVocabularies()) { - if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); - } - throw new NoSuchVocabularyException(name, vocabularyName); - } - - /** - * Get the {@link Taxonomy} from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Taxonomy getTaxonomy(String name) { - return Dtos.fromDto(getTaxonomyDto(name)); - } - - /** - * Get the {@link Taxonomy} as a Dto from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Opal.TaxonomyDto getTaxonomyDto(String name) { - Map taxonomies = getTaxonomiesInternal(); - - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); - } - - return Dtos.asDto(taxonomies.get(name)); - } - - /** - * Get the {@link Vocabulary} as a Dto from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { - Map taxonomies = getTaxonomiesInternal(); - - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); - } - - return Dtos.asDto(taxonomies.get(name).getVocabulary(vocabularyName)); - } - public List getProjectDtos(String opalUrl) throws URISyntaxException { if (Strings.isNullOrEmpty(opalUrl)) opalUrl = getDefaultOpal(); @@ -283,31 +127,54 @@ public List getProjectDtos(String opalUrl) throws URISyntax } // - // Private methods + // Entities count // - private synchronized Map getTaxonomiesInternal() { + public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, String entityType) { try { - return opalServiceHelper.getTaxonomies(getOpalJavaClient()); + return opalServiceHelper.getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); } catch (URISyntaxException e) { log.error("Malformed opal URI", e); throw new NoSuchElementException(); } } - public Search.EntitiesResultDto getEntitiesCount(String query, String entityType) { - return getEntitiesCount(getDefaultOpal(), query, entityType); - } + // + // Private/package methods + // - public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, String entityType) { + synchronized Map getTaxonomiesInternal() { try { - return opalServiceHelper.getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); + return opalServiceHelper.getTaxonomies(getOpalJavaClient()); } catch (URISyntaxException e) { log.error("Malformed opal URI", e); throw new NoSuchElementException(); } } + private RestDatasource createRestDatasource(OpalCredential opalCredential, String projectUrl, String opalUrl, + String project) { + if (opalCredential.getAuthType() == AuthType.CERTIFICATE) { + KeyStoreManager kms = keyStoreService.getKeyStore(OPAL_KEYSTORE); + + if (!kms.aliasExists(opalCredential.getOpalUrl())) throw new IllegalStateException( + "Trying to use opal certificate credential but could not be found in keystore."); + + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, kms.getKeyStore(), opalUrl, + micaConfigService.getConfig().getSecretKey(), project).create(); + } else if (opalCredential.getAuthType() == AuthType.TOKEN) + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getToken(), project).create(); + + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getUsername(), + opalCredential.getPassword(), project).create(); + } + + private String getOpalProjectUrl(String opalUrl, String project) { + String baseUrl = opalUrl == null ? getDefaultOpal() : opalUrl; + + return String.format("%s/ws/datasource/%s", StringUtils.stripEnd(baseUrl, "/"), project); + } + private String getOpalUsername() { return environment.getProperty("opal.username"); } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index c3b797f4f2..d5c0930b9f 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -153,7 +153,7 @@ private void initPlugins() { private void initSearchEngineServicePlugin(PluginResources plugin) { SearchEngineService service = SearchEngineServiceLoader.get(plugin.getURLClassLoader(false)).iterator().next(); - Properties properties = plugin.getProperties(); + Properties properties = cleanProperties(plugin.getProperties()); for (String key : ES_CONFIGURATION) { if (environment.containsProperty(key)) properties.setProperty(key, environment.getProperty(key)); @@ -166,15 +166,19 @@ private void initSearchEngineServicePlugin(PluginResources plugin) { private void initStudyTableSourceServicePlugin(PluginResources plugin) { StudyTableSourceServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { - Properties properties = plugin.getProperties(); - properties.setProperty("MICA_HOME", properties.getProperty("OPAL_HOME")); - properties.remove("OPAL_HOME"); + Properties properties = cleanProperties(plugin.getProperties()); service.configure(properties); service.start(); servicePlugins.add(service); }); } + private Properties cleanProperties(Properties properties) { + properties.setProperty("MICA_HOME", properties.getProperty("OPAL_HOME")); + properties.remove("OPAL_HOME"); + return properties; + } + private synchronized Collection getPlugins(boolean extract) { Map pluginsMap = Maps.newLinkedHashMap(); // make sure plugins directory exists diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java similarity index 86% rename from mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java rename to mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index c2d105378a..66408aae1a 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -10,7 +10,6 @@ package org.obiba.mica.micaConfig.service; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -37,9 +36,9 @@ import com.google.common.eventbus.Subscribe; @Service -public class TaxonomyService { +public class TaxonomiesService { - private final OpalService opalService; + private final VariableTaxonomiesService variableTaxonomiesService; private final MicaConfigService micaConfigService; @@ -70,8 +69,8 @@ public class TaxonomyService { private Taxonomy networkTaxonomy; @Inject - public TaxonomyService( - OpalService opalService, + public TaxonomiesService( + VariableTaxonomiesService variableTaxonomiesService, MicaConfigService micaConfigService, StudyIdAggregationMetaDataHelper studyHelper, DatasetIdAggregationMetaDataHelper datasetHelper, @@ -81,7 +80,7 @@ public TaxonomyService( NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper, StudiesSetsAggregationMetaDataHelper studiesSetsHelper, VariablesSetsAggregationMetaDataHelper variablesSetsHelper) { - this.opalService = opalService; + this.variableTaxonomiesService = variableTaxonomiesService; this.micaConfigService = micaConfigService; this.studyHelper = studyHelper; this.datasetHelper = datasetHelper; @@ -106,48 +105,73 @@ public boolean metaTaxonomyContains(String taxonomy) { return false; } + /** + * Get the taxonomy that describes the {@link org.obiba.mica.network.domain.Network} properties. + * + * @return + */ @NotNull public Taxonomy getNetworkTaxonomy() { initialize(); return networkTaxonomy; } + /** + * Get the taxonomy that describes the {@link org.obiba.mica.study.domain.BaseStudy} properties. + * + * @return + */ @NotNull public Taxonomy getStudyTaxonomy() { initialize(); return studyTaxonomy; } + /** + * Get the taxonomy that describes the {@link org.obiba.mica.dataset.domain.Dataset} properties. + * + * @return + */ @NotNull public Taxonomy getDatasetTaxonomy() { initialize(); return datasetTaxonomy; } + /** + * Get the taxonomy that describes the {@link org.obiba.mica.dataset.domain.DatasetVariable} properties. + * + * @return + */ @NotNull public Taxonomy getVariableTaxonomy() { initialize(); return variableTaxonomy; } + /** + * Get all taxonomies that apply to the variables, including the one about the built-in properties of the {@link org.obiba.mica.dataset.domain.DatasetVariable}. + * + * @return + */ @NotNull - public List getVariableTaxonomies() { - return Stream.concat(getOpalTaxonomies().stream(), Stream.of(getVariableTaxonomy())).collect(Collectors.toList()); + public List getAllVariableTaxonomies() { + return Stream.concat(getVariableTaxonomies().stream(), Stream.of(getVariableTaxonomy())).collect(Collectors.toList()); } + /** + * Get the taxonomies that apply to the variables' annotations. + * + * @return + */ @NotNull - public synchronized List getOpalTaxonomies() { - List taxonomies = null; - - try { - taxonomies = opalService.getTaxonomies(); - } catch(Exception e) { - // ignore - } - - return taxonomies == null ? Collections.emptyList() : taxonomies; + public synchronized List getVariableTaxonomies() { + return variableTaxonomiesService.getSafeTaxonomies(); } + /** + * Prepare taxonomies for being re-initialized. + */ public synchronized void refresh() { taxonomyTaxonomy = null; networkTaxonomy = null; diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java new file mode 100644 index 0000000000..d6b1ed88b7 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.service; + + +import com.google.common.collect.Lists; +import org.obiba.opal.core.cfg.NoSuchTaxonomyException; +import org.obiba.opal.core.cfg.NoSuchVocabularyException; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; +import org.obiba.opal.core.domain.taxonomy.Vocabulary; +import org.obiba.opal.web.model.Opal; +import org.obiba.opal.web.taxonomy.Dtos; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Access to the variable taxonomies, from the different providers, including opal, local files and plugins. + */ +@Component +public class VariableTaxonomiesService { + + @Inject + private OpalService opalService; + + public List getTaxonomies() { + Map taxonomies = opalService.getTaxonomiesInternal(); + List taxonomyList = Lists.newArrayList(taxonomies.values()); + Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); + return taxonomyList; + } + + /** + * Get a not-null list of taxonomies. + * + * @return + */ + @NotNull + public List getSafeTaxonomies() { + List taxonomies = null; + + try { + taxonomies = getTaxonomies(); + } catch(Exception e) { + // ignore + } + + return taxonomies == null ? Collections.emptyList() : taxonomies; + } + + public List getTaxonomyDtos() { + return getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); + } + + /** + * Get a summary of all the {@link Taxonomy}s available from Opal master. + * + * @return + */ + public Opal.TaxonomiesDto getTaxonomySummaryDtos() { + List summaries = getTaxonomies().stream().map(Dtos::asSummaryDto) + .collect(Collectors.toList()); + + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} available from Opal master. + * + * @param name the taxonomy name + * @return + */ + public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { + return Dtos.asSummaryDto(getTaxonomy(name)); + } + + /** + * Get a summary of all the {@link Taxonomy}s with their + * {@link Vocabulary}s from Opal master. + * + * @return + */ + public Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { + List summaries = getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) + .collect(Collectors.toList()); + + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} with its + * {@link Vocabulary}s from Opal master. + * + * @param name the taxonomy name + * @return + */ + public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { + return Dtos.asVocabularySummaryDto(getTaxonomy(name)); + } + + /** + * Get a summary of the {@link Vocabulary} from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, + String vocabularyName) { + for (Vocabulary voc : getTaxonomy(name).getVocabularies()) { + if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); + } + throw new NoSuchVocabularyException(name, vocabularyName); + } + + /** + * Get the {@link Taxonomy} from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + public Taxonomy getTaxonomy(String name) { + return Dtos.fromDto(getTaxonomyDto(name)); + } + + /** + * Get the {@link Taxonomy} as a Dto from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + public Opal.TaxonomyDto getTaxonomyDto(String name) { + Map taxonomies = opalService.getTaxonomiesInternal(); + + if (!taxonomies.containsKey(name)) { + throw new NoSuchTaxonomyException(name); + } + + return Dtos.asDto(taxonomies.get(name)); + } + + /** + * Get the {@link Vocabulary} as a Dto from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + public Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { + Map taxonomies = opalService.getTaxonomiesInternal(); + + if (!taxonomies.containsKey(name)) { + throw new NoSuchTaxonomyException(name); + } + + return Dtos.asDto(taxonomies.get(name).getVocabulary(vocabularyName)); + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java index 43611063c6..af8939af94 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java @@ -103,7 +103,8 @@ Mica.MicaConfigDto asDto(@NotNull MicaConfig config, String language) { } builder.setIsUsePublicUrlForSharedLink(config.isUsePublicUrlForSharedLink()); - builder.setOpal(config.getOpal()); + if (config.hasOpal()) + builder.setOpal(config.getOpal()); builder.setPrivacyThreshold(config.getPrivacyThreshold()); if(config.getMicaVersion() != null) { diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java index 16bacf229e..8273ad7e0d 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java @@ -72,11 +72,13 @@ public class MicaConfigResource { private MicaConfigService micaConfigService; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private OpalService opalService; + @Inject VariableTaxonomiesService variableTaxonomiesService; + @Inject private OpalCredentialService opalCredentialService; @@ -86,9 +88,6 @@ public class MicaConfigResource { @Inject private Dtos dtos; - @Inject - private CacheService cacheService; - @Inject private EventBus eventBus; @@ -119,7 +118,7 @@ public Mica.PublicMicaConfigDto getPublic() { @RequiresRoles(Roles.MICA_ADMIN) public Response create(@SuppressWarnings("TypeMayBeWeakened") Mica.MicaConfigDto dto) { MicaConfig micaConfig = dtos.fromDto(dto); - taxonomyService.refreshTaxonomyTaxonomyIfNeeded(micaConfigService.getConfig(), micaConfig); + taxonomiesService.refreshTaxonomyTaxonomyIfNeeded(micaConfigService.getConfig(), micaConfig); micaConfigService.save(micaConfig); return Response.noContent().build(); } @@ -678,7 +677,7 @@ public Map getAvailableLanguages(@QueryParam("locale") @DefaultV @RequiresAuthentication @Deprecated public List getTaxonomies() { - return opalService.getTaxonomyDtos(); + return variableTaxonomiesService.getTaxonomyDtos(); } /** @@ -690,7 +689,7 @@ public List getTaxonomies() { @RequiresAuthentication @Deprecated public Opal.TaxonomiesDto getTaxonomySummaries() { - return opalService.getTaxonomySummaryDtos(); + return variableTaxonomiesService.getTaxonomySummaryDtos(); } /** @@ -702,7 +701,7 @@ public Opal.TaxonomiesDto getTaxonomySummaries() { @RequiresAuthentication @Deprecated public Opal.TaxonomiesDto getTaxonomyVocabularySummaries() { - return opalService.getTaxonomyVocabularySummaryDtos(); + return variableTaxonomiesService.getTaxonomyVocabularySummaryDtos(); } /** @@ -714,6 +713,6 @@ public Opal.TaxonomiesDto getTaxonomyVocabularySummaries() { @RequiresAuthentication @Deprecated public Opal.TaxonomyDto getStudyTaxonomy() { - return org.obiba.opal.web.taxonomy.Dtos.asDto(taxonomyService.getStudyTaxonomy()); + return org.obiba.opal.web.taxonomy.Dtos.asDto(taxonomiesService.getStudyTaxonomy()); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java index f5d32b12c9..cf7aca5b62 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java @@ -10,28 +10,21 @@ package org.obiba.mica.taxonomy.rest; -import java.util.List; - -import javax.inject.Inject; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Response; - +import com.codahale.metrics.annotation.Timed; import com.google.common.eventbus.EventBus; -import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.security.Roles; import org.obiba.opal.web.model.Opal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; +import java.util.List; @Component @Path("/taxonomies") @@ -40,7 +33,7 @@ public class TaxonomiesResource { private static final Logger logger = LoggerFactory.getLogger(TaxonomiesResource.class); @Inject - private OpalService opalService; + private VariableTaxonomiesService variableTaxonomiesService; @Inject private EventBus eventBus; @@ -48,7 +41,7 @@ public class TaxonomiesResource { @GET @Timed public List getTaxonomies() { - return opalService.getTaxonomyDtos(); + return variableTaxonomiesService.getTaxonomyDtos(); } @GET @@ -56,8 +49,8 @@ public List getTaxonomies() { @Timed public Opal.TaxonomiesDto getTaxonomySummaries( @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return opalService.getTaxonomyVocabularySummaryDtos(); - return opalService.getTaxonomySummaryDtos(); + if(withVocabularies) return variableTaxonomiesService.getTaxonomyVocabularySummaryDtos(); + return variableTaxonomiesService.getTaxonomySummaryDtos(); } @PUT diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java index 924438856b..a537237c50 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java @@ -10,31 +10,25 @@ package org.obiba.mica.taxonomy.rest; -import javax.inject.Inject; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; - -import org.apache.shiro.authz.annotation.RequiresAuthentication; -import org.obiba.mica.micaConfig.service.OpalService; +import com.codahale.metrics.annotation.Timed; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.opal.web.model.Opal; import org.springframework.stereotype.Component; -import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.ws.rs.*; @Component @Path("/taxonomy/{name}") public class TaxonomyResource { @Inject - private OpalService opalService; + private VariableTaxonomiesService variableTaxonomiesService; @GET @Timed public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { - return opalService.getTaxonomyDto(name); + return variableTaxonomiesService.getTaxonomyDto(name); } @GET @@ -42,8 +36,8 @@ public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name") String name, @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return opalService.getTaxonomyVocabularySummaryDto(name); - return opalService.getTaxonomySummaryDto(name); + if(withVocabularies) return variableTaxonomiesService.getTaxonomyVocabularySummaryDto(name); + return variableTaxonomiesService.getTaxonomySummaryDto(name); } @GET @@ -51,7 +45,7 @@ public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name @Timed public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return opalService.getTaxonomyVocabularyDto(name, vocabularyName); + return variableTaxonomiesService.getTaxonomyVocabularyDto(name, vocabularyName); } @GET @@ -59,7 +53,7 @@ public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getVocabularySummary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return opalService.getTaxonomyVocabularySummaryDto(name, vocabularyName); + return variableTaxonomiesService.getTaxonomyVocabularySummaryDto(name, vocabularyName); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java index ce8c28ba2f..641a576408 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java @@ -21,7 +21,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.micaConfig.service.MicaConfigService; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; @@ -66,7 +66,7 @@ public abstract class AbstractPublishedDatasetResource { protected ObjectMapper objectMapper; @Inject - protected OpalService opalService; + protected VariableTaxonomiesService variableTaxonomiesService; @Inject protected MicaConfigService micaConfigService; @@ -301,7 +301,7 @@ protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull S protected List getTaxonomies() { List taxonomies = null; try { - taxonomies = opalService.getTaxonomies(); + taxonomies = variableTaxonomiesService.getTaxonomies(); } catch (Exception e) { // ignore } diff --git a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java index a4ea340c0c..64d5b0e76f 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java +++ b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java @@ -12,7 +12,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.mica.spi.search.support.JoinQuery; @@ -24,7 +24,6 @@ import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import sun.util.locale.LanguageTag; import javax.annotation.Nullable; import javax.inject.Inject; @@ -43,7 +42,7 @@ public class CoverageQueryExecutor { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; @Inject - private OpalService opalService; + private VariableTaxonomiesService variableTaxonomiesService; @Inject private JoinQueryExecutor joinQueryExecutor; @@ -332,7 +331,7 @@ private MicaSearch.BucketCoverageDto.Builder getBucketCoverageDtoBuilder(String private List getTaxonomies() { List taxonomies = null; try { - taxonomies = opalService.getTaxonomies(); + taxonomies = variableTaxonomiesService.getTaxonomies(); } catch (Exception e) { // ignore } diff --git a/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java b/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java index 2330389680..defe3889fd 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java +++ b/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java @@ -17,7 +17,7 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.UncheckedTimeoutException; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.search.queries.DatasetQuery; import org.obiba.mica.search.queries.DocumentQueryInterface; import org.obiba.mica.search.queries.NetworkQuery; @@ -74,7 +74,7 @@ public class JoinQueryExecutor { private long concurrentJoinQueriesWaitTimeout; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private VariableQuery variableQuery; @@ -182,22 +182,22 @@ private JoinQueryResultDto buildQueryResult(JoinQuery joinQueryDto) { builder.setVariableResultDto(joinQueryDto.isWithFacets() ? addAggregationTitles(variableQuery.getResultQuery(), - ImmutableList.builder().add(taxonomyService.getVariableTaxonomy()) - .addAll(taxonomyService.getOpalTaxonomies()).build(), aggregationPostProcessor()) + ImmutableList.builder().add(taxonomiesService.getVariableTaxonomy()) + .addAll(taxonomiesService.getVariableTaxonomies()).build(), aggregationPostProcessor()) : removeAggregations(variableQuery.getResultQuery())); builder.setDatasetResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(datasetQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getDatasetTaxonomy()), + ? addAggregationTitles(datasetQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getDatasetTaxonomy()), null) : removeAggregations(datasetQuery.getResultQuery())); builder.setStudyResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(studyQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getStudyTaxonomy()), + ? addAggregationTitles(studyQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getStudyTaxonomy()), null) : removeAggregations(studyQuery.getResultQuery())); builder.setNetworkResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(networkQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getNetworkTaxonomy()), + ? addAggregationTitles(networkQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getNetworkTaxonomy()), null) : removeAggregations(networkQuery.getResultQuery())); @@ -259,7 +259,7 @@ private String getVocabularyAggregationName(Vocabulary vocabulary) { protected Function, List> aggregationPostProcessor() { return (aggregationResultDtos) -> { - Map buildres = taxonomyService.getOpalTaxonomies().stream() + Map buildres = taxonomiesService.getVariableTaxonomies().stream() .collect(Collectors.toMap(Taxonomy::getName, t -> { MicaSearch.AggregationResultDto.Builder builder = MicaSearch.AggregationResultDto.newBuilder() .setAggregation(t.getName()); diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java index d476908e16..1be9371a78 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java @@ -18,7 +18,7 @@ import com.google.common.base.Strings; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Vocabulary; @@ -32,7 +32,7 @@ public abstract class ConfigurationTaxonomyMetaDataProvider implements Aggregati private static final Logger log = LoggerFactory.getLogger(ConfigurationTaxonomyMetaDataProvider.class); @Inject - protected TaxonomyService taxonomyService; + protected TaxonomiesService taxonomiesService; Map> cache; diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java index 3bba075774..503b0f15d8 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class DatasetTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaDa @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java index 9dfefbe1e2..e160d53c6d 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class NetworkTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaDa @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java index 65714bac1e..c885a14183 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class StudyTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaData @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java index 0fbedae7e2..5509c51b6e 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java @@ -10,31 +10,28 @@ package org.obiba.mica.search.aggregations; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import javax.inject.Inject; - -import org.obiba.mica.spi.search.support.AttributeKey; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; +import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Term; import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.springframework.stereotype.Component; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; +import javax.inject.Inject; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; @Component public class TaxonomyAggregationMetaDataProvider implements AggregationMetaDataProvider { @Inject - OpalService opalService; + VariableTaxonomiesService variableTaxonomiesService; Map> cache; @@ -105,7 +102,7 @@ private Optional getVocabulary(String aggregation) { String targetTaxonomy = attrKey.hasNamespace(null) ? "Default" : attrKey.getNamespace(); String targetVocabulary = attrKey.getName(); - return getTaxonomies().stream() // + return getVariableTaxonomies().stream() // .filter(taxonomy -> !Strings.isNullOrEmpty(targetTaxonomy) && taxonomy.getName().equals(targetTaxonomy)) // .map(Taxonomy::getVocabularies) // .flatMap(Collection::stream) // @@ -113,12 +110,7 @@ private Optional getVocabulary(String aggregation) { .findFirst(); } - protected List getTaxonomies() { - try { - return opalService.getTaxonomies(); - } catch(Exception e) { - // ignore - } - return Collections.emptyList(); + protected List getVariableTaxonomies() { + return variableTaxonomiesService.getSafeTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java index 1a8ba47814..7a102a116b 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class VariableTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaD @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java index 7e0bfe45ad..350171a1bf 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java @@ -16,7 +16,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.obiba.mica.micaConfig.service.MicaConfigService; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.mica.search.DocumentQueryHelper; import org.obiba.mica.search.aggregations.AggregationMetaDataResolver; @@ -55,7 +55,7 @@ public abstract class AbstractDocumentQuery implements DocumentQueryInterface { protected MicaConfigService micaConfigService; @Inject - protected TaxonomyService taxonomyService; + protected TaxonomiesService taxonomiesService; @Inject protected SubjectAclService subjectAclService; diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java index 05df358b7d..ad2af62a63 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java @@ -99,7 +99,7 @@ public Collection getValues() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } public void setDatasetIdProvider(DocumentQueryIdProvider provider) { @@ -135,7 +135,7 @@ public void query(List studyIds, CountStatsData counts, QueryScope scope @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getDatasetTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getDatasetTaxonomy()); if (!properties.containsKey(STUDY_JOIN_FIELD)) properties.put(STUDY_JOIN_FIELD, ""); if (!properties.containsKey(HARMONIZATION_STUDY_JOIN_FIELD)) properties.put(HARMONIZATION_STUDY_JOIN_FIELD, ""); if (!properties.containsKey(ID)) properties.put(ID, ""); diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java index 4633d4e16c..2931149467 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java @@ -103,13 +103,13 @@ public Collection getValues() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getNetworkTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getNetworkTaxonomy()); if (!properties.containsKey(JOIN_FIELD)) properties.put(JOIN_FIELD, ""); return properties; } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java index 8a7d83b887..b5aa2dc989 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java @@ -91,7 +91,7 @@ protected Searcher.IdFilter getAccessibleIdFilter() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } @Override @@ -139,7 +139,7 @@ private Consumer getStudyConsumer(QueryScope scope, StudyResultDto.Bu @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getStudyTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getStudyTaxonomy()); if (!properties.containsKey(JOIN_FIELD)) properties.put(JOIN_FIELD, ""); return properties; } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java index 6e9a448bf7..5262d0295a 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java @@ -14,11 +14,10 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import javax.inject.Inject; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.dataset.service.HarmonizedDatasetService; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.mica.network.domain.Network; import org.obiba.mica.search.DocumentQueryHelper; @@ -40,6 +39,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Nullable; +import javax.inject.Inject; import javax.validation.constraints.NotNull; import java.io.IOException; import java.util.*; @@ -63,7 +63,7 @@ public class VariableQuery extends AbstractDocumentQuery { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; - private final OpalService opalService; + private final VariableTaxonomiesService variableTaxonomiesService; private final PublishedStudyService publishedStudyService; @@ -91,7 +91,7 @@ public class VariableQuery extends AbstractDocumentQuery { @Inject public VariableQuery( - OpalService opalService, + VariableTaxonomiesService variableTaxonomiesService, PublishedStudyService publishedStudyService, CollectedDatasetService collectedDatasetService, HarmonizedDatasetService harmonizedDatasetService, @@ -103,7 +103,7 @@ public VariableQuery( PopulationAggregationMetaDataProvider populationAggregationMetaDataProvider, StudyAggregationMetaDataProvider studyAggregationMetaDataProvider, VariablesSetsAggregationMetaDataProvider variablesSetsAggregationMetaDataProvider) { - this.opalService = opalService; + this.variableTaxonomiesService = variableTaxonomiesService; this.publishedStudyService = publishedStudyService; this.dtos = dtos; this.datasetAggregationMetaDataProvider = datasetAggregationMetaDataProvider; @@ -320,7 +320,7 @@ protected Properties getAggregationsProperties(List filter) { if (mode != QueryMode.LIST && filter != null && !filter.isEmpty()) { List patterns = filter.stream().map(Pattern::compile).collect(Collectors.toList()); - getOpalTaxonomies().stream().filter(Taxonomy::hasVocabularies) + getVariableTaxonomies().stream().filter(Taxonomy::hasVocabularies) .forEach(taxonomy -> taxonomy.getVocabularies().stream().filter(Vocabulary::hasTerms).forEach(vocabulary -> { String field = vocabulary.getAttributes().containsKey("field") ? vocabulary.getAttributeValue("field") @@ -330,7 +330,7 @@ protected Properties getAggregationsProperties(List filter) { properties.put(field, ""); })); - taxonomyService.getVariableTaxonomy().getVocabularies().forEach(vocabulary -> { + taxonomiesService.getVariableTaxonomy().getVocabularies().forEach(vocabulary -> { String field = vocabulary.getAttributes().containsKey("field") ? vocabulary.getAttributeValue("field") : vocabulary.getName().replace('-', '.'); @@ -357,19 +357,11 @@ protected List getAdditionalAggregationBuckets() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } @NotNull - private List getOpalTaxonomies() { - List taxonomies = null; - - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - - return taxonomies == null ? Collections.emptyList() : taxonomies; + private List getVariableTaxonomies() { + return variableTaxonomiesService.getSafeTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java index 6106a22d22..6b6d87b3a5 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java @@ -17,7 +17,7 @@ import org.obiba.mica.micaConfig.event.DeleteTaxonomiesEvent; import org.obiba.mica.micaConfig.event.OpalTaxonomiesUpdatedEvent; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.opal.core.domain.taxonomy.Taxonomy; @@ -38,7 +38,7 @@ public class TaxonomyIndexer { private static final Logger log = LoggerFactory.getLogger(TaxonomyIndexer.class); @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private Indexer indexer; @@ -51,7 +51,7 @@ public void opalTaxonomiesUpdatedEvent(OpalTaxonomiesUpdatedEvent event) { TaxonomyTarget.VARIABLE, event.extractOpalTaxonomies() .stream() - .filter(t -> taxonomyService.metaTaxonomyContains(t.getName())) + .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())) .collect(Collectors.toList())); } @@ -73,13 +73,13 @@ public void taxonomiesUpdated(TaxonomiesUpdatedEvent event) { if(indexer.hasIndex(Indexer.TERM_INDEX)) indexer.dropIndex(Indexer.TERM_INDEX); index(TaxonomyTarget.VARIABLE, - ImmutableList.builder().addAll(taxonomyService.getOpalTaxonomies().stream() // - .filter(t -> taxonomyService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) // - .add(taxonomyService.getVariableTaxonomy()) // + ImmutableList.builder().addAll(taxonomiesService.getVariableTaxonomies().stream() // + .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) // + .add(taxonomiesService.getVariableTaxonomy()) // .build()); - index(TaxonomyTarget.STUDY, Lists.newArrayList(taxonomyService.getStudyTaxonomy())); - index(TaxonomyTarget.DATASET, Lists.newArrayList(taxonomyService.getDatasetTaxonomy())); - index(TaxonomyTarget.NETWORK, Lists.newArrayList(taxonomyService.getNetworkTaxonomy())); + index(TaxonomyTarget.STUDY, Lists.newArrayList(taxonomiesService.getStudyTaxonomy())); + index(TaxonomyTarget.DATASET, Lists.newArrayList(taxonomiesService.getDatasetTaxonomy())); + index(TaxonomyTarget.NETWORK, Lists.newArrayList(taxonomiesService.getNetworkTaxonomy())); } else { indexer.delete(Indexer.TAXONOMY_INDEX, new String[] {}, ImmutablePair.of("name", event.getTaxonomyName())); Map.Entry termQuery = ImmutablePair.of("taxonomyName", event.getTaxonomyName()); @@ -92,19 +92,19 @@ public void taxonomiesUpdated(TaxonomiesUpdatedEvent event) { switch (taxonomyTarget) { case STUDY: log.info("Study taxonomies were updated"); - taxonomy = taxonomyService.getStudyTaxonomy(); + taxonomy = taxonomiesService.getStudyTaxonomy(); break; case NETWORK: log.info("Network taxonomies were updated"); - taxonomy = taxonomyService.getNetworkTaxonomy(); + taxonomy = taxonomiesService.getNetworkTaxonomy(); break; case DATASET: log.info("Dataset taxonomies were updated"); - taxonomy = taxonomyService.getDatasetTaxonomy(); + taxonomy = taxonomiesService.getDatasetTaxonomy(); break; case VARIABLE: log.info("Variable taxonomies were updated"); - taxonomy = taxonomyService.getVariableTaxonomy(); + taxonomy = taxonomiesService.getVariableTaxonomy(); break; } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java index ce436b2a61..ffe63d81d4 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java @@ -17,7 +17,7 @@ import org.obiba.mica.dataset.service.VariableSetService; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.TaxonomyNotFoundException; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.taxonomy.EsTaxonomyTermService; import org.obiba.mica.taxonomy.EsTaxonomyVocabularyService; @@ -52,7 +52,7 @@ public class AbstractTaxonomySearchResource { private EsTaxonomyVocabularyService esTaxonomyVocabularyService; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private MicaConfigService micaConfigService; @@ -127,30 +127,30 @@ protected List filterTerms(TaxonomyTarget target, String query, String l protected List getTaxonomies(TaxonomyTarget target) { switch (target) { case NETWORK: - return Lists.newArrayList(taxonomyService.getNetworkTaxonomy()); + return Lists.newArrayList(taxonomiesService.getNetworkTaxonomy()); case STUDY: - return Lists.newArrayList(taxonomyService.getStudyTaxonomy()); + return Lists.newArrayList(taxonomiesService.getStudyTaxonomy()); case DATASET: - return Lists.newArrayList(taxonomyService.getDatasetTaxonomy()); + return Lists.newArrayList(taxonomiesService.getDatasetTaxonomy()); case TAXONOMY: - return Lists.newArrayList(taxonomyService.getTaxonomyTaxonomy()); + return Lists.newArrayList(taxonomiesService.getTaxonomyTaxonomy()); default: - return taxonomyService.getVariableTaxonomies(); + return taxonomiesService.getAllVariableTaxonomies(); } } protected Taxonomy getTaxonomy(TaxonomyTarget target, String name) { switch (target) { case NETWORK: - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); case STUDY: - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); case DATASET: - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); case TAXONOMY: - return taxonomyService.getTaxonomyTaxonomy(); + return taxonomiesService.getTaxonomyTaxonomy(); default: - Taxonomy foundTaxonomy = taxonomyService.getVariableTaxonomies().stream().filter(taxonomy -> taxonomy.getName().equals(name)) + Taxonomy foundTaxonomy = taxonomiesService.getAllVariableTaxonomies().stream().filter(taxonomy -> taxonomy.getName().equals(name)) .findFirst().orElse(null); if (foundTaxonomy == null) { @@ -193,6 +193,6 @@ private List getFields(String locale, String... fieldNames) { * Populate taxonomies cache and trigger taxonomies indexing. */ private void initTaxonomies() { - taxonomyService.getOpalTaxonomies(); + taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java index f17a782b88..ec2c9d30e7 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java @@ -25,7 +25,7 @@ import org.obiba.mica.security.SubjectUtils; import org.obiba.mica.spi.search.TaxonomyTarget; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.taxonomy.TaxonomyResolver; import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.taxonomy.Dtos; @@ -45,7 +45,7 @@ public class TaxonomiesSearchResource extends AbstractTaxonomySearchResource { private static final Logger logger = LoggerFactory.getLogger(TaxonomiesSearchResource.class); @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @GET @Path("/_filter") @@ -79,7 +79,7 @@ public List search(@Context HttpServletRequest request, List results = Lists.newArrayList(); List targets = target == null - ? taxonomyService.getTaxonomyTaxonomy().getVocabularies().stream() + ? taxonomiesService.getTaxonomyTaxonomy().getVocabularies().stream() .map(t -> TaxonomyTarget.valueOf(t.getName().toUpperCase())).collect(Collectors.toList()) : Lists.newArrayList(TaxonomyTarget.valueOf(target.toUpperCase())); diff --git a/mica-web-model/src/main/protobuf/Mica.proto b/mica-web-model/src/main/protobuf/Mica.proto index 5f1a4dc319..5bb2d5bed3 100644 --- a/mica-web-model/src/main/protobuf/Mica.proto +++ b/mica-web-model/src/main/protobuf/Mica.proto @@ -409,7 +409,7 @@ message MicaConfigDto { repeated string languages = 2; required string defaultCharSet = 3; optional string publicUrl = 4; - required string opal = 5; + optional string opal = 5; optional string version = 6; optional int32 privacyThreshold = 7; repeated string roles = 8; diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java index 0e682deca0..b92c3d10a8 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java @@ -16,7 +16,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.dataset.service.PublishedDatasetService; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; @@ -49,7 +49,7 @@ public class DatasetAnalysisController extends BaseController { private Dtos dtos; @Inject - private OpalService opalService; + private VariableTaxonomiesService variableTaxonomiesService; @Inject private PublishedDatasetService publishedDatasetService; @@ -133,17 +133,11 @@ private Dataset getDataset(String id) { return dataset; } - private List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + private List getVariableTaxonomies() { + return variableTaxonomiesService.getSafeTaxonomies(); } private String getDatasetVariableJSON(DatasetVariable variable) { - return JsonFormat.printToString(dtos.asDto(variable, getTaxonomies())); + return JsonFormat.printToString(dtos.asDto(variable, getVariableTaxonomies())); } } diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java index 681e8a8c97..5fb0cc14f0 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java @@ -10,7 +10,7 @@ import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.dataset.service.HarmonizedDatasetService; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.study.NoSuchStudyException; @@ -63,7 +63,7 @@ public class VariableController extends BaseController { private ObjectMapper objectMapper; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @GetMapping("/variable/{id:.+}") public ModelAndView variable(@PathVariable String id) { @@ -99,7 +99,7 @@ public ModelAndView variable(@PathVariable String id) { addStudyTableParameters(params, variable); - Map taxonomies = taxonomyService.getVariableTaxonomies().stream() + Map taxonomies = taxonomiesService.getAllVariableTaxonomies().stream() .collect(Collectors.toMap(TaxonomyEntity::getName, e -> e)); // annotations are attributes described by some taxonomies From bd0b7ebc31d558b4b7f422025c54805ff76e339d Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 12 Dec 2022 14:40:29 +0100 Subject: [PATCH 14/44] mica-source plugin type renamed to mica-tables. Ne plugin type: mica-taxonomies for loading taxonomies provider services. --- .../StudyTableSourceServiceRegistry.java | 2 +- .../mica/core/source/ExcelTableSource.java | 6 +-- .../mica/core/source/OpalTableSource.java | 4 +- .../mica/core/source/support/OpalDtos.java | 4 +- .../core/source/support/QueryTermsUtil.java | 2 +- .../obiba/mica/dataset/domain/Dataset.java | 2 +- .../mica/dataset/domain/DatasetCategory.java | 2 +- .../mica/dataset/domain/DatasetVariable.java | 2 +- .../service/CollectedDatasetService.java | 2 +- .../mica/dataset/service/DatasetService.java | 2 +- .../service/HarmonizedDatasetService.java | 2 +- .../micaConfig/service/PluginsService.java | 41 ++++++++++++++++--- .../service/VariableTaxonomiesService.java | 33 ++++++++++++++- .../obiba/mica/study/domain/BaseStudy.java | 2 +- .../AbstractStudyTableSource.java | 2 +- .../AbstractStudyTableSourceService.java | 2 +- .../spi/{source => tables}/ICategory.java | 2 +- .../mica/spi/{source => tables}/IDataset.java | 2 +- .../mica/spi/{source => tables}/IStudy.java | 2 +- .../spi/{source => tables}/IVariable.java | 2 +- .../{source => tables}/StudyTableContext.java | 2 +- .../StudyTableFileSource.java | 2 +- .../StudyTableFileStreamProvider.java | 2 +- .../{source => tables}/StudyTableSource.java | 2 +- .../StudyTableSourceService.java | 2 +- .../StudyTableSourceServiceLoader.java | 2 +- .../taxonomies/TaxonomiesProviderService.java | 25 +++++++++++ .../TaxonomiesProviderServiceLoader.java | 28 +++++++++++++ 28 files changed, 147 insertions(+), 36 deletions(-) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/AbstractStudyTableSource.java (97%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/AbstractStudyTableSourceService.java (96%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/ICategory.java (94%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/IDataset.java (94%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/IStudy.java (95%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/IVariable.java (96%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableContext.java (95%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableFileSource.java (96%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableFileStreamProvider.java (94%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableSource.java (98%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableSourceService.java (95%) rename mica-spi/src/main/java/org/obiba/mica/spi/{source => tables}/StudyTableSourceServiceLoader.java (95%) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 0342e2fff0..4b3c87e324 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -26,7 +26,7 @@ import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.micaConfig.service.PluginsService; -import org.obiba.mica.spi.source.*; +import org.obiba.mica.spi.tables.*; import org.obiba.mica.study.domain.Study; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 0ac4b420c9..3882a6d30d 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -15,9 +15,9 @@ import org.obiba.magma.ValueTable; import org.obiba.magma.datasource.excel.ExcelDatasource; import org.obiba.magma.support.Initialisables; -import org.obiba.mica.spi.source.AbstractStudyTableSource; -import org.obiba.mica.spi.source.StudyTableFileSource; -import org.obiba.mica.spi.source.StudyTableFileStreamProvider; +import org.obiba.mica.spi.tables.AbstractStudyTableSource; +import org.obiba.mica.spi.tables.StudyTableFileSource; +import org.obiba.mica.spi.tables.StudyTableFileStreamProvider; import javax.validation.constraints.NotNull; import java.util.List; diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index 271d34ab55..ba0dc9be06 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -15,8 +15,8 @@ import org.obiba.mica.core.source.support.OpalDtos; import org.obiba.mica.core.source.support.QueryTermsUtil; import org.obiba.mica.micaConfig.service.OpalService; -import org.obiba.mica.spi.source.AbstractStudyTableSource; -import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.spi.tables.AbstractStudyTableSource; +import org.obiba.mica.spi.tables.IVariable; import org.obiba.mica.web.model.Mica; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestValueTable; diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java index d2694ed8e2..6151e910fc 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java @@ -12,8 +12,8 @@ import com.google.common.collect.Lists; import org.obiba.magma.type.BooleanType; -import org.obiba.mica.spi.source.ICategory; -import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.spi.tables.ICategory; +import org.obiba.mica.spi.tables.IVariable; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Math; import org.obiba.opal.web.model.Search; diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java index 925d700471..80fe307282 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java @@ -12,7 +12,7 @@ import com.google.common.collect.Lists; import org.obiba.magma.type.BooleanType; -import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.spi.tables.IVariable; import org.obiba.opal.web.model.Search; import java.util.List; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java index b774a5fdd9..6321183b41 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; -import org.obiba.mica.spi.source.IDataset; +import org.obiba.mica.spi.tables.IDataset; import java.util.Map; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java index e3b6dc7f31..62e796c87e 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java @@ -16,7 +16,7 @@ import org.obiba.mica.core.domain.Attribute; import org.obiba.mica.core.domain.AttributeAware; import org.obiba.mica.core.domain.Attributes; -import org.obiba.mica.spi.source.ICategory; +import org.obiba.mica.spi.tables.ICategory; public class DatasetCategory implements AttributeAware, ICategory { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index 59d8817e16..0d4d939d62 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -18,7 +18,7 @@ import org.obiba.magma.support.VariableNature; import org.obiba.mica.core.domain.*; import org.obiba.mica.spi.search.Indexable; -import org.obiba.mica.spi.source.IVariable; +import org.obiba.mica.spi.tables.IVariable; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 9fa716201c..d669a068fd 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -41,7 +41,7 @@ import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; -import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.DataCollectionEvent; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 48849e22d0..873d00495f 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -23,7 +23,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; -import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.service.StudyService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index 058d3675c9..80e5dc900b 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -37,7 +37,7 @@ import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; -import org.obiba.mica.spi.source.StudyTableSource; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.HarmonizationStudy; diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index d5c0930b9f..46a63af317 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -16,11 +16,13 @@ import com.google.common.io.Files; import org.obiba.core.util.FileUtil; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; -import org.obiba.mica.spi.source.StudyTableSourceService; -import org.obiba.mica.spi.source.StudyTableSourceServiceLoader; +import org.obiba.mica.spi.tables.StudyTableSourceService; +import org.obiba.mica.spi.tables.StudyTableSourceServiceLoader; import org.obiba.mica.spi.search.ConfigurationProvider; import org.obiba.mica.spi.search.SearchEngineService; import org.obiba.mica.spi.search.SearchEngineServiceLoader; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderServiceLoader; import org.obiba.plugins.PluginRepositoryCache; import org.obiba.plugins.PluginRepositoryException; import org.obiba.plugins.PluginResources; @@ -51,6 +53,14 @@ public class PluginsService implements EnvironmentAware { private static final String PLUGINS_PATH = "${MICA_HOME}/plugins"; + private static final String MICA_SEARCH_PLUGIN_TYPE = "mica-search"; + + private static final String MICA_TABLES_PLUGIN_TYPE = "mica-tables"; + + private static final String MICA_TAXONOMIES_PLUGIN_TYPE = "mica-taxonomies"; + + + private static final String MICA_SEARCH_PLUGIN_NAME = "plugins.micaSearchPlugin"; private static final String DEFAULT_MICA_SEARCH_PLUGIN_NAME = "mica-search-es"; @@ -94,6 +104,12 @@ public Collection getStudyTableSourceServices() { .collect(Collectors.toList()); } + public Collection getTaxonomiesProviderServices() { + return getServicePlugins(TaxonomiesProviderService.class).stream() + .map(service -> (TaxonomiesProviderService)service) + .collect(Collectors.toList()); + } + // // Private methods // @@ -123,7 +139,7 @@ private void initPlugins() { try { String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(searchPluginName); // ensure there is a mica-search plugin installed - if (plugins.stream().noneMatch(p -> "mica-search".equals(p.getType())) + if (plugins.stream().noneMatch(p -> MICA_SEARCH_PLUGIN_TYPE.equals(p.getType())) || plugins.stream() .filter(plugin -> searchPluginName.equals(plugin.getName())) .filter(plugin -> plugin.getVersion().compareTo(new Version(pluginLatestVersion)) >= 0).count() == 0) { @@ -137,16 +153,20 @@ private void initPlugins() { boolean micaSearchFound = false; // mica-search plugin is a singleton List filteredPlugins = - plugins.stream().filter(plugin -> searchPluginName.equals(plugin.getName()) || "mica-source".equals(plugin.getType())) + plugins.stream().filter(plugin -> searchPluginName.equals(plugin.getName()) + || MICA_TABLES_PLUGIN_TYPE.equals(plugin.getType()) + || MICA_TAXONOMIES_PLUGIN_TYPE.equals(plugin.getType())) .sorted(Comparator.comparing(PluginResources::getVersion)) .collect(Collectors.toList()); for (PluginResources plugin : filteredPlugins) { - if ("mica-search".equals(plugin.getType()) && !micaSearchFound) { + if (MICA_SEARCH_PLUGIN_TYPE.equals(plugin.getType()) && !micaSearchFound) { initSearchEngineServicePlugin(plugin); micaSearchFound = true; - } else if ("mica-source".equals(plugin.getType())) { + } else if (MICA_TABLES_PLUGIN_TYPE.equals(plugin.getType())) { initStudyTableSourceServicePlugin(plugin); + } else if (MICA_TAXONOMIES_PLUGIN_TYPE.equals(plugin.getType())) { + initTaxonomiesProviderServicePlugin(plugin); } } } @@ -173,6 +193,15 @@ private void initStudyTableSourceServicePlugin(PluginResources plugin) { }); } + private void initTaxonomiesProviderServicePlugin(PluginResources plugin) { + TaxonomiesProviderServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { + Properties properties = cleanProperties(plugin.getProperties()); + service.configure(properties); + service.start(); + servicePlugins.add(service); + }); + } + private Properties cleanProperties(Properties properties) { properties.setProperty("MICA_HOME", properties.getProperty("OPAL_HOME")); properties.remove("OPAL_HOME"); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index d6b1ed88b7..58e0b53952 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -12,6 +12,9 @@ import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; import org.obiba.opal.core.cfg.NoSuchTaxonomyException; import org.obiba.opal.core.cfg.NoSuchVocabularyException; import org.obiba.opal.core.domain.taxonomy.Taxonomy; @@ -19,6 +22,8 @@ import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.taxonomy.Dtos; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -35,11 +40,35 @@ @Component public class VariableTaxonomiesService { + private static final Logger log = LoggerFactory.getLogger(VariableTaxonomiesService.class); + @Inject private OpalService opalService; + @Inject + private PluginsService pluginsService; + public List getTaxonomies() { - Map taxonomies = opalService.getTaxonomiesInternal(); + Map taxonomies; + try { + taxonomies = opalService.getTaxonomiesInternal(); + } catch (Exception e) { + taxonomies = Maps.newHashMap(); + } + for (TaxonomiesProviderService provider : pluginsService.getTaxonomiesProviderServices().stream() + .filter(provider -> provider.isFor(TaxonomyTarget.VARIABLE)) + .collect(Collectors.toList())) { + try { + for (Taxonomy taxonomy : provider.getTaxonomies()) { + // override any duplicated taxonomy + if (taxonomies.containsKey(taxonomy.getName())) + log.warn("Taxonomy with name {} is duplicated ({} plugin)", provider.getName(), taxonomy.getName()); + taxonomies.put(taxonomy.getName(), taxonomy); + } + } catch (Exception e) { + log.warn("Taxonomies retrieval from plugin {} failed", provider.getName(), e); + } + } List taxonomyList = Lists.newArrayList(taxonomies.values()); Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); return taxonomyList; @@ -56,7 +85,7 @@ public List getSafeTaxonomies() { try { taxonomies = getTaxonomies(); - } catch(Exception e) { + } catch (Exception e) { // ignore } diff --git a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java index 878bea61e6..5b8946975f 100644 --- a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java +++ b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java @@ -16,7 +16,7 @@ import org.obiba.mica.core.domain.*; import org.obiba.mica.file.Attachment; import org.obiba.mica.spi.search.Indexable; -import org.obiba.mica.spi.source.IStudy; +import org.obiba.mica.spi.tables.IStudy; import javax.validation.constraints.NotNull; import java.beans.Transient; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java similarity index 97% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java index ffcb50df13..dbeaba2208 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import org.obiba.mica.web.model.Mica; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java similarity index 96% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java index d28260528e..6cefa5992c 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/AbstractStudyTableSourceService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import java.util.Properties; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java similarity index 94% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java index bdced82663..3d3cce7efb 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/ICategory.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; /** * Category description, for summary statistics. diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java similarity index 94% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java index 52a1195afb..1ad987469a 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/IDataset.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import org.obiba.mica.spi.search.Identified; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java similarity index 95% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java index 0d7de882ae..4589de81dd 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/IStudy.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import org.obiba.mica.spi.search.Identified; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java similarity index 96% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java index cf5f690dee..c3d4280742 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/IVariable.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import java.util.List; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java similarity index 95% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java index dbde7e9793..d8fc3fa841 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableContext.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; public class StudyTableContext { diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java similarity index 96% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java index 3acafcb42e..a3d4703d5b 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; /** * Study table is to be extracted from a file which path applies to the Mica's internal file system. diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java similarity index 94% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java index c84477b7ec..e8321d73d1 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableFileStreamProvider.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import java.io.InputStream; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java similarity index 98% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java index e02bf60f30..3e29028796 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import org.obiba.magma.ValueTable; import org.obiba.mica.web.model.Mica; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java similarity index 95% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java index c28b2b008d..f775c248c9 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import org.obiba.plugins.spi.ServicePlugin; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java similarity index 95% rename from mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java rename to mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java index 865ab098fd..e4a358ec62 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/source/StudyTableSourceServiceLoader.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java @@ -8,7 +8,7 @@ * along with this program. If not, see . */ -package org.obiba.mica.spi.source; +package org.obiba.mica.spi.tables; import com.google.common.collect.Lists; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java new file mode 100644 index 0000000000..f75ce0450a --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.plugins.spi.ServicePlugin; + +import java.util.List; + +public interface TaxonomiesProviderService extends ServicePlugin { + + boolean isFor(TaxonomyTarget target); + + List getTaxonomies(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java new file mode 100644 index 0000000000..3766940303 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import com.google.common.collect.Lists; + +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.ServiceLoader; + +/** + * {@link TaxonomiesProviderService} loader. + */ +public class TaxonomiesProviderServiceLoader { + + public static Collection get(URLClassLoader classLoader) { + return Lists.newArrayList(ServiceLoader.load(TaxonomiesProviderService.class, classLoader).iterator()); + } + +} From 3b73a3281d7122063d35c0f346e236f3f9c29fa8 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Tue, 13 Dec 2022 09:57:39 +0100 Subject: [PATCH 15/44] Taxonomy of taxonomies is configurable. Added attribute to have a taxonomy declared but not visible. --- .../micaConfig/service/MicaConfigService.java | 29 ----- .../micaConfig/service/TaxonomiesService.java | 101 +++++++++++------- .../service/TaxonomyConfigService.java | 2 - .../rest/AbstractTaxonomySearchResource.java | 65 +++++------ .../rest/TaxonomiesSearchResource.java | 16 +-- 5 files changed, 104 insertions(+), 109 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java index 3f0af22c13..7e0d09706b 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java @@ -94,35 +94,6 @@ public ObjectMapper getObjectMapper() { return objectMapper; } - public Taxonomy getTaxonomy(TaxonomyTarget target) { - return taxonomyConfigService.findByTarget(target); - } - - @NotNull - Taxonomy getNetworkTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.NETWORK); - } - - @NotNull - Taxonomy getStudyTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.STUDY); - } - - @NotNull - Taxonomy getDatasetTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.DATASET); - } - - @NotNull - Taxonomy getVariableTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.VARIABLE); - } - - @NotNull - Taxonomy getTaxonomyTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.TAXONOMY); - } - @Cacheable(value = "micaConfig", key = "#root.methodName") public MicaConfig getConfig() { return getOrCreateMicaConfig(); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index 66408aae1a..735712d717 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -10,13 +10,8 @@ package org.obiba.mica.micaConfig.service; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.inject.Inject; -import javax.validation.constraints.NotNull; - +import com.google.common.base.Strings; +import com.google.common.eventbus.Subscribe; import org.obiba.mica.core.event.DocumentSetUpdatedEvent; import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; @@ -24,6 +19,7 @@ import org.obiba.mica.micaConfig.service.helper.*; import org.obiba.mica.network.event.NetworkPublishedEvent; import org.obiba.mica.network.event.NetworkUnpublishedEvent; +import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.study.event.StudyPublishedEvent; import org.obiba.mica.study.event.StudyUnpublishedEvent; import org.obiba.opal.core.domain.taxonomy.Taxonomy; @@ -33,7 +29,12 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import com.google.common.eventbus.Subscribe; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Service public class TaxonomiesService { @@ -58,6 +59,8 @@ public class TaxonomiesService { private final VariablesSetsAggregationMetaDataHelper variablesSetsHelper; + private final TaxonomyConfigService taxonomyConfigService; + private Taxonomy taxonomyTaxonomy; private Taxonomy variableTaxonomy; @@ -79,7 +82,8 @@ public TaxonomiesService( DceIdAggregationMetaDataHelper dceHelper, NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper, StudiesSetsAggregationMetaDataHelper studiesSetsHelper, - VariablesSetsAggregationMetaDataHelper variablesSetsHelper) { + VariablesSetsAggregationMetaDataHelper variablesSetsHelper, + TaxonomyConfigService taxonomyConfigService) { this.variableTaxonomiesService = variableTaxonomiesService; this.micaConfigService = micaConfigService; this.studyHelper = studyHelper; @@ -90,6 +94,7 @@ public TaxonomiesService( this.networksSetsAggregationMetaDataHelper = networksSetsAggregationMetaDataHelper; this.studiesSetsHelper = studiesSetsHelper; this.variablesSetsHelper = variablesSetsHelper; + this.taxonomyConfigService = taxonomyConfigService; } @NotNull @@ -99,8 +104,20 @@ public Taxonomy getTaxonomyTaxonomy() { } public boolean metaTaxonomyContains(String taxonomy) { - for(Vocabulary target : getTaxonomyTaxonomy().getVocabularies()) { - if(hasTerm(target, taxonomy)) return true; + for (Vocabulary targetVocabulary : getTaxonomyTaxonomy().getVocabularies()) { + Optional termOpt = getTerm(targetVocabulary, taxonomy); + if (termOpt.isPresent()) { + Term term = termOpt.get(); + String visibility = term.getAttributeValue("visible"); + // visible by default + if (Strings.isNullOrEmpty(visibility)) return true; + // check visible attribute value + try { + return Boolean.parseBoolean(visibility.toLowerCase()); + } catch (Exception e) { + return false; + } + } } return false; } @@ -193,45 +210,45 @@ private synchronized void initialize() { } private void initializeTaxonomyTaxonomy() { - if(taxonomyTaxonomy != null) return; - taxonomyTaxonomy = copy(micaConfigService.getTaxonomyTaxonomy()); + if (taxonomyTaxonomy != null) return; + taxonomyTaxonomy = copy(findTaxonomy(TaxonomyTarget.TAXONOMY)); MicaConfig config = micaConfigService.getConfig(); - if(!config.isNetworkEnabled() || config.isSingleNetworkEnabled()) { + if (!config.isNetworkEnabled() || config.isSingleNetworkEnabled()) { taxonomyTaxonomy.removeVocabulary("network"); } - if(!config.isStudyDatasetEnabled() && !config.isHarmonizationDatasetEnabled()) { + if (!config.isStudyDatasetEnabled() && !config.isHarmonizationDatasetEnabled()) { taxonomyTaxonomy.removeVocabulary("dataset"); taxonomyTaxonomy.removeVocabulary("variable"); } - if(config.isSingleStudyEnabled() && !config.isHarmonizationDatasetEnabled()) { + if (config.isSingleStudyEnabled() && !config.isHarmonizationDatasetEnabled()) { taxonomyTaxonomy.removeVocabulary("study"); } } private void initializeNetworkTaxonomy() { - if(networkTaxonomy != null) return; - networkTaxonomy = copy(micaConfigService.getNetworkTaxonomy()); + if (networkTaxonomy != null) return; + networkTaxonomy = copy(findTaxonomy(TaxonomyTarget.NETWORK)); networkHelper.applyIdTerms(networkTaxonomy, "id"); networksSetsAggregationMetaDataHelper.applyIdTerms(networkTaxonomy, "sets"); studyHelper.applyIdTerms(networkTaxonomy, "studyIds"); } private void initializeStudyTaxonomy() { - if(studyTaxonomy != null) return; - studyTaxonomy = copy(micaConfigService.getStudyTaxonomy()); + if (studyTaxonomy != null) return; + studyTaxonomy = copy(findTaxonomy(TaxonomyTarget.STUDY)); studyHelper.applyIdTerms(studyTaxonomy, "id"); studiesSetsHelper.applyIdTerms(studyTaxonomy, "sets"); } private void initializeDatasetTaxonomy() { - if(datasetTaxonomy != null) return; - datasetTaxonomy = copy(micaConfigService.getDatasetTaxonomy()); + if (datasetTaxonomy != null) return; + datasetTaxonomy = copy(findTaxonomy(TaxonomyTarget.STUDY)); datasetHelper.applyIdTerms(datasetTaxonomy, "id"); } private void initializeVariableTaxonomy() { - if(variableTaxonomy != null) return; - variableTaxonomy = copy(micaConfigService.getVariableTaxonomy()); + if (variableTaxonomy != null) return; + variableTaxonomy = copy(findTaxonomy(TaxonomyTarget.VARIABLE)); studyHelper.applyIdTerms(variableTaxonomy, "studyId"); datasetHelper.applyIdTerms(variableTaxonomy, "datasetId"); populationHelper.applyIdTerms(variableTaxonomy, "populationId"); @@ -242,11 +259,11 @@ private void initializeVariableTaxonomy() { private Taxonomy copy(Taxonomy source) { Taxonomy target = new Taxonomy(); BeanUtils.copyProperties(source, target, "vocabularies"); - if(source.hasVocabularies()) { + if (source.hasVocabularies()) { source.getVocabularies().forEach(sourceVoc -> { Vocabulary targetVoc = new Vocabulary(); BeanUtils.copyProperties(sourceVoc, targetVoc, "terms"); - if(sourceVoc.hasTerms()) { + if (sourceVoc.hasTerms()) { sourceVoc.getTerms().forEach(sourceTerm -> { Term targetTerm = new Term(); BeanUtils.copyProperties(sourceTerm, targetTerm); @@ -267,29 +284,35 @@ private Taxonomy copy(Taxonomy source) { * @param name * @return */ - private boolean hasTerm(Vocabulary vocabulary, String name) { - if(!vocabulary.hasTerms()) return false; - if(vocabulary.hasTerm(name)) return true; - for(Term t : vocabulary.getTerms()) { - if(hasTerm(t, name)) return true; + private Optional getTerm(Vocabulary vocabulary, String name) { + if (!vocabulary.hasTerms()) return Optional.empty(); + if (vocabulary.hasTerm(name)) return Optional.of(vocabulary.getTerm(name)); + for (Term t : vocabulary.getTerms()) { + Optional res = getTerm(t, name); + if (res.isPresent()) return res; } - return false; + return Optional.empty(); } /** - * Check if term has a term with the givane name. + * Check if term has a term with the given name. * * @param term * @param name * @return */ - private boolean hasTerm(Term term, String name) { - if(!term.hasTerms()) return false; - if(term.hasTerm(name)) return true; - for(Term t : term.getTerms()) { - if(hasTerm(t, name)) return true; + private Optional getTerm(Term term, String name) { + if (!term.hasTerms()) return Optional.empty(); + if (term.hasTerm(name)) return Optional.of(term.getTerm(name)); + for (Term t : term.getTerms()) { + Optional res = getTerm(t, name); + if (res.isPresent()) return res; } - return false; + return Optional.empty(); + } + + private Taxonomy findTaxonomy(TaxonomyTarget target) { + return taxonomyConfigService.findByTarget(target); } // diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java index 9476c2b5a2..d3936ee1c3 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java @@ -79,8 +79,6 @@ private Taxonomy readTaxonomyFromYaml(String yamlResourcePath) { } private Taxonomy findByTargetInternal(TaxonomyTarget target) { - // taxonomy of taxonomies is not editable so fall back to the one that comes from the classpath - if(TaxonomyTarget.TAXONOMY.equals(target)) return defaultTaxonomyTaxonomy; String id = target.asId(); Optional taxonomyEntityWrapper = taxonomyConfigRepository.findById(id); diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java index ffe63d81d4..eaf94bcd29 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java @@ -63,40 +63,41 @@ public class AbstractTaxonomySearchResource { protected void populate(Opal.TaxonomyDto.Builder tBuilder, Taxonomy taxonomy, Map>> taxoNamesMap, String anonymousUserId) { - taxonomy.getVocabularies().stream().filter(v -> taxoNamesMap.get(taxonomy.getName()).containsKey(v.getName())) - .forEach(voc -> { - Opal.VocabularyDto.Builder vBuilder = Dtos.asDto(voc, false).toBuilder(); - List termNames = taxoNamesMap.get(taxonomy.getName()).get(voc.getName()); - if (voc.hasTerms()) { - List terms = voc.getTerms(); - - // for now, sets are available for variable documents only - if (taxonomy.getName().equals("Mica_" + TaxonomyTarget.VARIABLE.asId()) && voc.getName().equals("sets")) { - List sets = SecurityUtils.getSubject().isAuthenticated() ? - variableSetService.getAllCurrentUser() : - variableSetService.getAllAnonymousUser(anonymousUserId); - List allSetsCurrentUser = sets.stream().map(DocumentSet::getId).collect(Collectors.toList()); - - if (allSetsCurrentUser.size() > 0) { - terms = voc.getTerms().stream().filter(term -> allSetsCurrentUser.contains(term.getName())).collect(Collectors.toList()); - } else { - terms = Lists.newArrayList(); - } + taxonomy.getVocabularies().stream() + .filter(v -> taxoNamesMap.get(taxonomy.getName()).containsKey(v.getName())) + .forEach(voc -> { + Opal.VocabularyDto.Builder vBuilder = Dtos.asDto(voc, false).toBuilder(); + List termNames = taxoNamesMap.get(taxonomy.getName()).get(voc.getName()); + if (voc.hasTerms()) { + List terms = voc.getTerms(); + + // for now, sets are available for variable documents only + if (taxonomy.getName().equals("Mica_" + TaxonomyTarget.VARIABLE.asId()) && voc.getName().equals("sets")) { + List sets = SecurityUtils.getSubject().isAuthenticated() ? + variableSetService.getAllCurrentUser() : + variableSetService.getAllAnonymousUser(anonymousUserId); + List allSetsCurrentUser = sets.stream().map(DocumentSet::getId).collect(Collectors.toList()); + + if (allSetsCurrentUser.size() > 0) { + terms = voc.getTerms().stream().filter(term -> allSetsCurrentUser.contains(term.getName())).collect(Collectors.toList()); + } else { + terms = Lists.newArrayList(); } - - if (termNames.isEmpty()) - vBuilder.addAllTerms(terms.stream().map(Dtos::asDto).collect(Collectors.toList())); - else terms.stream().filter(t -> termNames.contains(t.getName())) - .forEach(term -> vBuilder.addTerms(Dtos.asDto(term))); } - tBuilder.addVocabularies(vBuilder); - }); + + if (termNames.isEmpty()) + vBuilder.addAllTerms(terms.stream().map(Dtos::asDto).collect(Collectors.toList())); + else terms.stream().filter(t -> termNames.contains(t.getName())) + .forEach(term -> vBuilder.addTerms(Dtos.asDto(term))); + } + tBuilder.addVocabularies(vBuilder); + }); } protected List filterVocabularies(TaxonomyTarget target, String query, String locale) { try { return esTaxonomyVocabularyService.find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), - getFields(locale, VOCABULARY_FIELDS), null, null).getList(); + getFields(locale, VOCABULARY_FIELDS), null, null).getList(); } catch (Exception e) { initTaxonomies(); // for a 404 response @@ -109,14 +110,14 @@ protected List filterTerms(TaxonomyTarget target, String query, String l if (vocabularies != null && vocabularies.size() > 0) { // filter on vocabulary names; remove taxonomy prefixes ('Mica_study:') String vocabulariesQuery = vocabularies.stream() - .map(v -> String.format("vocabularyName:%s", v.replaceAll("^([^\\:]+):", ""))) - .collect(Collectors.joining(" AND ")); + .map(v -> String.format("vocabularyName:%s", v.replaceAll("^([^\\:]+):", ""))) + .collect(Collectors.joining(" AND ")); query = Strings.isNullOrEmpty(query) ? vocabulariesQuery : query + " " + vocabulariesQuery; } return esTaxonomyTermService - .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), getFields(locale, TERM_FIELDS)) - .getList(); + .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), getFields(locale, TERM_FIELDS)) + .getList(); } catch (Exception e) { initTaxonomies(); // for a 404 response @@ -151,7 +152,7 @@ protected Taxonomy getTaxonomy(TaxonomyTarget target, String name) { return taxonomiesService.getTaxonomyTaxonomy(); default: Taxonomy foundTaxonomy = taxonomiesService.getAllVariableTaxonomies().stream().filter(taxonomy -> taxonomy.getName().equals(name)) - .findFirst().orElse(null); + .findFirst().orElse(null); if (foundTaxonomy == null) { throw new TaxonomyNotFoundException(name); diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java index ec2c9d30e7..f57c8d1e72 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java @@ -59,20 +59,22 @@ public List filter(@Context HttpServletRequest request, @Query Map>> taxoNamesMap = TaxonomyResolver .asMap(filteredVocabularies, filterTerms(taxonomyTarget, query, locale, filteredVocabularies)); List results = Lists.newArrayList(); - getTaxonomies(taxonomyTarget).stream().filter(t -> taxoNamesMap.containsKey(t.getName())).forEach(taxo -> { - Opal.TaxonomyDto.Builder tBuilder = Dtos.asDto(taxo, false).toBuilder(); - populate(tBuilder, taxo, taxoNamesMap, SubjectUtils.getAnonymousUserId(request)); - results.add(tBuilder.build()); - }); + getTaxonomies(taxonomyTarget).stream() + .filter(t -> taxoNamesMap.containsKey(t.getName())) + .forEach(taxo -> { + Opal.TaxonomyDto.Builder tBuilder = Dtos.asDto(taxo, false).toBuilder(); + populate(tBuilder, taxo, taxoNamesMap, SubjectUtils.getAnonymousUserId(request)); + results.add(tBuilder.build()); + }); - return results; + return results; } @GET @Path("/_search") @Timed public List search(@Context HttpServletRequest request, @QueryParam("query") String query, @QueryParam("locale") String locale, - @Nullable @QueryParam("target") String target) { + @Nullable @QueryParam("target") String target) { logger.debug("TaxonomiesSearchResource#search called with query [%s], locale [%s] and target [%s]", query, locale, target); From f4b546eab829747648fc819dcfb68eb33973ff80 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Tue, 13 Dec 2022 10:41:26 +0100 Subject: [PATCH 16/44] Variable taxonomies can be read from local folder: MICA_HOME/conf/taxonomies/variable --- .../service/DataAccessEntityService.java | 4 +-- ...rceReader.java => YamlResourceReader.java} | 12 ++++++-- .../service/TaxonomyConfigService.java | 4 +-- .../service/VariableTaxonomiesService.java | 30 +++++++++++++++++++ ...rTest.java => YamlResourceReaderTest.java} | 6 ++-- 5 files changed, 47 insertions(+), 9 deletions(-) rename mica-core/src/main/java/org/obiba/mica/core/support/{YamlClassPathResourceReader.java => YamlResourceReader.java} (53%) rename mica-core/src/test/java/org/obiba/mica/core/support/{YamlClassPathResourceReaderTest.java => YamlResourceReaderTest.java} (80%) diff --git a/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java b/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java index 87f901ecd4..c84d115293 100644 --- a/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java +++ b/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java @@ -47,7 +47,7 @@ import org.obiba.mica.core.service.MailService; import org.obiba.mica.core.service.SchemaFormContentFileService; import org.obiba.mica.core.support.IdentifierGenerator; -import org.obiba.mica.core.support.YamlClassPathResourceReader; +import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.dataset.service.VariableSetService; import org.obiba.mica.micaConfig.domain.DataAccessConfig; import org.obiba.mica.micaConfig.service.DataAccessConfigService; @@ -396,7 +396,7 @@ protected void setAndLogStatus(T request, DataAccessEntityStatus to) { protected String generateId() { DataAccessConfig dataAccessConfig = dataAccessConfigService.getOrCreateConfig(); - Object exclusions = YamlClassPathResourceReader.read(EXCLUSION_IDS_YAML_RESOURCE_PATH, Map.class).get("exclusions"); + Object exclusions = YamlResourceReader.readClassPath(EXCLUSION_IDS_YAML_RESOURCE_PATH, Map.class).get("exclusions"); IdentifierGenerator.Builder builder = IdentifierGenerator.newBuilder().prefix(dataAccessConfig.getIdPrefix()) .size(dataAccessConfig.getIdLength()); diff --git a/mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java b/mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java similarity index 53% rename from mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java rename to mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java index 3d61ea3661..f3f1fac398 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java +++ b/mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java @@ -4,8 +4,9 @@ import org.springframework.core.io.ClassPathResource; import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.core.io.FileSystemResource; -public class YamlClassPathResourceReader { +public class YamlResourceReader { private static ObjectMapper mapper; @@ -13,10 +14,17 @@ public class YamlClassPathResourceReader { mapper = new ObjectMapper(); } - public static T read(String resourcePath, Class resultClass) { + public static T readClassPath(String resourcePath, Class resultClass) { YamlMapFactoryBean factory = new YamlMapFactoryBean(); factory.setResources(new ClassPathResource(resourcePath)); return mapper.convertValue(factory.getObject(), resultClass); } + + public static T readFile(String resourcePath, Class resultClass) { + YamlMapFactoryBean factory = new YamlMapFactoryBean(); + factory.setResources(new FileSystemResource(resourcePath)); + + return mapper.convertValue(factory.getObject(), resultClass); + } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java index d3936ee1c3..1f7bb1c259 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java @@ -17,7 +17,7 @@ import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.TaxonomyEntityWrapper; -import org.obiba.mica.core.support.YamlClassPathResourceReader; +import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; import org.obiba.mica.micaConfig.repository.TaxonomyConfigRepository; import org.obiba.mica.spi.search.TaxonomyTarget; @@ -75,7 +75,7 @@ public void mergeWithDefault(TaxonomyTarget target) { } private Taxonomy readTaxonomyFromYaml(String yamlResourcePath) { - return YamlClassPathResourceReader.read(yamlResourcePath, Taxonomy.class); + return YamlResourceReader.readClassPath(yamlResourcePath, Taxonomy.class); } private Taxonomy findByTargetInternal(TaxonomyTarget target) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index 58e0b53952..465fc80115 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -13,6 +13,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; import org.obiba.opal.core.cfg.NoSuchTaxonomyException; @@ -26,8 +27,10 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.validation.constraints.NotNull; +import java.io.File; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -42,19 +45,46 @@ public class VariableTaxonomiesService { private static final Logger log = LoggerFactory.getLogger(VariableTaxonomiesService.class); + private static final String VARIABLE_TAXONOMIES_PATH = "${MICA_HOME}/conf/taxonomies/variable"; + @Inject private OpalService opalService; @Inject private PluginsService pluginsService; + private File variableTaxonomiesDir; + + @PostConstruct + public void init() { + if (variableTaxonomiesDir == null) { + variableTaxonomiesDir = new File(VARIABLE_TAXONOMIES_PATH.replace("${MICA_HOME}", System.getProperty("MICA_HOME"))); + } + } + public List getTaxonomies() { Map taxonomies; + // init with the ones from opal try { taxonomies = opalService.getTaxonomiesInternal(); } catch (Exception e) { taxonomies = Maps.newHashMap(); } + // read local files + if (variableTaxonomiesDir.exists()) { + File[] yamlFiles = variableTaxonomiesDir.listFiles(file -> !file.isDirectory() && file.getName().endsWith(".yml")); + if (yamlFiles != null) { + for (File yamlFile : yamlFiles) { + try { + Taxonomy taxonomy = YamlResourceReader.readFile(yamlFile.getAbsolutePath(), Taxonomy.class); + taxonomies.put(taxonomy.getName(), taxonomy); + } catch (Exception e) { + log.error("Taxonomy file could not be read: {}", yamlFile.getAbsolutePath(), e); + } + } + } + } + // get the ones from plugins for (TaxonomiesProviderService provider : pluginsService.getTaxonomiesProviderServices().stream() .filter(provider -> provider.isFor(TaxonomyTarget.VARIABLE)) .collect(Collectors.toList())) { diff --git a/mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java b/mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java similarity index 80% rename from mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java rename to mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java index 7befaf8137..d76c96076b 100644 --- a/mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java +++ b/mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java @@ -11,11 +11,11 @@ import static org.junit.Assert.assertTrue; @RunWith(JUnit4.class) -public class YamlClassPathResourceReaderTest { +public class YamlResourceReaderTest { @Test public void readEmptyExclusionsListYamlTest() { - Map read = YamlClassPathResourceReader.read("/empty-exclusions-list.yml", Map.class); + Map read = YamlResourceReader.readClassPath("/empty-exclusions-list.yml", Map.class); assertTrue(read.containsKey("exclusions")); assertTrue(((List) read.get("exclusions")).isEmpty()); @@ -23,7 +23,7 @@ public void readEmptyExclusionsListYamlTest() { @Test public void readExclusionsListYamlTest() { - Map read = YamlClassPathResourceReader.read("/exclusions-list.yml", Map.class); + Map read = YamlResourceReader.readClassPath("/exclusions-list.yml", Map.class); assertTrue(read.containsKey("exclusions")); assertNotNull(read.get("exclusions")); From 1a422941cecc0950eec73d6ef42a7de81a38c0eb Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Tue, 13 Dec 2022 11:24:37 +0100 Subject: [PATCH 17/44] AbstractTaxonomiesProviderService added --- .../AbstractTaxonomiesProviderService.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java new file mode 100644 index 0000000000..44c55b9c29 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import java.util.Properties; + +public abstract class AbstractTaxonomiesProviderService implements TaxonomiesProviderService { + + protected Properties properties; + + protected boolean running; + + @Override + public Properties getProperties() { + return properties; + } + + @Override + public void configure(Properties properties) { + this.properties = properties; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } +} From 1af18b2565f1c6e08904353850b3c5d23069e4bf Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Tue, 13 Dec 2022 13:46:21 +0100 Subject: [PATCH 18/44] Caching of taxonomies refactored, not functional yet --- .../org/obiba/mica/micaConfig/service/CacheService.java | 8 +++++++- .../micaConfig/service/VariableTaxonomiesService.java | 8 ++++++++ mica-core/src/main/resources/ehcache.xml | 8 ++++++++ .../main/java/org/obiba/mica/web/rest/CacheResource.java | 7 ++++--- .../mica/web/rest/IllegalArgumentExceptionMapper.java | 2 +- mica-webapp/src/main/resources/i18n/en.json | 4 ++-- mica-webapp/src/main/resources/i18n/fr.json | 4 ++-- mica-webapp/src/main/webapp/app/admin/admin-controller.js | 4 ++-- mica-webapp/src/main/webapp/app/admin/views/caching.html | 6 +++--- 9 files changed, 37 insertions(+), 14 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index 2cbc639898..a210660d14 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -51,6 +51,12 @@ public void clearOpalTaxonomiesCache() { taxonomiesService.getVariableTaxonomies(); } + @CacheEvict(value = "variable-taxonomies", allEntries = true, beforeInvocation = true) + public void clearTaxonomiesCache() { + log.info("Clearing variable taxonomies cache"); + clearOpalTaxonomiesCache(); + } + @CacheEvict(value = "micaConfig", allEntries = true) public void clearMicaConfigCache() { log.info("Clearing mica config cache"); @@ -77,7 +83,7 @@ public void buildDatasetVariablesCache() { public void clearAllCaches() { log.info("Clearing all caches"); - clearOpalTaxonomiesCache(); + clearTaxonomiesCache(); clearMicaConfigCache(); clearAggregationsMetadataCache(); clearDatasetVariablesCache(); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index 465fc80115..64b2e68b9b 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -25,6 +25,7 @@ import org.obiba.opal.web.taxonomy.Dtos; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -62,7 +63,12 @@ public void init() { } } + @Cacheable(value = "variable-taxonomies", key = "'variable'") public List getTaxonomies() { + return getInternalTaxonomies(); + } + + public List getInternalTaxonomies() { Map taxonomies; // init with the ones from opal try { @@ -74,6 +80,7 @@ public List getTaxonomies() { if (variableTaxonomiesDir.exists()) { File[] yamlFiles = variableTaxonomiesDir.listFiles(file -> !file.isDirectory() && file.getName().endsWith(".yml")); if (yamlFiles != null) { + log.info("Fetching local taxonomies"); for (File yamlFile : yamlFiles) { try { Taxonomy taxonomy = YamlResourceReader.readFile(yamlFile.getAbsolutePath(), Taxonomy.class); @@ -88,6 +95,7 @@ public List getTaxonomies() { for (TaxonomiesProviderService provider : pluginsService.getTaxonomiesProviderServices().stream() .filter(provider -> provider.isFor(TaxonomyTarget.VARIABLE)) .collect(Collectors.toList())) { + log.info("Fetching taxonomies from plugin: {}", provider.getName()); try { for (Taxonomy taxonomy : provider.getTaxonomies()) { // override any duplicated taxonomy diff --git a/mica-core/src/main/resources/ehcache.xml b/mica-core/src/main/resources/ehcache.xml index 119df23ac4..460f2c5d08 100644 --- a/mica-core/src/main/resources/ehcache.xml +++ b/mica-core/src/main/resources/ehcache.xml @@ -72,6 +72,14 @@ + + + + diff --git a/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java b/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java index da97d065bc..00b98da417 100644 --- a/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java @@ -27,7 +27,7 @@ @RequiresRoles(Roles.MICA_ADMIN) public class CacheResource { - private static final Logger logger = LoggerFactory.getLogger(IllegalArgumentExceptionMapper.class); + private static final Logger logger = LoggerFactory.getLogger(CacheResource.class); @Inject private CacheService cacheService; @@ -43,14 +43,15 @@ public Response deleteCaches() { @DELETE public Response deleteCache(@PathParam("id") String id) { - logger.info("clear cache [{}]", id); + logger.info("Clear cache [{}]", id); switch(id) { case "micaConfig" : cacheService.clearMicaConfigCache(); break; case "opalTaxonomies": - cacheService.clearOpalTaxonomiesCache(); + case "variableTaxonomies": + cacheService.clearTaxonomiesCache(); break; case "aggregationsMetadata": cacheService.clearAggregationsMetadataCache(); diff --git a/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java index 2fbdb9ed25..1990efe7ef 100644 --- a/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java @@ -24,7 +24,7 @@ public class IllegalArgumentExceptionMapper implements ExceptionMapper - caching.opal-taxonomies - caching.opal-taxonomies-help + caching.variable-taxonomies + caching.variable-taxonomies-help - + From e846e96aba5a205fbac78bed67cf7bb502769493 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 15 Dec 2022 12:51:06 +0100 Subject: [PATCH 19/44] Variable taxonomies are cached --- .../mica/micaConfig/service/OpalService.java | 36 ++-- .../micaConfig/service/TaxonomiesService.java | 15 +- .../service/VariableTaxonomiesService.java | 154 ++---------------- .../micaConfig/rest/MicaConfigResource.java | 38 ----- .../rest/AbstractTaxonomyResource.java | 139 ++++++++++++++++ .../taxonomy/rest/TaxonomiesResource.java | 12 +- .../mica/taxonomy/rest/TaxonomyResource.java | 17 +- .../AbstractPublishedDatasetResource.java | 11 +- .../mica/search/CoverageQueryExecutor.java | 12 +- .../TaxonomyAggregationMetaDataProvider.java | 6 +- .../mica/search/queries/VariableQuery.java | 6 +- .../controller/DatasetAnalysisController.java | 25 ++- 12 files changed, 216 insertions(+), 255 deletions(-) create mode 100644 mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java index 20e36d3002..755aab9d9b 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java @@ -11,6 +11,7 @@ package org.obiba.mica.micaConfig.service; import com.google.common.base.Strings; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; import org.obiba.magma.support.Initialisables; @@ -76,7 +77,7 @@ public void setEnvironment(Environment environment) { */ public synchronized RestDatasource getDatasource(@Nullable String opalUrl, String project) { final String projectUrl = getOpalProjectUrl(opalUrl, project); - opalUrl = Strings.isNullOrEmpty(opalUrl) ? getDefaultOpal() : opalUrl; + opalUrl = Strings.isNullOrEmpty(opalUrl) ? getPrimaryOpal() : opalUrl; OpalCredential opalCredential = getOpalCredential(opalUrl); @@ -107,18 +108,23 @@ public synchronized RestDatasource getDatasource(@Nullable String opalUrl, Strin * * @return */ - public String getDefaultOpal() { + private String getPrimaryOpal() { String opalConf = micaConfigService.getConfig().getOpal(); String opalDefault = environment.getProperty("opal.url"); return Strings.isNullOrEmpty(opalConf) ? opalDefault : opalConf; } + public boolean hasPrimaryOpal() { + String primaryOpal = getPrimaryOpal(); + return !Strings.isNullOrEmpty(primaryOpal) && primaryOpal.toLowerCase().startsWith("http"); + } + // // Opal Project // public List getProjectDtos(String opalUrl) throws URISyntaxException { - if (Strings.isNullOrEmpty(opalUrl)) opalUrl = getDefaultOpal(); + if (Strings.isNullOrEmpty(opalUrl)) opalUrl = getPrimaryOpal(); OpalJavaClient opalJavaClient = getOpalJavaClient(opalUrl); URI uri = opalJavaClient.newUri().segment("projects").build(); @@ -144,11 +150,15 @@ public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, S // synchronized Map getTaxonomiesInternal() { - try { - return opalServiceHelper.getTaxonomies(getOpalJavaClient()); - } catch (URISyntaxException e) { - log.error("Malformed opal URI", e); - throw new NoSuchElementException(); + if (hasPrimaryOpal()) { + try { + return opalServiceHelper.getTaxonomies(getOpalJavaClient()); + } catch (Exception e) { + log.error("Cannot retrieve Opal taxonomies", e); + throw new NoSuchElementException(); + } + } else { + return Maps.newHashMap(); } } @@ -170,7 +180,7 @@ private RestDatasource createRestDatasource(OpalCredential opalCredential, Strin } private String getOpalProjectUrl(String opalUrl, String project) { - String baseUrl = opalUrl == null ? getDefaultOpal() : opalUrl; + String baseUrl = opalUrl == null ? getPrimaryOpal() : opalUrl; return String.format("%s/ws/datasource/%s", StringUtils.stripEnd(baseUrl, "/"), project); } @@ -191,9 +201,9 @@ private OpalJavaClient getOpalJavaClient() throws URISyntaxException { if (opalJavaClient != null) return opalJavaClient; if (Strings.isNullOrEmpty(getOpalToken())) - opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getDefaultOpal()), getOpalUsername(), getOpalPassword()); + opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getPrimaryOpal()), getOpalUsername(), getOpalPassword()); else - opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getDefaultOpal()), getOpalToken()); + opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getPrimaryOpal()), getOpalToken()); return opalJavaClient; } @@ -221,9 +231,9 @@ private OpalCredential getOpalCredential(String opalUrl) { if (opalCredential.isPresent()) return opalCredential.get(); if (Strings.isNullOrEmpty(getOpalToken())) - return new OpalCredential(getDefaultOpal(), AuthType.USERNAME, getOpalUsername(), getOpalPassword()); + return new OpalCredential(getPrimaryOpal(), AuthType.USERNAME, getOpalUsername(), getOpalPassword()); else - return new OpalCredential(getDefaultOpal(), AuthType.TOKEN, getOpalToken()); + return new OpalCredential(getPrimaryOpal(), AuthType.TOKEN, getOpalToken()); } private String cleanupOpalUrl(String opalUrl) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index 735712d717..0fa9a2c866 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.validation.constraints.NotNull; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -108,12 +109,12 @@ public boolean metaTaxonomyContains(String taxonomy) { Optional termOpt = getTerm(targetVocabulary, taxonomy); if (termOpt.isPresent()) { Term term = termOpt.get(); - String visibility = term.getAttributeValue("visible"); + String hidden = term.getAttributeValue("hidden"); // visible by default - if (Strings.isNullOrEmpty(visibility)) return true; + if (Strings.isNullOrEmpty(hidden)) return true; // check visible attribute value try { - return Boolean.parseBoolean(visibility.toLowerCase()); + return !Boolean.parseBoolean(hidden.toLowerCase()); } catch (Exception e) { return false; } @@ -183,7 +184,13 @@ public List getAllVariableTaxonomies() { */ @NotNull public synchronized List getVariableTaxonomies() { - return variableTaxonomiesService.getSafeTaxonomies(); + List taxonomies = null; + try { + taxonomies = variableTaxonomiesService.getTaxonomies(); + } catch (Exception e) { + // ignore + } + return taxonomies == null ? Collections.emptyList() : taxonomies; } /** diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index 64b2e68b9b..81aa78ed5a 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -16,13 +16,8 @@ import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; -import org.obiba.opal.core.cfg.NoSuchTaxonomyException; -import org.obiba.opal.core.cfg.NoSuchVocabularyException; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; -import org.obiba.opal.core.domain.taxonomy.Vocabulary; -import org.obiba.opal.web.model.Opal; -import org.obiba.opal.web.taxonomy.Dtos; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; @@ -30,7 +25,6 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; -import javax.validation.constraints.NotNull; import java.io.File; import java.util.Collections; import java.util.Comparator; @@ -65,16 +59,22 @@ public void init() { @Cacheable(value = "variable-taxonomies", key = "'variable'") public List getTaxonomies() { - return getInternalTaxonomies(); + List taxonomyList = Lists.newArrayList(getTaxonomiesMap().values()); + Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); + return taxonomyList; } - public List getInternalTaxonomies() { - Map taxonomies; + // + // Private methods + // + + private Map getTaxonomiesMap() { + Map taxonomies = Maps.newHashMap(); // init with the ones from opal try { - taxonomies = opalService.getTaxonomiesInternal(); + taxonomies.putAll(opalService.getTaxonomiesInternal()); } catch (Exception e) { - taxonomies = Maps.newHashMap(); + // ignore } // read local files if (variableTaxonomiesDir.exists()) { @@ -107,137 +107,7 @@ public List getInternalTaxonomies() { log.warn("Taxonomies retrieval from plugin {} failed", provider.getName(), e); } } - List taxonomyList = Lists.newArrayList(taxonomies.values()); - Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); - return taxonomyList; - } - - /** - * Get a not-null list of taxonomies. - * - * @return - */ - @NotNull - public List getSafeTaxonomies() { - List taxonomies = null; - - try { - taxonomies = getTaxonomies(); - } catch (Exception e) { - // ignore - } - - return taxonomies == null ? Collections.emptyList() : taxonomies; - } - - public List getTaxonomyDtos() { - return getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); - } - - /** - * Get a summary of all the {@link Taxonomy}s available from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asSummaryDto) - .collect(Collectors.toList()); - - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); - } - - /** - * Get a summary of the {@link Taxonomy} available from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { - return Dtos.asSummaryDto(getTaxonomy(name)); - } - - /** - * Get a summary of all the {@link Taxonomy}s with their - * {@link Vocabulary}s from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) - .collect(Collectors.toList()); - - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); - } - - /** - * Get a summary of the {@link Taxonomy} with its - * {@link Vocabulary}s from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { - return Dtos.asVocabularySummaryDto(getTaxonomy(name)); - } - - /** - * Get a summary of the {@link Vocabulary} from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, - String vocabularyName) { - for (Vocabulary voc : getTaxonomy(name).getVocabularies()) { - if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); - } - throw new NoSuchVocabularyException(name, vocabularyName); - } - - /** - * Get the {@link Taxonomy} from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Taxonomy getTaxonomy(String name) { - return Dtos.fromDto(getTaxonomyDto(name)); - } - - /** - * Get the {@link Taxonomy} as a Dto from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Opal.TaxonomyDto getTaxonomyDto(String name) { - Map taxonomies = opalService.getTaxonomiesInternal(); - - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); - } - - return Dtos.asDto(taxonomies.get(name)); - } - - /** - * Get the {@link Vocabulary} as a Dto from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { - Map taxonomies = opalService.getTaxonomiesInternal(); - - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); - } - - return Dtos.asDto(taxonomies.get(name).getVocabulary(vocabularyName)); + return taxonomies; } } diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java index 8273ad7e0d..89998266d6 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java @@ -77,8 +77,6 @@ public class MicaConfigResource { @Inject private OpalService opalService; - @Inject VariableTaxonomiesService variableTaxonomiesService; - @Inject private OpalCredentialService opalCredentialService; @@ -668,42 +666,6 @@ public Map getAvailableLanguages(@QueryParam("locale") @DefaultV .collect(Collectors.toMap(lang -> lang, lang -> new Locale(lang).getDisplayLanguage(locale))); } - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies") - @RequiresAuthentication - @Deprecated - public List getTaxonomies() { - return variableTaxonomiesService.getTaxonomyDtos(); - } - - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies/summaries") - @RequiresAuthentication - @Deprecated - public Opal.TaxonomiesDto getTaxonomySummaries() { - return variableTaxonomiesService.getTaxonomySummaryDtos(); - } - - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies/vocabularies/summaries") - @RequiresAuthentication - @Deprecated - public Opal.TaxonomiesDto getTaxonomyVocabularySummaries() { - return variableTaxonomiesService.getTaxonomyVocabularySummaryDtos(); - } - /** * @deprecated kept for backward compatibility. * @return diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java new file mode 100644 index 0000000000..978b71e231 --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.taxonomy.rest; + +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; +import org.obiba.opal.core.cfg.NoSuchTaxonomyException; +import org.obiba.opal.core.cfg.NoSuchVocabularyException; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.Vocabulary; +import org.obiba.opal.web.model.Opal; +import org.obiba.opal.web.taxonomy.Dtos; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class AbstractTaxonomyResource { + + @Inject + protected VariableTaxonomiesService variableTaxonomiesService; + + protected List getTaxonomyDtos() { + return variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); + } + + /** + * Get a summary of all the {@link Taxonomy}s available from Opal master. + * + * @return + */ + protected Opal.TaxonomiesDto getTaxonomySummaryDtos() { + List summaries = variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asSummaryDto) + .collect(Collectors.toList()); + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} available from Opal master. + * + * @param name the taxonomy name + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { + return Dtos.asSummaryDto(getTaxonomyInternal(name)); + } + + /** + * Get a summary of all the {@link Taxonomy}s with their + * {@link Vocabulary}s from Opal master. + * + * @return + */ + protected Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { + List summaries = variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) + .collect(Collectors.toList()); + + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} with its + * {@link Vocabulary}s from Opal master. + * + * @param name the taxonomy name + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { + return Dtos.asVocabularySummaryDto(getTaxonomyInternal(name)); + } + + /** + * Get a summary of the {@link Vocabulary} from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, + String vocabularyName) { + for (Vocabulary voc : getTaxonomyInternal(name).getVocabularies()) { + if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); + } + throw new NoSuchVocabularyException(name, vocabularyName); + } + + + /** + * Get the {@link Taxonomy} as a Dto from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + protected Opal.TaxonomyDto getTaxonomyDto(String name) { + return Dtos.asDto(getTaxonomyInternal(name)); + } + + /** + * Get the {@link Vocabulary} as a Dto from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + protected Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { + return Dtos.asDto(getTaxonomyInternal(name).getVocabulary(vocabularyName)); + } + + // + // Private methods + // + + /** + * Get the {@link Taxonomy} from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + private Taxonomy getTaxonomyInternal(String name) { + Optional taxonomy = variableTaxonomiesService.getTaxonomies().stream() + .filter(taxo -> taxo.getName().equals(name)) + .findFirst(); + if (!taxonomy.isPresent()) { + throw new NoSuchTaxonomyException(name); + } + return taxonomy.get(); + } + +} diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java index cf7aca5b62..0736ee2dc6 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java @@ -14,7 +14,6 @@ import com.google.common.eventbus.EventBus; import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; -import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.security.Roles; import org.obiba.opal.web.model.Opal; import org.slf4j.Logger; @@ -28,20 +27,17 @@ @Component @Path("/taxonomies") -public class TaxonomiesResource { +public class TaxonomiesResource extends AbstractTaxonomyResource { private static final Logger logger = LoggerFactory.getLogger(TaxonomiesResource.class); - @Inject - private VariableTaxonomiesService variableTaxonomiesService; - @Inject private EventBus eventBus; @GET @Timed public List getTaxonomies() { - return variableTaxonomiesService.getTaxonomyDtos(); + return getTaxonomyDtos(); } @GET @@ -49,8 +45,8 @@ public List getTaxonomies() { @Timed public Opal.TaxonomiesDto getTaxonomySummaries( @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return variableTaxonomiesService.getTaxonomyVocabularySummaryDtos(); - return variableTaxonomiesService.getTaxonomySummaryDtos(); + if(withVocabularies) return getTaxonomyVocabularySummaryDtos(); + return getTaxonomySummaryDtos(); } @PUT diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java index a537237c50..6b09619f96 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java @@ -11,24 +11,19 @@ package org.obiba.mica.taxonomy.rest; import com.codahale.metrics.annotation.Timed; -import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.opal.web.model.Opal; import org.springframework.stereotype.Component; -import javax.inject.Inject; import javax.ws.rs.*; @Component @Path("/taxonomy/{name}") -public class TaxonomyResource { - - @Inject - private VariableTaxonomiesService variableTaxonomiesService; +public class TaxonomyResource extends AbstractTaxonomyResource { @GET @Timed public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { - return variableTaxonomiesService.getTaxonomyDto(name); + return getTaxonomyDto(name); } @GET @@ -36,8 +31,8 @@ public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name") String name, @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return variableTaxonomiesService.getTaxonomyVocabularySummaryDto(name); - return variableTaxonomiesService.getTaxonomySummaryDto(name); + if(withVocabularies) return getTaxonomyVocabularySummaryDto(name); + return getTaxonomySummaryDto(name); } @GET @@ -45,7 +40,7 @@ public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name @Timed public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return variableTaxonomiesService.getTaxonomyVocabularyDto(name, vocabularyName); + return getTaxonomyVocabularyDto(name, vocabularyName); } @GET @@ -53,7 +48,7 @@ public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getVocabularySummary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return variableTaxonomiesService.getTaxonomyVocabularySummaryDto(name, vocabularyName); + return getTaxonomyVocabularySummaryDto(name, vocabularyName); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java index 641a576408..be779167a1 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/AbstractPublishedDatasetResource.java @@ -21,6 +21,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.micaConfig.service.MicaConfigService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.spi.search.Indexer; @@ -66,7 +67,7 @@ public abstract class AbstractPublishedDatasetResource { protected ObjectMapper objectMapper; @Inject - protected VariableTaxonomiesService variableTaxonomiesService; + protected TaxonomiesService taxonomiesService; @Inject protected MicaConfigService micaConfigService; @@ -299,13 +300,7 @@ protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull S @NotNull protected List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = variableTaxonomiesService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + return taxonomiesService.getVariableTaxonomies(); } protected void checkContingencyAccess() { diff --git a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java index 64d5b0e76f..9b3f0c8318 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java +++ b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java @@ -12,7 +12,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.mica.spi.search.support.JoinQuery; @@ -42,7 +42,7 @@ public class CoverageQueryExecutor { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; @Inject - private VariableTaxonomiesService variableTaxonomiesService; + private TaxonomiesService taxonomiesService; @Inject private JoinQueryExecutor joinQueryExecutor; @@ -329,13 +329,7 @@ private MicaSearch.BucketCoverageDto.Builder getBucketCoverageDtoBuilder(String @NotNull private List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = variableTaxonomiesService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + return taxonomiesService.getVariableTaxonomies(); } @Nullable diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java index 5509c51b6e..2e6fcc7c08 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java @@ -13,7 +13,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.opal.core.domain.taxonomy.Taxonomy; @@ -31,7 +31,7 @@ public class TaxonomyAggregationMetaDataProvider implements AggregationMetaDataProvider { @Inject - VariableTaxonomiesService variableTaxonomiesService; + TaxonomiesService taxonomiesService; Map> cache; @@ -111,6 +111,6 @@ private Optional getVocabulary(String aggregation) { } protected List getVariableTaxonomies() { - return variableTaxonomiesService.getSafeTaxonomies(); + return taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java index 5262d0295a..150d9ee3f4 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java @@ -63,8 +63,6 @@ public class VariableQuery extends AbstractDocumentQuery { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; - private final VariableTaxonomiesService variableTaxonomiesService; - private final PublishedStudyService publishedStudyService; private final Dtos dtos; @@ -91,7 +89,6 @@ public class VariableQuery extends AbstractDocumentQuery { @Inject public VariableQuery( - VariableTaxonomiesService variableTaxonomiesService, PublishedStudyService publishedStudyService, CollectedDatasetService collectedDatasetService, HarmonizedDatasetService harmonizedDatasetService, @@ -103,7 +100,6 @@ public VariableQuery( PopulationAggregationMetaDataProvider populationAggregationMetaDataProvider, StudyAggregationMetaDataProvider studyAggregationMetaDataProvider, VariablesSetsAggregationMetaDataProvider variablesSetsAggregationMetaDataProvider) { - this.variableTaxonomiesService = variableTaxonomiesService; this.publishedStudyService = publishedStudyService; this.dtos = dtos; this.datasetAggregationMetaDataProvider = datasetAggregationMetaDataProvider; @@ -362,6 +358,6 @@ protected Taxonomy getTaxonomy() { @NotNull private List getVariableTaxonomies() { - return variableTaxonomiesService.getSafeTaxonomies(); + return taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java index b92c3d10a8..9baeca25bb 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java @@ -1,13 +1,8 @@ package org.obiba.mica.web.controller; -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; -import javax.servlet.http.HttpServletRequest; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.googlecode.protobuf.format.JsonFormat; import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.Subject; import org.obiba.magma.NoSuchVariableException; @@ -16,7 +11,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.dataset.service.PublishedDatasetService; -import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; @@ -30,9 +25,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.ModelAndView; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; -import com.googlecode.protobuf.format.JsonFormat; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; @Controller public class DatasetAnalysisController extends BaseController { @@ -49,7 +46,7 @@ public class DatasetAnalysisController extends BaseController { private Dtos dtos; @Inject - private VariableTaxonomiesService variableTaxonomiesService; + private TaxonomiesService taxonomiesService; @Inject private PublishedDatasetService publishedDatasetService; @@ -134,7 +131,7 @@ private Dataset getDataset(String id) { } private List getVariableTaxonomies() { - return variableTaxonomiesService.getSafeTaxonomies(); + return taxonomiesService.getVariableTaxonomies(); } private String getDatasetVariableJSON(DatasetVariable variable) { From 02b75e9731a2aefb97d6b7ec35d71e50ce4caa8f Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 16 Dec 2022 09:04:35 +0100 Subject: [PATCH 20/44] Fix dataset taxonomy init --- .../org/obiba/mica/micaConfig/service/TaxonomiesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index 0fa9a2c866..ebb2e165ad 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -249,7 +249,7 @@ private void initializeStudyTaxonomy() { private void initializeDatasetTaxonomy() { if (datasetTaxonomy != null) return; - datasetTaxonomy = copy(findTaxonomy(TaxonomyTarget.STUDY)); + datasetTaxonomy = copy(findTaxonomy(TaxonomyTarget.DATASET)); datasetHelper.applyIdTerms(datasetTaxonomy, "id"); } From 6c21d285696675b8293351e90c842e48d5e10e09 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 16 Dec 2022 09:43:18 +0100 Subject: [PATCH 21/44] Opal taxonomies cache replaced by generic variable taxonomies cache. Variable taxonomies are decorated with variable attributes --- ...va => VariableTaxonomiesUpdatedEvent.java} | 18 ++--- .../mica/micaConfig/service/CacheService.java | 21 ++---- .../service/VariableTaxonomiesService.java | 50 ++++++++++++- .../service/helper/OpalServiceHelper.java | 75 +++---------------- mica-core/src/main/resources/ehcache.xml | 8 -- .../obiba/mica/taxonomy/TaxonomyIndexer.java | 14 ++-- .../rest/AbstractTaxonomySearchResource.java | 6 +- 7 files changed, 82 insertions(+), 110 deletions(-) rename mica-core/src/main/java/org/obiba/mica/micaConfig/event/{OpalTaxonomiesUpdatedEvent.java => VariableTaxonomiesUpdatedEvent.java} (51%) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java similarity index 51% rename from mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java rename to mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java index becf167450..8ccd2407a4 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java @@ -15,21 +15,17 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentMap; +import java.util.Map; -public class OpalTaxonomiesUpdatedEvent { +public class VariableTaxonomiesUpdatedEvent { - private ConcurrentMap opalTaxonomies; + private Map taxonomies; - public OpalTaxonomiesUpdatedEvent(ConcurrentMap opalTaxonomies) { - this.opalTaxonomies = opalTaxonomies; + public VariableTaxonomiesUpdatedEvent(Map taxonomies) { + this.taxonomies = taxonomies; } - public List extractOpalTaxonomies() { - return new ArrayList<>(opalTaxonomies.values()); - } - - public boolean hasOpalTaxonomies() { - return opalTaxonomies != null; + public List getTaxonomies() { + return new ArrayList<>(taxonomies.values()); } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index a210660d14..a6c61924b5 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -10,8 +10,7 @@ package org.obiba.mica.micaConfig.service; -import javax.inject.Inject; - +import com.google.common.eventbus.EventBus; import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.service.CollectedDatasetService; @@ -23,7 +22,7 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import com.google.common.eventbus.EventBus; +import javax.inject.Inject; @Component public class CacheService { @@ -45,16 +44,10 @@ public class CacheService { @Inject private TaxonomiesService taxonomiesService; - @CacheEvict(value = "opal-taxonomies", allEntries = true, beforeInvocation = true) - public void clearOpalTaxonomiesCache() { - log.info("Clearing opal taxonomies cache"); - taxonomiesService.getVariableTaxonomies(); - } - @CacheEvict(value = "variable-taxonomies", allEntries = true, beforeInvocation = true) public void clearTaxonomiesCache() { log.info("Clearing variable taxonomies cache"); - clearOpalTaxonomiesCache(); + taxonomiesService.getVariableTaxonomies(); } @CacheEvict(value = "micaConfig", allEntries = true) @@ -115,9 +108,9 @@ public void buildDatasetVariablesCache() { try { harmonizedDatasetService .getVariableSummary(dataset, v.getName(), studyId, st.getSource()); - } catch(NoSuchVariableException ex) { + } catch (NoSuchVariableException ex) { //ignore - } catch(Exception e) { + } catch (Exception e) { log.warn("Error building dataset variable cache of harmonization dataset {}: {} {}", dataset.getId(), st, v, e); } @@ -127,9 +120,9 @@ public void buildDatasetVariablesCache() { .forEach(dataset -> collectedDatasetService.getDatasetVariables(dataset).forEach(v -> { try { collectedDatasetService.getVariableSummary(dataset, v.getName()); - } catch(NoSuchVariableException ex) { + } catch (NoSuchVariableException ex) { //ignore - } catch(Exception e) { + } catch (Exception e) { log.warn("Error building dataset variable cache of study dataset {}: {}", dataset.getId(), v, e); } })); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index 81aa78ed5a..f567fbc2ca 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -11,16 +11,22 @@ package org.obiba.mica.micaConfig.service; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; import org.obiba.mica.core.support.YamlResourceReader; +import org.obiba.mica.micaConfig.event.VariableTaxonomiesUpdatedEvent; import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; +import org.springframework.context.EnvironmentAware; +import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -36,12 +42,17 @@ * Access to the variable taxonomies, from the different providers, including opal, local files and plugins. */ @Component -public class VariableTaxonomiesService { +public class VariableTaxonomiesService implements EnvironmentAware { private static final Logger log = LoggerFactory.getLogger(VariableTaxonomiesService.class); private static final String VARIABLE_TAXONOMIES_PATH = "${MICA_HOME}/conf/taxonomies/variable"; + private Environment environment; + + @Inject + private EventBus eventBus; + @Inject private OpalService opalService; @@ -64,12 +75,17 @@ public List getTaxonomies() { return taxonomyList; } + @Override + public void setEnvironment(Environment environment) { + this.environment = environment; + } + // // Private methods // private Map getTaxonomiesMap() { - Map taxonomies = Maps.newHashMap(); + Map taxonomies = Maps.newConcurrentMap(); // init with the ones from opal try { taxonomies.putAll(opalService.getTaxonomiesInternal()); @@ -107,7 +123,37 @@ private Map getTaxonomiesMap() { log.warn("Taxonomies retrieval from plugin {} failed", provider.getName(), e); } } + // apply mica attributes + taxonomies.replaceAll((k, v) -> applyAttributes(taxonomies.get(k))); + eventBus.post(new VariableTaxonomiesUpdatedEvent(taxonomies)); return taxonomies; } + /** + * Decorate the variable taxonomies with some Mica specific attributes. + * + * @param taxonomy + * @return + */ + private Taxonomy applyAttributes(Taxonomy taxonomy) { + String defaultTermsSortOrder = environment.getProperty("opalTaxonomies.defaultTermsSortOrder"); + + taxonomy.getVocabularies().forEach(vocabulary -> { + String field = vocabulary.getAttributeValue("field"); + if (Strings.isNullOrEmpty(field)) { + vocabulary.addAttribute("field", + "attributes." + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + ".und"); + } + String alias = vocabulary.getAttributeValue("alias"); + if (Strings.isNullOrEmpty(alias)) { + vocabulary.addAttribute("alias", + "attributes-" + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + "-und"); + } + if (!Strings.isNullOrEmpty(defaultTermsSortOrder)) { + vocabulary.addAttribute("termsSortKey", defaultTermsSortOrder); + } + }); + return taxonomy; + } + } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java index 934a0c9683..d6bc85949f 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java @@ -10,16 +10,7 @@ package org.obiba.mica.micaConfig.service.helper; -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Collectors; - -import javax.inject.Inject; - -import org.obiba.mica.spi.search.support.AttributeKey; -import org.obiba.mica.micaConfig.event.OpalTaxonomiesUpdatedEvent; +import com.google.common.base.Strings; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.obiba.opal.rest.client.magma.OpalJavaClient; @@ -28,80 +19,34 @@ import org.obiba.opal.web.taxonomy.Dtos; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.context.EnvironmentAware; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; -import com.google.common.base.Strings; -import com.google.common.eventbus.EventBus; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Component -public class OpalServiceHelper implements EnvironmentAware { +public class OpalServiceHelper { private static final Logger log = LoggerFactory.getLogger(OpalServiceHelper.class); - private Environment environment; - - @Inject - private EventBus eventBus; - - @Cacheable(value = "opal-taxonomies", key = "#opalJavaClient.newUri().build()") //opal root url as key public Map getTaxonomies(OpalJavaClient opalJavaClient) { log.info("Fetching opal taxonomies"); URI uri = opalJavaClient.newUri().segment("system", "conf", "taxonomies").build(); List taxonomies = opalJavaClient .getResources(Opal.TaxonomyDto.class, uri, Opal.TaxonomyDto.newBuilder()); - - ConcurrentMap taxonomiesList = taxonomies.stream().map(taxonomyDto -> { - Taxonomy taxonomy = fromDto(taxonomyDto); - String defaultTermsSortOrder = environment.getProperty("opalTaxonomies.defaultTermsSortOrder"); - - if (!Strings.isNullOrEmpty(defaultTermsSortOrder)) { - taxonomy.getVocabularies().forEach(vocabulary -> vocabulary.addAttribute("termsSortKey", defaultTermsSortOrder)); - } - - return taxonomy; - }).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); - eventBus.post(new OpalTaxonomiesUpdatedEvent(taxonomiesList)); - return taxonomiesList; + return taxonomies.stream() + .map(Dtos::fromDto).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); } public Search.EntitiesResultDto getEntitiesCount(OpalJavaClient opalJavaClient, String query, String entityType) { log.info("Fetching opal entities count"); log.debug(" Entities query: {}", query); URI uri = opalJavaClient.newUri().segment("datasources", "entities", "_count") - .query("query", query) - .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); + .query("query", query) + .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); Search.EntitiesResultDto result = opalJavaClient.getResource(Search.EntitiesResultDto.class, uri, Search.EntitiesResultDto.newBuilder()); return result; } - - /** - * Decorate the variable taxonomies with some Mica specific attributes. - * - * @param dto - * @return - */ - private Taxonomy fromDto(Opal.TaxonomyDto dto) { - Taxonomy taxonomy = Dtos.fromDto(dto); - taxonomy.getVocabularies().forEach(vocabulary -> { - String field = vocabulary.getAttributeValue("field"); - if(Strings.isNullOrEmpty(field)) { - vocabulary.addAttribute("field", - "attributes." + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + ".und"); - } - String alias = vocabulary.getAttributeValue("alias"); - if(Strings.isNullOrEmpty(alias)) { - vocabulary.addAttribute("alias", - "attributes-" + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + "-und"); - } - }); - return taxonomy; - } - - @Override - public void setEnvironment(Environment environment) { - this.environment = environment; - } } diff --git a/mica-core/src/main/resources/ehcache.xml b/mica-core/src/main/resources/ehcache.xml index 460f2c5d08..bcaf3d4549 100644 --- a/mica-core/src/main/resources/ehcache.xml +++ b/mica-core/src/main/resources/ehcache.xml @@ -64,14 +64,6 @@ timeToLiveSeconds="0" timeToIdleSeconds="0"/> - - - - taxonomiesService.metaTaxonomyContains(t.getName())) .collect(Collectors.toList())); @@ -73,9 +73,9 @@ public void taxonomiesUpdated(TaxonomiesUpdatedEvent event) { if(indexer.hasIndex(Indexer.TERM_INDEX)) indexer.dropIndex(Indexer.TERM_INDEX); index(TaxonomyTarget.VARIABLE, - ImmutableList.builder().addAll(taxonomiesService.getVariableTaxonomies().stream() // - .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) // - .add(taxonomiesService.getVariableTaxonomy()) // + ImmutableList.builder().addAll(taxonomiesService.getVariableTaxonomies().stream() + .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) + .add(taxonomiesService.getVariableTaxonomy()) .build()); index(TaxonomyTarget.STUDY, Lists.newArrayList(taxonomiesService.getStudyTaxonomy())); index(TaxonomyTarget.DATASET, Lists.newArrayList(taxonomiesService.getDatasetTaxonomy())); diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java index eaf94bcd29..def8f306f6 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java @@ -96,7 +96,7 @@ protected void populate(Opal.TaxonomyDto.Builder tBuilder, Taxonomy taxonomy, protected List filterVocabularies(TaxonomyTarget target, String query, String locale) { try { - return esTaxonomyVocabularyService.find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), + return esTaxonomyVocabularyService.find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargetedQuery(target, query), getFields(locale, VOCABULARY_FIELDS), null, null).getList(); } catch (Exception e) { initTaxonomies(); @@ -116,7 +116,7 @@ protected List filterTerms(TaxonomyTarget target, String query, String l } return esTaxonomyTermService - .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), getFields(locale, TERM_FIELDS)) + .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargetedQuery(target, query), getFields(locale, TERM_FIELDS)) .getList(); } catch (Exception e) { initTaxonomies(); @@ -170,7 +170,7 @@ protected TaxonomyTarget getTaxonomyTarget(String target) { } } - private String getTargettedQuery(TaxonomyTarget target, String query) { + private String getTargetedQuery(TaxonomyTarget target, String query) { return String.format(Strings.isNullOrEmpty(query) ? "target:%s" : "target:%s AND (%s)", target.name(), query); } From deb6620392402162004e9c6cedb8eee9d694a935 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 16 Dec 2022 12:54:45 +0100 Subject: [PATCH 22/44] Code cleaning: Opal service helper merged into opal service --- .../mica/micaConfig/service/OpalService.java | 31 ++++++++--- .../service/helper/OpalServiceHelper.java | 52 ------------------- 2 files changed, 25 insertions(+), 58 deletions(-) delete mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java index 755aab9d9b..01c02296b8 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java @@ -18,13 +18,15 @@ import org.obiba.mica.dataset.service.KeyStoreService; import org.obiba.mica.micaConfig.AuthType; import org.obiba.mica.micaConfig.domain.OpalCredential; -import org.obiba.mica.micaConfig.service.helper.OpalServiceHelper; import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.obiba.opal.rest.client.magma.OpalJavaClient; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestDatasourceFactory; +import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.model.Projects; import org.obiba.opal.web.model.Search; +import org.obiba.opal.web.taxonomy.Dtos; import org.obiba.security.KeyStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.*; +import java.util.stream.Collectors; @Component public class OpalService implements EnvironmentAware { @@ -60,9 +63,6 @@ public class OpalService implements EnvironmentAware { @Inject private OpalCredentialService opalCredentialService; - @Inject - private OpalServiceHelper opalServiceHelper; - @Override public void setEnvironment(Environment environment) { this.environment = environment; @@ -138,7 +138,7 @@ public List getProjectDtos(String opalUrl) throws URISyntax public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, String entityType) { try { - return opalServiceHelper.getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); + return getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); } catch (URISyntaxException e) { log.error("Malformed opal URI", e); throw new NoSuchElementException(); @@ -152,7 +152,7 @@ public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, S synchronized Map getTaxonomiesInternal() { if (hasPrimaryOpal()) { try { - return opalServiceHelper.getTaxonomies(getOpalJavaClient()); + return getTaxonomies(getOpalJavaClient()); } catch (Exception e) { log.error("Cannot retrieve Opal taxonomies", e); throw new NoSuchElementException(); @@ -162,6 +162,25 @@ synchronized Map getTaxonomiesInternal() { } } + private Map getTaxonomies(OpalJavaClient opalJavaClient) { + log.info("Fetching opal taxonomies"); + URI uri = opalJavaClient.newUri().segment("system", "conf", "taxonomies").build(); + List taxonomies = opalJavaClient + .getResources(Opal.TaxonomyDto.class, uri, Opal.TaxonomyDto.newBuilder()); + return taxonomies.stream() + .map(Dtos::fromDto).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); + } + + private Search.EntitiesResultDto getEntitiesCount(OpalJavaClient opalJavaClient, String query, String entityType) { + log.info("Fetching opal entities count"); + log.debug(" Entities query: {}", query); + URI uri = opalJavaClient.newUri().segment("datasources", "entities", "_count") + .query("query", query) + .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); + Search.EntitiesResultDto result = opalJavaClient.getResource(Search.EntitiesResultDto.class, uri, Search.EntitiesResultDto.newBuilder()); + return result; + } + private RestDatasource createRestDatasource(OpalCredential opalCredential, String projectUrl, String opalUrl, String project) { if (opalCredential.getAuthType() == AuthType.CERTIFICATE) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java deleted file mode 100644 index d6bc85949f..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.micaConfig.service.helper; - -import com.google.common.base.Strings; -import org.obiba.opal.core.domain.taxonomy.Taxonomy; -import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; -import org.obiba.opal.rest.client.magma.OpalJavaClient; -import org.obiba.opal.web.model.Opal; -import org.obiba.opal.web.model.Search; -import org.obiba.opal.web.taxonomy.Dtos; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Component -public class OpalServiceHelper { - - private static final Logger log = LoggerFactory.getLogger(OpalServiceHelper.class); - - public Map getTaxonomies(OpalJavaClient opalJavaClient) { - log.info("Fetching opal taxonomies"); - URI uri = opalJavaClient.newUri().segment("system", "conf", "taxonomies").build(); - List taxonomies = opalJavaClient - .getResources(Opal.TaxonomyDto.class, uri, Opal.TaxonomyDto.newBuilder()); - return taxonomies.stream() - .map(Dtos::fromDto).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); - } - - public Search.EntitiesResultDto getEntitiesCount(OpalJavaClient opalJavaClient, String query, String entityType) { - log.info("Fetching opal entities count"); - log.debug(" Entities query: {}", query); - URI uri = opalJavaClient.newUri().segment("datasources", "entities", "_count") - .query("query", query) - .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); - Search.EntitiesResultDto result = opalJavaClient.getResource(Search.EntitiesResultDto.class, uri, Search.EntitiesResultDto.newBuilder()); - return result; - } -} From f2ea72387fa09f06887e9520a9c93734b9f3a056 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 11 Jan 2023 22:51:27 +0100 Subject: [PATCH 23/44] classifications administration --- .../micaConfig/service/TaxonomiesService.java | 129 ++++++++++++++++++ .../service/VariableTaxonomiesService.java | 11 +- .../taxonomy/rest/MetaTaxonomyResource.java | 52 +++++++ .../rest/TaxonomiesSearchResource.java | 29 +++- mica-webapp/bower.json | 2 +- mica-webapp/src/main/resources/i18n/en.json | 10 +- mica-webapp/src/main/resources/i18n/fr.json | 10 +- .../main/webapp/app/admin/admin-controller.js | 101 ++++++++++++++ .../src/main/webapp/app/admin/admin-router.js | 7 + .../webapp/app/admin/views/admin-view.html | 28 ++-- .../app/admin/views/classifications.html | 37 +++++ .../webapp/app/config/views/config-form.html | 3 +- .../src/main/webapp/app/search/search.js | 7 - .../views/classifications-view-header.html | 19 --- .../app/search/views/search-view-header.html | 19 --- mica-webapp/src/main/webapp/app/services.js | 2 +- .../src/main/webapp/assets/js/mica-search.js | 12 +- 17 files changed, 395 insertions(+), 83 deletions(-) create mode 100644 mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java create mode 100644 mica-webapp/src/main/webapp/app/admin/views/classifications.html delete mode 100644 mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html delete mode 100644 mica-webapp/src/main/webapp/app/search/views/search-view-header.html diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index ebb2e165ad..a74e115311 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -12,6 +12,7 @@ import com.google.common.base.Strings; import com.google.common.eventbus.Subscribe; +import org.apache.commons.compress.utils.Lists; import org.obiba.mica.core.event.DocumentSetUpdatedEvent; import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; @@ -23,8 +24,11 @@ import org.obiba.mica.study.event.StudyPublishedEvent; import org.obiba.mica.study.event.StudyUnpublishedEvent; import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.obiba.opal.core.domain.taxonomy.Term; import org.obiba.opal.core.domain.taxonomy.Vocabulary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -40,6 +44,8 @@ @Service public class TaxonomiesService { + private static final Logger log = LoggerFactory.getLogger(TaxonomiesService.class); + private final VariableTaxonomiesService variableTaxonomiesService; private final MicaConfigService micaConfigService; @@ -101,6 +107,40 @@ public TaxonomiesService( @NotNull public Taxonomy getTaxonomyTaxonomy() { initialize(); + boolean modified = false; + List variableTaxonomies = getAllVariableTaxonomies(); + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals("variable")) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + // check variable taxonomies to be added to meta + List variableTaxonomiesNames = variableChars.getTerms().stream() + .map(TaxonomyEntity::getName).collect(Collectors.toList()); + for (Taxonomy variableTaxonomy : variableTaxonomies) { + if (!variableTaxonomiesNames.contains(variableTaxonomy.getName())) { + Term newTerm = new Term(variableTaxonomy.getName()); + newTerm.addAttribute("hidden", "true"); + newTerm.setTitle(variableTaxonomy.getTitle()); + newTerm.setDescription(variableTaxonomy.getDescription()); + variableChars.getTerms().add(newTerm); + modified = true; + } + } + // check variable taxonomies to be removed from meta + List reverseVariableTaxonomiesNames = variableTaxonomies.stream() + .map(TaxonomyEntity::getName).collect(Collectors.toList()); + List newTerms = variableChars.getTerms().stream() + .filter(term -> "Mica_variable".equals(term.getName()) || reverseVariableTaxonomiesNames.contains(term.getName())) + .collect(Collectors.toList()); + if (newTerms.size() < variableChars.getTerms().size()) { + variableChars.setTerms(newTerms); + modified = true; + } + } + } + if (modified) { + log.debug("Taxonomy of taxonomies was modified"); + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, taxonomyTaxonomy); + } return taxonomyTaxonomy; } @@ -123,6 +163,94 @@ public boolean metaTaxonomyContains(String taxonomy) { return false; } + /** + * Change described taxonomy position when there are several for the target considered. + * + * @param target + * @param name + * @param up + */ + public void moveTaxonomy(TaxonomyTarget target, String name, boolean up) { + Taxonomy metaTaxonomy = getTaxonomyTaxonomy(); + boolean modified = false; + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals(target.asId())) { + if (TaxonomyTarget.VARIABLE.equals(target)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + modified = moveTerm(variableChars.getTerms(), name, up); + } else if (vocabulary.hasTerm(name) && vocabulary.getTerms().size() > 1) { + modified = moveTerm(vocabulary.getTerms(), name, up); + } + } + } + if (modified) { + this.taxonomyTaxonomy = metaTaxonomy; + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, metaTaxonomy); + } + } + + /** + * Modify term list by moving up/down the term with provided name. + * + * @param terms + * @param taxonomyName + * @param up + * @return Whether the list was modified + */ + private boolean moveTerm(List terms, String taxonomyName, boolean up) { + int idx = -1; + for (Term term : terms) { + idx++; + if (term.getName().equals(taxonomyName)) { + break; + } + } + if (idx > -1) { + int newIdx = up ? idx - 1 : idx + 1; + if (newIdx > -1 && newIdx < terms.size()) { + Term term = terms.remove(idx); + terms.add(newIdx, term); + return true; + } + } + return false; + } + + /** + * Hide/show the described taxonomy for the target considered. + * + * @param target + * @param taxonomyName + * @param name + * @param value + */ + public void setTaxonomyAttribute(TaxonomyTarget target, String taxonomyName, String name, String value) { + if (Strings.isNullOrEmpty(name)) return; + Taxonomy metaTaxonomy = getTaxonomyTaxonomy(); + boolean modified = false; + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals(target.asId())) { + if (TaxonomyTarget.VARIABLE.equals(target)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + Optional found = variableChars.getTerms().stream().filter(term -> term.getName().equals(taxonomyName)).findFirst(); + if (found.isPresent()) { + Term term = found.get(); + term.getAttributes().put(name, value); + modified = true; + } + } else if (vocabulary.hasTerm(taxonomyName)) { + Term term = vocabulary.getTerm(taxonomyName); + term.getAttributes().put(name, value); + modified = true; + } + } + } + if (modified) { + this.taxonomyTaxonomy = metaTaxonomy; + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, metaTaxonomy); + } + } + /** * Get the taxonomy that describes the {@link org.obiba.mica.network.domain.Network} properties. * @@ -378,4 +506,5 @@ public void refreshTaxonomyTaxonomyIfNeeded(MicaConfig currentConfig, MicaConfig taxonomyTaxonomy = null; } } + } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java index f567fbc2ca..ca40443d7d 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -89,6 +89,8 @@ private Map getTaxonomiesMap() { // init with the ones from opal try { taxonomies.putAll(opalService.getTaxonomiesInternal()); + if (log.isDebugEnabled()) + taxonomies.keySet().forEach(name -> log.debug("Taxonomy from opal: {}", name)); } catch (Exception e) { // ignore } @@ -96,10 +98,14 @@ private Map getTaxonomiesMap() { if (variableTaxonomiesDir.exists()) { File[] yamlFiles = variableTaxonomiesDir.listFiles(file -> !file.isDirectory() && file.getName().endsWith(".yml")); if (yamlFiles != null) { - log.info("Fetching local taxonomies"); + log.info("Fetching local taxonomies: {}", VARIABLE_TAXONOMIES_PATH); for (File yamlFile : yamlFiles) { try { Taxonomy taxonomy = YamlResourceReader.readFile(yamlFile.getAbsolutePath(), Taxonomy.class); + log.debug("Taxonomy from folder {}: {}", variableTaxonomiesDir.getAbsolutePath(), taxonomy.getName()); + // override any duplicated taxonomy + if (taxonomies.containsKey(taxonomy.getName())) + log.warn("Taxonomy is duplicated and will be overridden: {}", taxonomy.getName()); taxonomies.put(taxonomy.getName(), taxonomy); } catch (Exception e) { log.error("Taxonomy file could not be read: {}", yamlFile.getAbsolutePath(), e); @@ -114,9 +120,10 @@ private Map getTaxonomiesMap() { log.info("Fetching taxonomies from plugin: {}", provider.getName()); try { for (Taxonomy taxonomy : provider.getTaxonomies()) { + log.debug("Taxonomy from plugin {}: {}", provider.getName(), taxonomy.getName()); // override any duplicated taxonomy if (taxonomies.containsKey(taxonomy.getName())) - log.warn("Taxonomy with name {} is duplicated ({} plugin)", provider.getName(), taxonomy.getName()); + log.warn("Taxonomy is duplicated and will be overridden: {}", taxonomy.getName()); taxonomies.put(taxonomy.getName(), taxonomy); } } catch (Exception e) { diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java new file mode 100644 index 0000000000..52dd9ac737 --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.taxonomy.rest; + + +import com.codahale.metrics.annotation.Timed; +import org.obiba.mica.micaConfig.service.TaxonomiesService; +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.opal.web.model.Opal; +import org.obiba.opal.web.taxonomy.Dtos; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +@Component +@Path("/meta-taxonomy") +public class MetaTaxonomyResource { + + @Inject + private TaxonomiesService taxonomiesService; + + @GET + @Timed + public Opal.TaxonomyDto getTaxonomy() { + return Dtos.asDto(taxonomiesService.getTaxonomyTaxonomy()); + } + + @PUT + @Path("/{target}/{taxonomy}/_move") + public Response up(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("dir") String dir) { + taxonomiesService.moveTaxonomy(TaxonomyTarget.fromId(target), taxonomy, "up".equals(dir)); + return Response.ok().build(); + } + + @PUT + @Path("/{target}/{taxonomy}/_attribute") + public Response hide(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("name") String name, @QueryParam("value") String value) { + taxonomiesService.setTaxonomyAttribute(TaxonomyTarget.fromId(target), taxonomy, name, value); + return Response.ok().build(); + } + +} diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java index f57c8d1e72..a93d483ac2 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java @@ -10,6 +10,7 @@ package org.obiba.mica.taxonomy.rest; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -23,10 +24,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; +import com.google.common.collect.Maps; import org.obiba.mica.security.SubjectUtils; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.taxonomy.TaxonomyResolver; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.taxonomy.Dtos; import org.slf4j.Logger; @@ -51,15 +55,28 @@ public class TaxonomiesSearchResource extends AbstractTaxonomySearchResource { @Path("/_filter") @Timed public List filter(@Context HttpServletRequest request, @QueryParam("target") @DefaultValue("variable") String target, + @QueryParam("mode") @DefaultValue("search") String mode, @QueryParam("query") String query, @QueryParam("locale") String locale) { TaxonomyTarget taxonomyTarget = getTaxonomyTarget(target); + List taxonomies = getTaxonomies(taxonomyTarget); + + boolean searchMode = "search".equals(mode); + + Map>> taxoNamesMap; + if (searchMode) { + List filteredVocabularies = filterVocabularies(taxonomyTarget, query, locale); + taxoNamesMap = TaxonomyResolver + .asMap(filteredVocabularies, filterTerms(taxonomyTarget, query, locale, filteredVocabularies)); + } else { + taxoNamesMap = Maps.newHashMap(); + for (Taxonomy taxonomy : taxonomies) { + Map> taxoMap = taxonomy.getVocabularies().stream().collect(Collectors.toMap(TaxonomyEntity::getName, vocabulary -> new ArrayList())); + taxoNamesMap.put(taxonomy.getName(), taxoMap); + } + } - List filteredVocabularies = filterVocabularies(taxonomyTarget, query, locale); - - Map>> taxoNamesMap = TaxonomyResolver - .asMap(filteredVocabularies, filterTerms(taxonomyTarget, query, locale, filteredVocabularies)); List results = Lists.newArrayList(); - getTaxonomies(taxonomyTarget).stream() + taxonomies.stream() .filter(t -> taxoNamesMap.containsKey(t.getName())) .forEach(taxo -> { Opal.TaxonomyDto.Builder tBuilder = Dtos.asDto(taxo, false).toBuilder(); @@ -85,7 +102,7 @@ public List search(@Context HttpServletRequest request, .map(t -> TaxonomyTarget.valueOf(t.getName().toUpperCase())).collect(Collectors.toList()) : Lists.newArrayList(TaxonomyTarget.valueOf(target.toUpperCase())); - targets.forEach(t -> filter(request, t.name(), query, locale).stream() + targets.forEach(t -> filter(request, t.name(), "search", query, locale).stream() .map(taxo -> Opal.TaxonomyBundleDto.newBuilder().setTarget(t.name().toLowerCase()).setTaxonomy(taxo).build()) .forEach(results::add)); return results; diff --git a/mica-webapp/bower.json b/mica-webapp/bower.json index 87b4d7d9d4..0013d27d93 100644 --- a/mica-webapp/bower.json +++ b/mica-webapp/bower.json @@ -30,7 +30,7 @@ "mica-study-timeline": "https://github.com/obiba/mica-study-timeline.git#1.1.4", "ng-file-upload": "12.2.13", "ng-obiba": "https://github.com/obiba/ng-obiba.git#2.0.2", - "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#4.3.0", + "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#no-search", "filesize": "3.6.0", "ngclipboard": "1.1.3", "angular-utils-pagination": "0.11.1", diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 8813c04943..42dd5fd0db 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -1481,7 +1481,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Navigate the classification system and select the search criteria." + "help": "Navigate the classification system and manage the search criteria." }, "all-null-classifications": "", "all-variable-classifications": "All variable classifications", @@ -1539,6 +1539,10 @@ "message": "Are you sure you want to delete the access of {{arg0}} {{arg1}}" } }, + "up": "Up", + "down": "Down", + "hide": "Hide", + "show": "Show", "draft": "Draft", "published": "Published", "role": "Role", @@ -1591,7 +1595,7 @@ "security": "Security", "system": "System", "content": "Content", - "data-management": "Data Management", + "search-management": "Search", "data-access": "Data Access", "research": "Research", "properties": "Properties", @@ -1629,7 +1633,7 @@ "project-config": "Configure Research Project form.", "file-system": "Manage file system.", "search": "Search in the published repository.", - "classifications": "Browse taxonomies.", + "classifications": "Manage the taxonomies that are used as search criteria.", "translations": "Manage labels translations.", "cart": "Cart of variables.", "sets": "Lists of variables.", diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index 31858fb0be..ab664037e6 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -1481,7 +1481,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Naviguer dans le système de classification et sélectionner les critères de recherche." + "help": "Naviguer dans le système de classification et gérer les critères de recherche." }, "all-null-classifications": "", "all-variable-classifications": "Toutes les classifications de variables", @@ -1539,6 +1539,10 @@ "message": "Êtes-vous sûr de vouloir supprimer l'accès de {{arg0}} {{arg1}}" } }, + "up": "Monter", + "down": "Descendre", + "hide": "Cacher", + "show": "Montrer", "draft": "Brouillon", "published": "Publié", "role": "Rôle", @@ -1591,7 +1595,7 @@ "security": "Sécurité", "system": "Système", "content": "Contenu", - "data-management": "Gestion de données", + "search-management": "Recherche", "data-access": "Accès aux données", "research": "Recherche", "properties": "Propriétés", @@ -1629,7 +1633,7 @@ "project-config": "Configurer le formulaire de projet de recherche.", "file-system": "Gérer le système de fichiers.", "search": "Chercher dans le catalogue publié.", - "classifications": "Parcourir les taxonomies.", + "classifications": "Gérer les taxonomies servant à bâtir les critères de recherche.", "translations": "Gérer les traductions des libellés.", "cart": "Panier de variables.", "sets": "Listes de variables.", diff --git a/mica-webapp/src/main/webapp/app/admin/admin-controller.js b/mica-webapp/src/main/webapp/app/admin/admin-controller.js index fa105fcf74..aa0cea2f64 100644 --- a/mica-webapp/src/main/webapp/app/admin/admin-controller.js +++ b/mica-webapp/src/main/webapp/app/admin/admin-controller.js @@ -261,4 +261,105 @@ mica.admin }); }; + }]) + + .controller('ClassificationsController', ['$timeout', + '$scope', + '$location', + '$translate', + function ($timeout, $scope, + $location, + $translate) { + + //$scope.options = options; + + $scope.taxonomyTypeMap = { //backwards compatibility for pluralized naming in configs. + variable: 'variables', + study: 'studies', + network: 'networks', + dataset: 'datasets' + }; + + $translate(['search.classifications-title', 'search.classifications-link', 'search.faceted-navigation-help']) + .then(function (translation) { + $scope.hasClassificationsTitle = translation['search.classifications-title']; + $scope.hasClassificationsLinkLabel = translation['search.classifications-link']; + $scope.hasFacetedNavigationHelp = translation['search.faceted-navigation-help']; + }); + + var searchTaxonomyDisplay = { + variable: true, //$scope.options.variables.showSearchTab, + dataset: true, //$scope.options.datasets.showSearchTab, + study: true, //$scope.options.studies.showSearchTab, + network: true, //$scope.options.networks.showSearchTab + }; + + $scope.lang = $translate.use(); + + function initSearchTabs() { + function getTabsOrderParam(arg) { + var value = $location.search()[arg]; + + return value && value.split(',') + .filter(function (t) { + return t; + }) + .map(function (t) { + return t.trim(); + }); + } + + const defaultTargetTabsOrderParam = ['variable', 'dataset', 'study', 'network']; + var targetTabsOrderParam = getTabsOrderParam('targetTabsOrder'); + $scope.targetTabsOrder = (targetTabsOrderParam || defaultTargetTabsOrderParam).filter(function (t) { + return searchTaxonomyDisplay[t]; + }); + + if ($location.search().target) { + $scope.target = $location.search().target; + } else if (!$scope.target) { + $scope.target = $scope.targetTabsOrder[0]; + } + } + + var onSelectTerm = function (target, taxonomy, vocabulary, args) { + args = args || {}; + + if (args.text) { + args.text = args.text.replace(/[^a-zA-Z0-9*" _-]/g, ''); + } + + if (angular.isString(args)) { + args = { term: args }; + } + + console.log('onSelectTerm'); + }; + + $scope.navigateToTarget = function (target) { + $location.search('target', target); + $location.search('taxonomy', null); + $location.search('vocabulary', null); + $scope.target = target; + }; + + $scope.onSelectTerm = onSelectTerm; + + $scope.toggleFullscreen = function (fullscreen) { + if ($scope.isFullscreen && $scope.isFullscreen !== fullscreen) { + // in case the ESC key was pressed + $timeout(function() {$scope.isFullscreen = fullscreen;}); + } else { + $scope.isFullscreen = fullscreen; + } + }; + + $scope.isFullscreen = false; + + function init() { + $scope.lang = $translate.use(); + initSearchTabs(); + } + + init(); }]); diff --git a/mica-webapp/src/main/webapp/app/admin/admin-router.js b/mica-webapp/src/main/webapp/app/admin/admin-router.js index 3c46db896b..df97acab2c 100644 --- a/mica-webapp/src/main/webapp/app/admin/admin-router.js +++ b/mica-webapp/src/main/webapp/app/admin/admin-router.js @@ -36,6 +36,13 @@ mica.admin authorizedRoles: ['mica-administrator'] } }) + .when('/admin/classifications', { + templateUrl: 'app/admin/views/classifications.html', + controller: 'ClassificationsController', + access: { + authorizedRoles: ['mica-administrator'] + } + }) .when('/admin/indexing', { templateUrl: 'app/admin/views/indexing.html', controller: 'IndexingController', diff --git a/mica-webapp/src/main/webapp/app/admin/views/admin-view.html b/mica-webapp/src/main/webapp/app/admin/views/admin-view.html index c8eb812fdb..1d67cc29bf 100644 --- a/mica-webapp/src/main/webapp/app/admin/views/admin-view.html +++ b/mica-webapp/src/main/webapp/app/admin/views/admin-view.html @@ -49,18 +49,12 @@

    admin-page.help.caching

    • -
  • - {{'indexing.title' | translate}} -

    - admin-page.help.indexing -

    -
    • -
  • +
  • {{'logs.title' | translate}} @@ -128,24 +122,20 @@

      • -
    • data-management
      • +
    • search-management
      • -
    • - - {{'search.title' | translate}} +
    • + + {{'classifications.title' | translate}}

      - admin-page.help.search + admin-page.help.classifications

    • - - {{'classifications.title' | translate}} - + {{'indexing.title' | translate}}

      - admin-page.help.classifications + admin-page.help.indexing

      • diff --git a/mica-webapp/src/main/webapp/app/admin/views/classifications.html b/mica-webapp/src/main/webapp/app/admin/views/classifications.html new file mode 100644 index 0000000000..6cd08ab801 --- /dev/null +++ b/mica-webapp/src/main/webapp/app/admin/views/classifications.html @@ -0,0 +1,37 @@ + + +
      + +

      +
        +
      1. global.menu.admin
        2. +
      2. classifications.title
        3. +
      +

      +

      classifications.help

      + + + + + +
      + + + +
      + +
      diff --git a/mica-webapp/src/main/webapp/app/config/views/config-form.html b/mica-webapp/src/main/webapp/app/config/views/config-form.html index 9e6d732e1b..0dad8b8d39 100644 --- a/mica-webapp/src/main/webapp/app/config/views/config-form.html +++ b/mica-webapp/src/main/webapp/app/config/views/config-form.html @@ -39,8 +39,7 @@

      config.serverIdentif

      config.dataSource

      -
      +
      diff --git a/mica-webapp/src/main/webapp/app/search/search.js b/mica-webapp/src/main/webapp/app/search/search.js index 7944fc716c..5a54390eaa 100644 --- a/mica-webapp/src/main/webapp/app/search/search.js +++ b/mica-webapp/src/main/webapp/app/search/search.js @@ -13,10 +13,3 @@ mica.search = angular.module('mica.search', [ 'obiba.mica.search' ]); - -mica.search - .config(['ngObibaMicaSearchTemplateUrlProvider', - function (ngObibaMicaSearchTemplateUrlProvider) { - ngObibaMicaSearchTemplateUrlProvider.setHeaderUrl('search', 'app/search/views/search-view-header.html'); - ngObibaMicaSearchTemplateUrlProvider.setHeaderUrl('classifications', 'app/search/views/classifications-view-header.html'); - }]); diff --git a/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html b/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html deleted file mode 100644 index a7df2ad9e4..0000000000 --- a/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html +++ /dev/null @@ -1,19 +0,0 @@ - - -
      -

      -
        -
      1. global.menu.admin
        2. -
      2. classifications.title
        3. -
      -

      -

      classifications.help

      -
      diff --git a/mica-webapp/src/main/webapp/app/search/views/search-view-header.html b/mica-webapp/src/main/webapp/app/search/views/search-view-header.html deleted file mode 100644 index dd35b9ef57..0000000000 --- a/mica-webapp/src/main/webapp/app/search/views/search-view-header.html +++ /dev/null @@ -1,19 +0,0 @@ - - -
      -

      -
        -
      1. global.menu.admin
        2. -
      2. search.title
        3. -
      -

      -

      search.help

      -
      diff --git a/mica-webapp/src/main/webapp/app/services.js b/mica-webapp/src/main/webapp/app/services.js index baeae4805f..cb33186eef 100644 --- a/mica-webapp/src/main/webapp/app/services.js +++ b/mica-webapp/src/main/webapp/app/services.js @@ -194,7 +194,7 @@ mica.service('AuthenticationSharedService', ['$rootScope', '$q', '$http', '$cook mica.factory('MetricsService', ['$resource', function ($resource) { - return $resource('jvm', {}, { + return $resource('/jvm', {}, { 'get': { method: 'GET'} }); }]); diff --git a/mica-webapp/src/main/webapp/assets/js/mica-search.js b/mica-webapp/src/main/webapp/assets/js/mica-search.js index 574068563e..b256a04570 100644 --- a/mica-webapp/src/main/webapp/assets/js/mica-search.js +++ b/mica-webapp/src/main/webapp/assets/js/mica-search.js @@ -131,11 +131,21 @@ Vue.component('search-criteria', { return ai - bi; }); + const isTermHidden = function(term) { + if (term.attributes) { + const hiddenAttr = term.attributes.filter(attr => attr.key === 'hidden').pop(); + if (hiddenAttr) { + return hiddenAttr.value === 'true'; + } + } + return false; + } + for (let target of filteredTargets) { this.criteriaMenu.items[target.name].title = StringLocalizer.localize(target.title); switch (target.name) { case 'variable': - let level = target.terms[0].terms; + let level = target.terms[0].terms.filter(t => !isTermHidden(t)); const theRest = target.terms.slice(1); if (theRest.length > 0) { From 701ccb8acbd9791658f9d54ed38307d77d8f256a Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 11:01:27 +0100 Subject: [PATCH 24/44] make sure variable taxo title/description in the meta taxo matches the one of the original taxo --- .../micaConfig/service/TaxonomiesService.java | 15 +++++++++++++++ mica-webapp/src/main/resources/i18n/en.json | 2 +- mica-webapp/src/main/resources/i18n/fr.json | 2 +- .../src/main/webapp/assets/js/mica-search.js | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index a74e115311..d438eac53d 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -11,6 +11,8 @@ package org.obiba.mica.micaConfig.service; import com.google.common.base.Strings; +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; import com.google.common.eventbus.Subscribe; import org.apache.commons.compress.utils.Lists; import org.obiba.mica.core.event.DocumentSetUpdatedEvent; @@ -123,6 +125,19 @@ public Taxonomy getTaxonomyTaxonomy() { newTerm.setDescription(variableTaxonomy.getDescription()); variableChars.getTerms().add(newTerm); modified = true; + } else { + // check any title/description modifications + Term term = variableChars.getTerm(variableTaxonomy.getName()); + MapDifference diff = Maps.difference(term.getTitle(), variableTaxonomy.getTitle()); + if (!diff.areEqual()) { + term.setTitle(variableTaxonomy.getTitle()); + modified = true; + } + diff = Maps.difference(term.getDescription(), variableTaxonomy.getDescription()); + if (!diff.areEqual()) { + term.setDescription(variableTaxonomy.getDescription()); + modified = true; + } } } // check variable taxonomies to be removed from meta diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 42dd5fd0db..6b937c1cdd 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -1481,7 +1481,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Navigate the classification system and manage the search criteria." + "help": "Navigate the classification system and manage order and visibility of the search criteria." }, "all-null-classifications": "", "all-variable-classifications": "All variable classifications", diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index ab664037e6..91ad8c6e36 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -1481,7 +1481,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Naviguer dans le système de classification et gérer les critères de recherche." + "help": "Naviguer dans le système de classification et gérer l'ordre et la visibilité des critères de recherche." }, "all-null-classifications": "", "all-variable-classifications": "Toutes les classifications de variables", diff --git a/mica-webapp/src/main/webapp/assets/js/mica-search.js b/mica-webapp/src/main/webapp/assets/js/mica-search.js index b256a04570..06b3d052cc 100644 --- a/mica-webapp/src/main/webapp/assets/js/mica-search.js +++ b/mica-webapp/src/main/webapp/assets/js/mica-search.js @@ -163,7 +163,7 @@ Vue.component('search-criteria', { case 'dataset': case 'study': case 'network': - this.criteriaMenu.items[target.name].menus = target.terms; + this.criteriaMenu.items[target.name].menus = target.terms ? target.terms.filter(t => !isTermHidden(t)) : []; break; } if (this.criteriaMenu.items[target.name].menus && this.criteriaMenu.items[target.name].menus.length > 0) { From 26795aa8d5132c69b5526aec696c6637ff9c3f34 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 11:50:09 +0100 Subject: [PATCH 25/44] hide instead of remove a target vocabulary if type is disabled --- .../micaConfig/service/TaxonomiesService.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java index d438eac53d..7f1af9cb1e 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -360,18 +360,32 @@ private synchronized void initialize() { } private void initializeTaxonomyTaxonomy() { - if (taxonomyTaxonomy != null) return; - taxonomyTaxonomy = copy(findTaxonomy(TaxonomyTarget.TAXONOMY)); + if (taxonomyTaxonomy == null) + taxonomyTaxonomy = copy(findTaxonomy(TaxonomyTarget.TAXONOMY)); MicaConfig config = micaConfigService.getConfig(); if (!config.isNetworkEnabled() || config.isSingleNetworkEnabled()) { - taxonomyTaxonomy.removeVocabulary("network"); + hideMetaVocabularyTerms("network"); } if (!config.isStudyDatasetEnabled() && !config.isHarmonizationDatasetEnabled()) { - taxonomyTaxonomy.removeVocabulary("dataset"); - taxonomyTaxonomy.removeVocabulary("variable"); + hideMetaVocabularyTerms("dataset"); + hideMetaVocabularyTerms("variable"); } if (config.isSingleStudyEnabled() && !config.isHarmonizationDatasetEnabled()) { - taxonomyTaxonomy.removeVocabulary("study"); + hideMetaVocabularyTerms("study"); + } + } + + private void hideMetaVocabularyTerms(String vocabularyName) { + if (taxonomyTaxonomy.hasVocabulary(vocabularyName)) { + Vocabulary vocabulary = taxonomyTaxonomy.getVocabulary(vocabularyName); + if (TaxonomyTarget.VARIABLE.asId().equals(vocabularyName)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + if (variableChars.hasTerms()) + variableChars.getTerms().forEach(term -> term.addAttribute("hidden", "true")); + } + else if (vocabulary.hasTerms()) { + vocabulary.getTerms().forEach(term -> term.addAttribute("hidden", "true")); + } } } From 87485e76968839834341f2c308369269853caa48 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 14:03:37 +0100 Subject: [PATCH 26/44] Plugins REST API added --- .../mica/core/upgrade/Mica510upgrade.java | 10 +- .../micaConfig/service/PluginsService.java | 141 ++++++++++++++---- .../org/obiba/mica/web/model/PluginDtos.java | 62 ++++++++ .../mica/micaConfig/rest/PluginsResource.java | 71 +++++++++ .../src/main/protobuf/MicaPlugins.proto | 39 +++++ 5 files changed, 294 insertions(+), 29 deletions(-) create mode 100644 mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java create mode 100644 mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java create mode 100644 mica-web-model/src/main/protobuf/MicaPlugins.proto diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java index 1c0c2ed354..1bb13592b5 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java @@ -6,6 +6,7 @@ import org.bson.Document; import org.json.JSONException; import org.obiba.mica.micaConfig.service.CacheService; +import org.obiba.mica.micaConfig.service.PluginsService; import org.obiba.runtime.Version; import org.obiba.runtime.upgrade.UpgradeStep; import org.slf4j.Logger; @@ -18,14 +19,17 @@ public class Mica510upgrade implements UpgradeStep { private static final Logger logger = LoggerFactory.getLogger(Mica510upgrade.class); - private CacheService cacheService; + private final CacheService cacheService; private final MongoTemplate mongoTemplate; + private final PluginsService pluginsService; + @Inject - public Mica510upgrade(CacheService cacheService, MongoTemplate mongoTemplate) { + public Mica510upgrade(CacheService cacheService, MongoTemplate mongoTemplate, PluginsService pluginsService) { this.cacheService = cacheService; this.mongoTemplate = mongoTemplate; + this.pluginsService = pluginsService; } @Override @@ -53,6 +57,8 @@ public void execute(Version currentVersion) { cacheService.clearAllCaches(); // TODO get updated search plugin and rebuild search index + logger.info("Updating search plugin..."); + } private void updateMicaConfig() throws JSONException { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index 46a63af317..36c693c4d5 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -16,17 +16,14 @@ import com.google.common.io.Files; import org.obiba.core.util.FileUtil; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; -import org.obiba.mica.spi.tables.StudyTableSourceService; -import org.obiba.mica.spi.tables.StudyTableSourceServiceLoader; import org.obiba.mica.spi.search.ConfigurationProvider; import org.obiba.mica.spi.search.SearchEngineService; import org.obiba.mica.spi.search.SearchEngineServiceLoader; +import org.obiba.mica.spi.tables.StudyTableSourceService; +import org.obiba.mica.spi.tables.StudyTableSourceServiceLoader; import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; import org.obiba.mica.spi.taxonomies.TaxonomiesProviderServiceLoader; -import org.obiba.plugins.PluginRepositoryCache; -import org.obiba.plugins.PluginRepositoryException; -import org.obiba.plugins.PluginResources; -import org.obiba.plugins.PluginsManagerHelper; +import org.obiba.plugins.*; import org.obiba.plugins.spi.ServicePlugin; import org.obiba.runtime.Version; import org.slf4j.Logger; @@ -39,11 +36,7 @@ import javax.inject.Inject; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.stream.Collectors; @Component @@ -110,6 +103,115 @@ public Collection getTaxonomiesProviderServices() { .collect(Collectors.toList()); } + /** + * Get the location of the plugin packages repository. + * + * @return + */ + public String getUpdateSite() { + return environment.getProperty("plugins.updateSite", DEFAULT_PLUGINS_UPDATE_SITE); + } + + /** + * Get the last time at which the update site was successfully. + * + * @return + */ + public Date getLastUpdate() { + return getPluginRepositoryCache().getLastUpdate(); + } + + /** + * Reports if system restart is required to finalize plugin installation. + * + * @return + */ + public boolean restartRequired() { + File[] children = pluginsDir.listFiles(pathname -> !pathname.getName().startsWith(".")); + if (children == null || children.length == 0) return false; + for (File child : children) { + if (child.isFile() && child.getName().endsWith(PluginResources.PLUGIN_DIST_SUFFIX)) return true; + if (child.isDirectory() && new File(child, PluginResources.UNINSTALL_FILE).exists()) return true; + } + return false; + } + + /** + * Get the plugins registered in the system. + * + * @return + */ + public List getInstalledPlugins() { + return registeredPlugins.stream() + .map(PluginPackage::new) + .collect(Collectors.toList()); + } + + /** + * Get the list of plugins that are marked to uninstallation. + * + * @return + */ + public Collection getUninstalledPluginNames() { + List names = Lists.newArrayList(); + File[] children = pluginsDir.listFiles(pathname -> pathname.isDirectory() && !pathname.getName().startsWith(".")); + if (children == null || children.length == 0) return names; + for (File child : children) { + PluginResources plugin = new MicaPlugin(child); + if (plugin.isToUninstall()) names.add(plugin.getName()); + } + return names; + } + + /** + * Get the plugins registered in the system that can be updated according to the update site registry. + * + * @return + */ + public List getUpdatablePlugins() { + // exclude already installed plugin packages whatever the version is + return getPluginRepositoryCache().getOrUpdatePluginRepository().getPlugins().stream() + .filter(PluginPackage::hasMicaVersion) + .filter(pp -> registeredPlugins.stream().anyMatch(rp -> pp.isNewerThan(rp.getName(), rp.getVersion()))) + .filter(pp -> runtimeVersionProvider.getVersion().compareTo(pp.getMicaVersion()) >= 0) + .collect(Collectors.toList()); + } + + /** + * Get the plugins that are not installed and that are available from the update site registry. + * + * @return + */ + public List getAvailablePlugins() { + // exclude already installed plugin packages whatever the version is + return getPluginRepositoryCache().getOrUpdatePluginRepository().getPlugins().stream() + .filter(PluginPackage::hasMicaVersion) + .filter(pp -> registeredPlugins.stream().noneMatch(rp -> pp.isSameAs(rp.getName()))) + .filter(pp -> runtimeVersionProvider.getVersion().compareTo(pp.getMicaVersion()) >= 0) + .collect(Collectors.toList()); + } + + /** + * Perform the plugin installation by retrieving the plugin package from the update site. + * + * @param name + * @param version + */ + public void installPlugin(String name, String version) { + String pVersion = version; + if (Strings.isNullOrEmpty(version)) { + // no version specified: get the latest + pVersion = getPluginRepositoryCache().getPluginLatestVersion(name); + } + try { + File tmpDir = Files.createTempDir(); + installPlugin(getPluginRepositoryCache().downloadPlugin(name, pVersion, tmpDir), true); + FileUtil.delete(tmpDir); + } catch (IOException e) { + throw new PluginRepositoryException("Failed to install plugin " + name + ":" + version + " : " + e.getMessage(), e); + } + } + // // Private methods // @@ -234,21 +336,6 @@ private void processPlugins(Map pluginsMap, File plugin } } - private void installPlugin(String name, String version) { - String pVersion = version; - if (Strings.isNullOrEmpty(version)) { - // no version specified: get the latest - pVersion = getPluginRepositoryCache().getPluginLatestVersion(name); - } - try { - File tmpDir = Files.createTempDir(); - installPlugin(getPluginRepositoryCache().downloadPlugin(name, pVersion, tmpDir), true); - FileUtil.delete(tmpDir); - } catch (IOException e) { - throw new PluginRepositoryException("Failed to install plugin " + name + ":" + version + " : " + e.getMessage(), e); - } - } - private void installPlugin(File pluginFile, boolean rmAfterInstall) { try { if (!pluginsDir.exists()) pluginsDir.mkdirs(); @@ -261,7 +348,7 @@ private void installPlugin(File pluginFile, boolean rmAfterInstall) { private PluginRepositoryCache getPluginRepositoryCache() { if (pluginRepositoryCache == null) - pluginRepositoryCache = new PluginRepositoryCache(runtimeVersionProvider, environment.getProperty("plugins.updateSite", DEFAULT_PLUGINS_UPDATE_SITE)); + pluginRepositoryCache = new PluginRepositoryCache(runtimeVersionProvider, getUpdateSite()); return pluginRepositoryCache; } } diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java new file mode 100644 index 0000000000..6fa897bb52 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.web.model; + +import com.google.common.base.Strings; +import org.obiba.magma.type.DateTimeType; +import org.obiba.plugins.PluginPackage; + +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +public class PluginDtos { + + public static MicaPlugins.PluginPackagesDto asDto(String site, Date updated, boolean restart, List packages) { + return asDto(site, updated, restart, packages, null); + } + + public static MicaPlugins.PluginPackagesDto asDto(String site, Date updated, boolean restart, List packages, Collection uninstalledNames) { + MicaPlugins.PluginPackagesDto.Builder builder = asDto(site, updated, restart); + builder.addAllPackages(packages.stream().map(p -> asDto(p, uninstalledNames == null ? null : uninstalledNames.contains(p.getName()))) + .collect(Collectors.toList())); + return builder.build(); + } + + public static MicaPlugins.PluginPackagesDto.Builder asDto(String site, Date updated, boolean restart) { + MicaPlugins.PluginPackagesDto.Builder builder = MicaPlugins.PluginPackagesDto.newBuilder() + .setSite(site) + .setRestart(restart); + if (updated != null) builder.setUpdated(DateTimeType.get().valueOf(updated).toString()); + return builder; + } + + public static MicaPlugins.PluginPackageDto asDto(PluginPackage pluginPackage, Boolean uninstalled) { + MicaPlugins.PluginPackageDto.Builder buider = MicaPlugins.PluginPackageDto.newBuilder() + .setName(pluginPackage.getName()) + .setType(pluginPackage.getType()) + .setTitle(pluginPackage.getTitle()) + .setDescription(pluginPackage.getDescription()) + .setAuthor(Strings.isNullOrEmpty(pluginPackage.getAuthor()) ? "-" : pluginPackage.getAuthor()) + .setMaintainer(Strings.isNullOrEmpty(pluginPackage.getMaintainer()) ? "-" : pluginPackage.getMaintainer()) + .setLicense(Strings.isNullOrEmpty(pluginPackage.getLicense()) ? "-" : pluginPackage.getLicense()) + .setVersion(pluginPackage.getVersion().toString()) + .setMicaVersion(pluginPackage.getMicaVersion().toString()); + if (!Strings.isNullOrEmpty(pluginPackage.getWebsite())) + buider.setWebsite(pluginPackage.getWebsite()); + if (!Strings.isNullOrEmpty(pluginPackage.getFileName())) + buider.setFile(pluginPackage.getFileName()); + if (uninstalled != null) buider.setUninstalled(uninstalled); + return buider.build(); + } + +} diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java new file mode 100644 index 0000000000..28917508aa --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.rest; + + +import com.google.common.base.Strings; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.web.model.MicaPlugins; +import org.obiba.mica.web.model.PluginDtos; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Response; +import java.util.stream.Collectors; + +@Component +@Path("/config/plugins") +@RequiresAuthentication +public class PluginsResource { + + @Inject + private PluginsService pluginsService; + + @GET + public MicaPlugins.PluginPackagesDto getInstalledPlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getInstalledPlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList()), pluginsService.getUninstalledPluginNames()); + } + + @GET + @Path("/_updates") + public MicaPlugins.PluginPackagesDto getUpdatablePlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getUpdatablePlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList())); + } + + @GET + @Path("/_available") + public MicaPlugins.PluginPackagesDto getAvailablePlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getAvailablePlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList())); + } + + @POST + public Response installPlugin(@QueryParam("name") String name, @QueryParam("version") String version) { + if (!Strings.isNullOrEmpty(name)) { + pluginsService.installPlugin(name, version); + return Response.ok().build(); + } + return Response.status(Response.Status.BAD_REQUEST).build(); + } +} diff --git a/mica-web-model/src/main/protobuf/MicaPlugins.proto b/mica-web-model/src/main/protobuf/MicaPlugins.proto new file mode 100644 index 0000000000..d483d8dcc0 --- /dev/null +++ b/mica-web-model/src/main/protobuf/MicaPlugins.proto @@ -0,0 +1,39 @@ +package obiba.mica; +option java_package = "org.obiba.mica.web.model"; + +message PluginPackageDto { + required string name = 1; + required string title = 2; + required string description = 3; + required string author = 4; + required string maintainer = 5; + required string license = 6; + optional string website = 7; + required string version = 8; + required string micaVersion = 9; + required string type = 10; + optional string file = 11; + optional bool uninstalled = 12; + extensions 1000 to max; +} + +message PluginPackagesDto { + required string site = 1; + optional string updated = 2; + required bool restart = 3; + repeated PluginPackageDto packages = 4; +} + +message PluginDto { + required string name = 1; + required string title = 2; + required string description = 3; + required string author = 4; + required string maintainer = 5; + required string license = 6; + optional string website = 7; + required string version = 8; + required string micaVersion = 9; + required string type = 10; + required string siteProperties = 11; +} From 4ebe42da5e95661b788bb567411bb64a78f6aa03 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 14:34:07 +0100 Subject: [PATCH 27/44] Plugin REST API added --- .../micaConfig/service/PluginsService.java | 61 +++++++++++++++++++ .../org/obiba/mica/web/model/PluginDtos.java | 16 +++++ .../mica/micaConfig/rest/PluginResource.java | 59 ++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index 36c693c4d5..4d2497dd20 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -212,10 +212,71 @@ public void installPlugin(String name, String version) { } } + /** + * Get the installed plugin with the given name. + * + * @param name + * @return + */ + public PluginResources getInstalledPlugin(String name) { + Optional plugin = registeredPlugins.stream().filter(p -> p.getName().equals(name)).findFirst(); + if (!plugin.isPresent()) throw new NoSuchElementException("No such plugin with name: " + name); + return plugin.get(); + } + + /** + * Uninstall a plugin. + * + * @param name + */ + public void prepareUninstallPlugin(String name) { + PluginResources plugin = getInstalledPlugin(name); + plugin.prepareForUninstall(); + } + + /** + * Cancel plugin uninstallation before it is effective. + * + * @param name + */ + public void cancelUninstallPlugin(String name) { + PluginResources plugin = getInstalledPlugin(name); + plugin.cancelUninstall(); + } + + /** + * Set the site properties of the installed plugin with the given name. + * + * @param name + * @param properties + */ + public void setInstalledPluginSiteProperties(String name, String properties) { + try { + PluginResources thePlugin = getInstalledPlugin(name); + thePlugin.writeSiteProperties(properties); + updateServiceProperties(name, thePlugin.getProperties()); + } catch (IOException e) { + throw new PluginRepositoryException("Failed to save plugin " + name + " site properties: " + e.getMessage(), e); + } + } + // // Private methods // + private void updateServiceProperties(String name, Properties properties) { + ServicePlugin servicePlugin = getServicePlugin(name); + if (servicePlugin != null) { + servicePlugin.configure(properties); + } + } + + ServicePlugin getServicePlugin(String name) { + Optional service = servicePlugins.stream().filter(s -> name.equals(s.getName())).findFirst(); + if (!service.isPresent()) throw new NoSuchElementException(name); + return service.get(); + } + private Collection getServicePlugins(Class clazz) { return servicePlugins.stream().filter(s -> clazz.isAssignableFrom(s.getClass())).collect(Collectors.toList()); } diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java index 6fa897bb52..3b50fe6430 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java @@ -13,6 +13,7 @@ import com.google.common.base.Strings; import org.obiba.magma.type.DateTimeType; import org.obiba.plugins.PluginPackage; +import org.obiba.plugins.PluginResources; import java.util.Collection; import java.util.Date; @@ -59,4 +60,19 @@ public static MicaPlugins.PluginPackageDto asDto(PluginPackage pluginPackage, Bo return buider.build(); } + public static MicaPlugins.PluginDto asDto(PluginResources plugin) { + MicaPlugins.PluginDto.Builder builder = MicaPlugins.PluginDto.newBuilder() + .setName(plugin.getName()) + .setTitle(plugin.getTitle()) + .setDescription(plugin.getDescription()) + .setAuthor(plugin.getAuthor()) + .setMaintainer(plugin.getMaintainer()) + .setLicense(plugin.getLicense()) + .setVersion(plugin.getVersion().toString()) + .setMicaVersion(plugin.getHostVersion().toString()) + .setType(plugin.getType()) + .setSiteProperties(plugin.getSitePropertiesString()); + return builder.build(); + } + } diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java new file mode 100644 index 0000000000..4fd416b1a5 --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.rest; + + +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.web.model.MicaPlugins; +import org.obiba.mica.web.model.PluginDtos; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +@Component +@Scope("request") +@Path("/config/plugin/{name}") +public class PluginResource { + + @PathParam("name") + private String name; + + @Inject + private PluginsService pluginsService; + + @GET + public MicaPlugins.PluginDto get() { + return PluginDtos.asDto(pluginsService.getInstalledPlugin(name)); + } + + @DELETE + public Response uninstall() { + pluginsService.prepareUninstallPlugin(name); + return Response.noContent().build(); + } + + @PUT + public Response cancelUninstallation() { + pluginsService.cancelUninstallPlugin(name); + return Response.noContent().build(); + } + + @PUT + @Path("/cfg") + @Consumes("text/plain") + public Response saveConfig(String properties) { + pluginsService.setInstalledPluginSiteProperties(name, properties); + return Response.ok().build(); + } +} From 8b7acd7a285ce8fa81255efcc2ecebfde54243b6 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 15:27:53 +0100 Subject: [PATCH 28/44] Service Plugin REST API added --- .../micaConfig/service/PluginsService.java | 12 +++++++ .../mica/micaConfig/rest/PluginResource.java | 36 +++++++++++++++---- .../src/main/protobuf/MicaPlugins.proto | 10 ++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index 4d2497dd20..66a927973b 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -260,6 +260,18 @@ public void setInstalledPluginSiteProperties(String name, String properties) { } } + public boolean isInstalledPluginRunning(String name) { + return getServicePlugin(name).isRunning(); + } + + public void startInstalledPlugin(String name) { + getServicePlugin(name).start(); + } + + public void stopInstalledPlugin(String name) { + getServicePlugin(name).stop(); + } + // // Private methods // diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java index 4fd416b1a5..f969f74cee 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java @@ -11,6 +11,7 @@ package org.obiba.mica.micaConfig.rest; +import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.obiba.mica.micaConfig.service.PluginsService; import org.obiba.mica.web.model.MicaPlugins; import org.obiba.mica.web.model.PluginDtos; @@ -24,27 +25,25 @@ @Component @Scope("request") @Path("/config/plugin/{name}") +@RequiresAuthentication public class PluginResource { - @PathParam("name") - private String name; - @Inject private PluginsService pluginsService; @GET - public MicaPlugins.PluginDto get() { + public MicaPlugins.PluginDto get(@PathParam("name") String name) { return PluginDtos.asDto(pluginsService.getInstalledPlugin(name)); } @DELETE - public Response uninstall() { + public Response uninstall(@PathParam("name") String name) { pluginsService.prepareUninstallPlugin(name); return Response.noContent().build(); } @PUT - public Response cancelUninstallation() { + public Response cancelUninstallation(@PathParam("name") String name) { pluginsService.cancelUninstallPlugin(name); return Response.noContent().build(); } @@ -52,8 +51,31 @@ public Response cancelUninstallation() { @PUT @Path("/cfg") @Consumes("text/plain") - public Response saveConfig(String properties) { + public Response saveConfig(@PathParam("name") String name, String properties) { pluginsService.setInstalledPluginSiteProperties(name, properties); return Response.ok().build(); } + + @GET + @Path("/service") + public MicaPlugins.ServicePluginDto getServiceStatus(@PathParam("name") String name) { + return MicaPlugins.ServicePluginDto.newBuilder() + .setName(name) + .setStatus(pluginsService.isInstalledPluginRunning(name) ? MicaPlugins.ServicePluginStatus.RUNNING : MicaPlugins.ServicePluginStatus.STOPPED) + .build(); + } + + @PUT + @Path("/service") + public Response startService(@PathParam("name") String name) { + pluginsService.startInstalledPlugin(name); + return Response.ok().build(); + } + + @DELETE + @Path("/service") + public Response stopService(@PathParam("name") String name) { + pluginsService.stopInstalledPlugin(name); + return Response.ok().build(); + } } diff --git a/mica-web-model/src/main/protobuf/MicaPlugins.proto b/mica-web-model/src/main/protobuf/MicaPlugins.proto index d483d8dcc0..2afd7487fb 100644 --- a/mica-web-model/src/main/protobuf/MicaPlugins.proto +++ b/mica-web-model/src/main/protobuf/MicaPlugins.proto @@ -37,3 +37,13 @@ message PluginDto { required string type = 10; required string siteProperties = 11; } + +enum ServicePluginStatus { + RUNNING = 1; + STOPPED = 2; +} + +message ServicePluginDto { + required string name = 1; + required ServicePluginStatus status = 2; +} From 38c6ed783547c4818e0eb88e99ed002046521747 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 15:43:34 +0100 Subject: [PATCH 29/44] Update the search plugin in the upgrade procedure --- .../mica/core/upgrade/Mica510upgrade.java | 20 +++++++++++++++++-- .../micaConfig/service/PluginsService.java | 6 +++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java index 1bb13592b5..c15c1cb293 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java @@ -7,6 +7,7 @@ import org.json.JSONException; import org.obiba.mica.micaConfig.service.CacheService; import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.plugins.PluginResources; import org.obiba.runtime.Version; import org.obiba.runtime.upgrade.UpgradeStep; import org.slf4j.Logger; @@ -14,6 +15,8 @@ import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Component; +import java.util.NoSuchElementException; + @Component public class Mica510upgrade implements UpgradeStep { @@ -56,9 +59,22 @@ public void execute(Version currentVersion) { logger.info("Clearing all caches..."); cacheService.clearAllCaches(); - // TODO get updated search plugin and rebuild search index logger.info("Updating search plugin..."); - + try { + String searchPluginName = pluginsService.getSearchPluginName(); + // check it is installed + pluginsService.getInstalledPlugin(searchPluginName); + // check it is updatable + if (pluginsService.getUpdatablePlugins().stream().anyMatch(pkg -> pkg.getName().equals(searchPluginName))) { + // install latest version + pluginsService.installPlugin(searchPluginName, null); + } + // TODO rebuild search index? + } catch (Exception e) { + // not installed, not to be upgraded + // OR upgrade failed for unknown reason + logger.error("Error occurred while Updating 'Search Plugin'", e); + } } private void updateMicaConfig() throws JSONException { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index 66a927973b..15fa85c264 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -304,12 +304,16 @@ public void init() { initPlugins(); } + public String getSearchPluginName() { + return environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); + } + /** * Initialize plugin resources. */ private void initPlugins() { Collection plugins = getPlugins(true); - String searchPluginName = environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); + String searchPluginName = getSearchPluginName(); try { String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(searchPluginName); From 8d470c4a4d3131bd061fe33bf77d97e6095d55d9 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 12 Jan 2023 15:46:14 +0100 Subject: [PATCH 30/44] ng-obiba-mica version --- mica-webapp/bower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mica-webapp/bower.json b/mica-webapp/bower.json index 0013d27d93..7bf0ca83da 100644 --- a/mica-webapp/bower.json +++ b/mica-webapp/bower.json @@ -30,7 +30,7 @@ "mica-study-timeline": "https://github.com/obiba/mica-study-timeline.git#1.1.4", "ng-file-upload": "12.2.13", "ng-obiba": "https://github.com/obiba/ng-obiba.git#2.0.2", - "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#no-search", + "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#master", "filesize": "3.6.0", "ngclipboard": "1.1.3", "angular-utils-pagination": "0.11.1", From c0c664e74055fd0277ede4b6c174f03e374cdcb6 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 13 Jan 2023 10:00:15 +0100 Subject: [PATCH 31/44] No server error when opal projects cannot be retrieved --- .../network/rest/DraftNetworkResource.java | 12 +++- .../mica/study/rest/StudyStateResource.java | 13 +++- mica-webapp/src/main/resources/i18n/en.json | 4 +- .../webapp/app/dataset/dataset-controller.js | 60 ++++++++++++------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java index c384acdd6d..758b94bd10 100644 --- a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java @@ -12,6 +12,7 @@ import com.google.common.base.Strings; +import org.apache.commons.compress.utils.Lists; import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; import org.obiba.mica.NoSuchEntityException; @@ -29,6 +30,8 @@ import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Projects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -52,6 +55,8 @@ @Scope("request") public class DraftNetworkResource extends AbstractGitPersistableResource { + private static final Logger log = LoggerFactory.getLogger(DraftNetworkResource.class); + @Inject private NetworkService networkService; @@ -191,7 +196,12 @@ public SubjectAclResource accesses() { @Path("/projects") public List projects() throws URISyntaxException { checkPermission("/draft/network", "VIEW"); - return opalService.getProjectDtos(networkService.findById(id).getOpal()); + try { + return opalService.getProjectDtos(networkService.findById(id).getOpal()); + } catch (Exception e) { + log.warn("Failed at retrieving opal projects: {}", e.getMessage()); + return Lists.newArrayList(); + } } @Override diff --git a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java index daef38d827..8f74a31bcf 100644 --- a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java @@ -18,6 +18,7 @@ import javax.ws.rs.GET; import javax.ws.rs.Path; +import org.apache.commons.compress.utils.Lists; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.study.domain.BaseStudy; @@ -25,6 +26,8 @@ import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Projects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -37,6 +40,8 @@ @Scope("request") public class StudyStateResource { + private static final Logger log = LoggerFactory.getLogger(StudyStateResource.class); + @Inject private StudyService studyService; @@ -69,7 +74,11 @@ public Mica.StudySummaryDto get() { public List projects() throws URISyntaxException { subjectAclService.checkPermission(resource, "VIEW", id); String opalUrl = Optional.ofNullable(studyService.findStudy(id)).map(BaseStudy::getOpal).orElse(null); - - return opalService.getProjectDtos(opalUrl); + try { + return opalService.getProjectDtos(opalUrl); + } catch (Exception e) { + log.warn("Failed at retrieving opal projects: {}", e.getMessage()); + return Lists.newArrayList(); + } } } diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 6b937c1cdd..79a97dc971 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -965,8 +965,8 @@ }, "file": { "title": "File Source", - "info": "Path to the file from which table's variables will be extracted from. If the path is relative, it is first looked up in the dataset's folder, then in the associated study's folder. The table name is optional if there is only one described in the file.", - "info-schema": "Path to the file from which table's variables will be extracted from. If the path is relative, it is first looked up in the harmonization protocol's folder, then in the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." + "info": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the dataset's folder, then in the associated study's folder. The table name is optional if there is only one described in the file.", + "info-schema": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the harmonization protocol's folder, then in the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." } }, "delete-dialog": { diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js index 5078e21188..074e1d49b6 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js @@ -175,20 +175,26 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise.then(function (projects) { - $scope.projects = projects; - var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === $scope.studyTable.project; })[0]; + StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise + .then(function (projects) { + $scope.projects = projects; + var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === $scope.studyTable.project; })[0]; - if (selectedProject) { - $scope.selected.project = selectedProject; + if (selectedProject) { + $scope.selected.project = selectedProject; - selectedTable = selectedProject.datasource.table.filter(function (t) {return t === $scope.studyTable.table; })[0]; + selectedTable = selectedProject.datasource.table.filter(function (t) {return t === $scope.studyTable.table; })[0]; - if (selectedTable) { - $scope.selected.project.table = selectedTable; + if (selectedTable) { + $scope.selected.project.table = selectedTable; + } } - } - }); + }) + .catch(function (err) { + console.log(err); + $scope.projects = []; + $scope.selected.project = undefined; + }); } }); @@ -952,24 +958,32 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise.then(function (projects) { - $scope.projects = projects; - var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === table.project; })[0]; + StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise + .then(function (projects) { + $scope.projects = projects; + var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === table.project; })[0]; - if (selectedProject) { - $scope.selected.project = selectedProject; + if (selectedProject) { + $scope.selected.project = selectedProject; - selectedTable = selectedProject.datasource.table.filter(function (t) {return t === table.table; })[0]; + selectedTable = selectedProject.datasource.table.filter(function (t) {return t === table.table; })[0]; - if (selectedTable) { - $scope.selected.project.table = selectedTable; + if (selectedTable) { + $scope.selected.project.table = selectedTable; + } + } else { + $scope.selected.project = $scope.projects.pop(); + if ($scope.selected.project) { + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); + } } - } else { - $scope.selected.project = $scope.projects.pop(); - $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); - } - }); + }) + .catch(function (err) { + console.log(err); + $scope.projects = []; + $scope.selected.project = undefined; + }); } }); From cd1defc84b483964349c87893bf1cd6b87e72c45 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 13 Jan 2023 15:04:50 +0100 Subject: [PATCH 32/44] Fix some wording and showVariableStatistics template setting added --- .../src/main/resources/_templates/dataset.ftl | 2 +- .../resources/_templates/libs/settings.ftl | 1 + .../main/resources/_templates/variable.ftl | 50 ++++++++++--------- mica-webapp/src/main/resources/i18n/en.json | 14 +++--- mica-webapp/src/main/resources/i18n/fr.json | 14 +++--- 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/mica-webapp/src/main/resources/_templates/dataset.ftl b/mica-webapp/src/main/resources/_templates/dataset.ftl index 257235f70f..c6a6b00210 100644 --- a/mica-webapp/src/main/resources/_templates/dataset.ftl +++ b/mica-webapp/src/main/resources/_templates/dataset.ftl @@ -118,7 +118,7 @@ ${initiativeIcon}<#else>${studyIcon}"> ${localize(study.acronym)} - <#if showDatasetContingencyLink> + <#if showVariableStatistics && showDatasetContingencyLink> <@message "dataset.crosstab.title"/> diff --git a/mica-webapp/src/main/resources/_templates/libs/settings.ftl b/mica-webapp/src/main/resources/_templates/libs/settings.ftl index fb287edd96..1c8188ec92 100644 --- a/mica-webapp/src/main/resources/_templates/libs/settings.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/settings.ftl @@ -140,6 +140,7 @@ <#assign searchCriteriaMenus = ["variable", "dataset", "study", "network"]/> +<#assign showVariableStatistics = true/> <#assign showHarmonizedVariableSummarySelector = true/> diff --git a/mica-webapp/src/main/resources/_templates/variable.ftl b/mica-webapp/src/main/resources/_templates/variable.ftl index d27c252a3a..11382f1708 100644 --- a/mica-webapp/src/main/resources/_templates/variable.ftl +++ b/mica-webapp/src/main/resources/_templates/variable.ftl @@ -289,34 +289,36 @@

    -
    -
    -
    -
    -

    <@message "summary-statistics"/>

    - <#if showDatasetContingencyLink> - <#if variable.nature == "CATEGORICAL"> - - <@message "dataset.crosstab.title"/> - - <#elseif variable.nature == "CONTINUOUS"> - - <@message "dataset.crosstab.title"/> - + <#if showVariableStatistics> +
    +
    +
    +
    +

    <@message "summary-statistics"/>

    + <#if showDatasetContingencyLink> + <#if variable.nature == "CATEGORICAL"> + + <@message "dataset.crosstab.title"/> + + <#elseif variable.nature == "CONTINUOUS"> + + <@message "dataset.crosstab.title"/> + + - -
    -
    - <#if user?? || !config.variableSummaryRequiresAuthentication> - <@variableSummary variable=variable/> - <#else> - <@message "sign-in-for-variable-statistics"/> - <@message "sign-in"/> - +
    +
    + <#if user?? || !config.variableSummaryRequiresAuthentication> + <@variableSummary variable=variable/> + <#else> + <@message "sign-in-for-variable-statistics"/> + <@message "sign-in"/> + +
    -
    + <#if type == "Dataschema">
    diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 79a97dc971..1b98ef4b0b 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -559,12 +559,12 @@ "help": "The complete translations will be exported in the Gettext format." } }, - "variables-count-enabled": "Variables statistics enabled", - "variables-count-enabled-help": "Show variables statistics summary in the administration section.", - "projects-count-enabled": "Projects statistics enabled", - "projects-count-enabled-help": "Show projects statistics summary in the administration section.", - "data-access-requests-count-enabled": "Data Access Requests statistics enabled", - "data-access-requests-count-enabled-help": "Show data access requests statistics summary in the administration section." + "variables-count-enabled": "Variables metrics enabled", + "variables-count-enabled-help": "Show variables counts in the administration section and the home page.", + "projects-count-enabled": "Projects metrics enabled", + "projects-count-enabled-help": "Show projects counts in the administration section.", + "data-access-requests-count-enabled": "Data Access Requests metrics enabled", + "data-access-requests-count-enabled-help": "Show data access requests counts in the administration section." }, "publish": { "label": "Publish", @@ -2582,7 +2582,7 @@ }, "entity-statistics-summary": { "title": "Statistics Summary", - "full-title": "Content Statistics Summary", + "full-title": "Content Metrics Summary", "Study": "Individual Studies", "HarmonizationStudy": "Harmonization Initiatives", "StudyDataset": "Collected Datasets", diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index 91ad8c6e36..ed8f76d6ca 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -559,12 +559,12 @@ "help": "Les traductions complètes seront exportées au format Gettext." } }, - "variables-count-enabled": "Statistiques des variables activées", - "variables-count-enabled-help": "Afficher le résumé des statistiques des variables dans la section d'administration.", - "projects-count-enabled": "Statistiques des projets activées", - "projects-count-enabled-help": "Afficher le résumé des statistiques des projets dans la section d'administration.", - "data-access-requests-count-enabled": "Statistiques des demandes d'accès aux données activées", - "data-access-requests-count-enabled-help": "Afficher le résumé des statistiques des demandes d'accès aux données dans la section d'administration." + "variables-count-enabled": "Métriques des variables activées", + "variables-count-enabled-help": "Afficher les nombres de variables dans la section d'administration.", + "projects-count-enabled": "Métriques des projets activées", + "projects-count-enabled-help": "Afficher les nombres de projets dans la section d'administration.", + "data-access-requests-count-enabled": "Métriques de demandes d'accès aux données activées", + "data-access-requests-count-enabled-help": "Afficher les nombres de demandes d'accès aux données dans la section d'administration." }, "publish": { "label": "Publier", @@ -2582,7 +2582,7 @@ }, "entity-statistics-summary": { "title": "Résumé des statistiques", - "full-title": "Résumé des statistiques de contenu", + "full-title": "Métriques du contenu", "Study": "Étude individuelles", "HarmonizationStudy": "Initiatives d'harmonisation", "StudyDataset": "Ensemble de données de collecte", From 340e6b7688b5528ad828daae4cd9f7762e12d879 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 13 Jan 2023 15:20:48 +0100 Subject: [PATCH 33/44] Fix some wording and showVariableStatistics template setting added --- .../src/main/resources/_templates/libs/variable-scripts.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl b/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl index 15b1e381d4..54532e836a 100644 --- a/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl @@ -119,7 +119,7 @@ }); - <#if user?? || !config.variableSummaryRequiresAuthentication> + <#if showVariableStatistics && (user?? || !config.variableSummaryRequiresAuthentication)> makeSummary(${showHarmonizedVariableSummarySelector?c}); From 0247a14e950348de8ecfe1c82c7d03364a849daf Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 18 Jan 2023 15:42:31 +0100 Subject: [PATCH 34/44] Fix harmonized variable id encoding for url --- .../src/main/resources/_templates/libs/dataset-scripts.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl b/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl index 4004441cc5..baa3e8bdbd 100644 --- a/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl @@ -106,7 +106,7 @@ if (!harmonizedVariable.status || harmonizedVariable.status.length === 0) { row.push(''); } else { - const url = harmonizedVariable.harmonizedVariableRef ? '../variable/' + harmonizedVariable.harmonizedVariableRef.id : '#'; + const url = harmonizedVariable.harmonizedVariableRef ? '../variable/' + encodeURIComponent(harmonizedVariable.harmonizedVariableRef.id) : '#'; row.push(''); } } From b7a4b31528ad28f39e850562d58814f6874e8002 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 18 Jan 2023 19:45:37 +0100 Subject: [PATCH 35/44] Fix harmonized variable id encoding for url for reverse proxy --- .../org/obiba/mica/dataset/domain/DatasetVariable.java | 10 ++++++---- .../obiba/mica/web/controller/VariableController.java | 6 ++++-- .../main/resources/_templates/libs/dataset-scripts.ftl | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index 0d4d939d62..f5ecfcf08b 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -425,7 +425,7 @@ private String cleanStringForSearch(String string) { } public static class IdEncoderDecoder { - private static Pattern encodePattern = Pattern.compile("&|\\||\\(|\\)|=|<|>|,"); + private static Pattern encodePattern = Pattern.compile("&|\\||\\(|\\)|=|<|>|,|/"); // Use underscore to make sure the RQLParser does not try to decode private static final Map encodeMap = Stream.of(new String[][] { { "&", "_26" }, @@ -435,10 +435,11 @@ public static class IdEncoderDecoder { { "=", "_3d" }, { "<", "_3c" }, { ">", "_3e" }, - { ",", "_2c" } + { ",", "_2c" }, + { "/", "_2f" } }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - private static Pattern decodePattern = Pattern.compile("(_26|_7c|_28|_29|_3d|_3c|_3e|_2c)"); + private static Pattern decodePattern = Pattern.compile("(_26|_7c|_28|_29|_3d|_3c|_3e|_2c|_2f)"); private static final Map decodeMap = Stream.of(new String[][] { { "_26", "&" }, { "_7c" , "|"}, @@ -447,7 +448,8 @@ public static class IdEncoderDecoder { { "_3d" , "="}, { "_3c" , "<"}, { "_3e" , ">"}, - { "_2c" , ","} + { "_2c" , ","}, + { "_2f" , "/"} }).collect(Collectors.toMap(data -> data[0], data -> data[1])); public static String encode(String value) { diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java index 5fb0cc14f0..9c54a1056a 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java @@ -69,7 +69,9 @@ public class VariableController extends BaseController { public ModelAndView variable(@PathVariable String id) { Map params = newParameters(); - DatasetVariable.IdResolver resolver = DatasetVariable.IdResolver.from(id); + String decodedId = DatasetVariable.IdEncoderDecoder.decode(id); + + DatasetVariable.IdResolver resolver = DatasetVariable.IdResolver.from(decodedId); String datasetId = resolver.getDatasetId(); String variableName = resolver.getName(); @@ -89,7 +91,7 @@ public ModelAndView variable(@PathVariable String id) { variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), resolver.getId(), variableName); } catch (NoSuchVariableException e) { // legacy variable id format - variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), id, variableName); + variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), decodedId, variableName); } } else variable = getDatasetVariable(resolver.getId(), variableName); diff --git a/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl b/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl index baa3e8bdbd..4004441cc5 100644 --- a/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/dataset-scripts.ftl @@ -106,7 +106,7 @@ if (!harmonizedVariable.status || harmonizedVariable.status.length === 0) { row.push(''); } else { - const url = harmonizedVariable.harmonizedVariableRef ? '../variable/' + encodeURIComponent(harmonizedVariable.harmonizedVariableRef.id) : '#'; + const url = harmonizedVariable.harmonizedVariableRef ? '../variable/' + harmonizedVariable.harmonizedVariableRef.id : '#'; row.push(''); } } From 38b5e7a943ef3aaaab7389a155ea2a46062b890d Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 19 Jan 2023 10:00:37 +0100 Subject: [PATCH 36/44] NoSuchTableSourceException added with their web service mapper --- .../StudyTableSourceServiceRegistry.java | 2 +- .../mica/dataset/service/DatasetService.java | 2 +- .../service/InvalidDatasetException.java | 30 ------------------- ...oSuchStudyTableSourceExceptionMapper.java} | 12 ++++---- .../rest/NoSuchValueTableExceptionMapper.java | 2 +- .../NoSuchStudyTableFileSourceException.java | 21 +++++++++++++ .../NoSuchStudyTableSourceException.java | 19 ++++++++++++ 7 files changed, 50 insertions(+), 38 deletions(-) delete mode 100644 mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java rename mica-rest/src/main/java/org/obiba/mica/dataset/rest/{InvalidDatasetExceptionMapper.java => NoSuchStudyTableSourceExceptionMapper.java} (61%) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableFileSourceException.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java index 4b3c87e324..4ca867e5d1 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -150,7 +150,7 @@ public InputStream getInputStream() { if (attachmentState.isPresent()) { return fileStoreService.getFile(attachmentState.get().getAttachment().getFileReference()); } else { - throw new NoSuchValueTableException("No value table at " + fullPath); + throw new NoSuchStudyTableFileSourceException("No such file at " + fullPath); } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 873d00495f..9eb04c2c22 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -139,7 +139,7 @@ protected Iterable wrappedGetDatasetVariables(T dataset) { try { return getDatasetVariables(dataset); } catch (NoSuchValueTableException e) { - throw new InvalidDatasetException(e); + throw e; } catch (MagmaRuntimeException e) { throw new DatasourceNotAvailableException(e); } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java deleted file mode 100644 index c4844e3a00..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.dataset.service; - -public class InvalidDatasetException extends RuntimeException { - - public InvalidDatasetException() { - super(); - } - - public InvalidDatasetException(String message) { - super(message); - } - - public InvalidDatasetException(Throwable e) { - super(e); - } - - public InvalidDatasetException(String message, Throwable e) { - super(message, e); - } -} diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java similarity index 61% rename from mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java rename to mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java index 2f512f3513..203e43a583 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java @@ -10,17 +10,19 @@ package org.obiba.mica.dataset.rest; +import org.obiba.mica.spi.tables.NoSuchStudyTableSourceException; + import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; -import org.obiba.mica.dataset.service.InvalidDatasetException; - @Provider -public class InvalidDatasetExceptionMapper implements ExceptionMapper { +public class NoSuchStudyTableSourceExceptionMapper implements ExceptionMapper { + @Override - public Response toResponse(InvalidDatasetException e) { - return Response.status(Status.CONFLICT).entity(e.getMessage()).build(); + public Response toResponse(NoSuchStudyTableSourceException exception) { + return Response.status(Status.NOT_FOUND).entity(exception.getMessage()).build(); } + } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java index 00a02cd95d..9d4ad180a0 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java @@ -22,7 +22,7 @@ public class NoSuchValueTableExceptionMapper implements ExceptionMapper. + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.magma.NoSuchValueTableException; + +public class NoSuchStudyTableFileSourceException extends NoSuchStudyTableSourceException { + + public NoSuchStudyTableFileSourceException(String s) { + super(s); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java new file mode 100644 index 0000000000..3318365424 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +public class NoSuchStudyTableSourceException extends RuntimeException { + + public NoSuchStudyTableSourceException(String s) { + super(s); + } + +} From 2f75ffe8108e170dcd16a8aa5590bf837eb050dd Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 19 Jan 2023 18:45:48 +0100 Subject: [PATCH 37/44] ng-obiba-mica latest release --- .../mica/core/upgrade/Mica520upgrade.java | 99 +++++++++++++++++++ mica-webapp/bower.json | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java new file mode 100644 index 0000000000..b1bf198ed1 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java @@ -0,0 +1,99 @@ +package org.obiba.mica.core.upgrade; + +import javax.inject.Inject; + +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.json.JSONException; +import org.obiba.mica.micaConfig.service.CacheService; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.runtime.Version; +import org.obiba.runtime.upgrade.UpgradeStep; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Component; + +@Component +public class Mica520upgrade implements UpgradeStep { + + private static final Logger logger = LoggerFactory.getLogger(Mica520upgrade.class); + + private final CacheService cacheService; + + private final MongoTemplate mongoTemplate; + + private final PluginsService pluginsService; + + @Inject + public Mica520upgrade(CacheService cacheService, MongoTemplate mongoTemplate, PluginsService pluginsService) { + this.cacheService = cacheService; + this.mongoTemplate = mongoTemplate; + this.pluginsService = pluginsService; + } + + @Override + public String getDescription() { + return "Clear caches, update config and search plugin for 5.2.0"; + } + + @Override + public Version getAppliesTo() { + return new Version(5, 1, 0); + } + + @Override + public void execute(Version currentVersion) { + logger.info("Executing Mica upgrade to version 5.2.0"); + + try { + logger.info("Updating 'Mica Config'..."); + updateMicaConfig(); + } catch (JSONException e) { + logger.error("Error occurred while Updating 'Mica Config'"); + } + + logger.info("Clearing all caches..."); + cacheService.clearAllCaches(); + + logger.info("Updating search plugin..."); + try { + updateSearchPlugin(); + // TODO rebuild search index? + } catch (Exception e) { + // not installed, not to be upgraded + // OR upgrade failed for unknown reason + logger.error("Error occurred while updating 'Search Plugin' to its latest version", e); + } + } + + private void updateMicaConfig() throws JSONException { + Document micaConfig = getDBObjectSafely("micaConfig"); + // delete field opal as it was not used + if (null != micaConfig) { + micaConfig.remove("opal"); + mongoTemplate.save(micaConfig, "micaConfig"); + } + } + + private void updateSearchPlugin() { + String searchPluginName = pluginsService.getSearchPluginName(); + // check it is installed + pluginsService.getInstalledPlugin(searchPluginName); + // check it is updatable + if (pluginsService.getUpdatablePlugins().stream().anyMatch(pkg -> pkg.getName().equals(searchPluginName))) { + // install latest version + pluginsService.installPlugin(searchPluginName, null); + } + } + + private Document getDBObjectSafely(String collectionName) { + if (mongoTemplate.collectionExists(collectionName)) { + MongoCollection existingCollection = mongoTemplate.getCollection(collectionName); + return existingCollection.find().first(); + } + + return null; + } + +} diff --git a/mica-webapp/bower.json b/mica-webapp/bower.json index 7bf0ca83da..eeaccb1177 100644 --- a/mica-webapp/bower.json +++ b/mica-webapp/bower.json @@ -30,7 +30,7 @@ "mica-study-timeline": "https://github.com/obiba/mica-study-timeline.git#1.1.4", "ng-file-upload": "12.2.13", "ng-obiba": "https://github.com/obiba/ng-obiba.git#2.0.2", - "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#master", + "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#4.4.0", "filesize": "3.6.0", "ngclipboard": "1.1.3", "angular-utils-pagination": "0.11.1", From d7389c338071ce01149897c4888aca98dc6f82a3 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 20 Jan 2023 08:49:10 +0100 Subject: [PATCH 38/44] prepare for upgrade to 5.2 --- .../mica/config/UpgradeConfiguration.java | 6 +- .../mica/core/upgrade/Mica510upgrade.java | 98 ------------------- .../mica/core/upgrade/Mica520upgrade.java | 2 +- 3 files changed, 4 insertions(+), 102 deletions(-) delete mode 100644 mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java diff --git a/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java b/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java index bafe87dea5..ad306e8e50 100644 --- a/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java +++ b/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java @@ -16,7 +16,7 @@ import org.obiba.mica.core.upgrade.Mica460Upgrade; import org.obiba.mica.core.upgrade.Mica500Upgrade; -import org.obiba.mica.core.upgrade.Mica510upgrade; +import org.obiba.mica.core.upgrade.Mica520upgrade; import org.obiba.mica.core.upgrade.MicaVersionModifier; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; import org.obiba.runtime.upgrade.UpgradeManager; @@ -31,8 +31,8 @@ public class UpgradeConfiguration { private List upgradeSteps; - public UpgradeConfiguration(Mica460Upgrade mica460Upgrade, Mica500Upgrade mica500Upgrade, Mica510upgrade mica510upgrade) { - upgradeSteps = Arrays.asList(mica460Upgrade, mica500Upgrade, mica510upgrade); + public UpgradeConfiguration(Mica460Upgrade mica460Upgrade, Mica500Upgrade mica500Upgrade, Mica520upgrade mica520upgrade) { + upgradeSteps = Arrays.asList(mica460Upgrade, mica500Upgrade, mica520upgrade); } @Bean diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java deleted file mode 100644 index c15c1cb293..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.obiba.mica.core.upgrade; - -import javax.inject.Inject; - -import com.mongodb.client.MongoCollection; -import org.bson.Document; -import org.json.JSONException; -import org.obiba.mica.micaConfig.service.CacheService; -import org.obiba.mica.micaConfig.service.PluginsService; -import org.obiba.plugins.PluginResources; -import org.obiba.runtime.Version; -import org.obiba.runtime.upgrade.UpgradeStep; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.stereotype.Component; - -import java.util.NoSuchElementException; - -@Component -public class Mica510upgrade implements UpgradeStep { - - private static final Logger logger = LoggerFactory.getLogger(Mica510upgrade.class); - - private final CacheService cacheService; - - private final MongoTemplate mongoTemplate; - - private final PluginsService pluginsService; - - @Inject - public Mica510upgrade(CacheService cacheService, MongoTemplate mongoTemplate, PluginsService pluginsService) { - this.cacheService = cacheService; - this.mongoTemplate = mongoTemplate; - this.pluginsService = pluginsService; - } - - @Override - public String getDescription() { - return "Clear caches and updating config for 5.1.0"; - } - - @Override - public Version getAppliesTo() { - return new Version(5, 1, 0); - } - - @Override - public void execute(Version currentVersion) { - logger.info("Executing Mica upgrade to version 5.1.0"); - - try { - logger.info("Updating 'Mica Config'..."); - updateMicaConfig(); - } catch (JSONException e) { - logger.error("Error occurred while Updating 'Mica Config'"); - } - - logger.info("Clearing all caches..."); - cacheService.clearAllCaches(); - - logger.info("Updating search plugin..."); - try { - String searchPluginName = pluginsService.getSearchPluginName(); - // check it is installed - pluginsService.getInstalledPlugin(searchPluginName); - // check it is updatable - if (pluginsService.getUpdatablePlugins().stream().anyMatch(pkg -> pkg.getName().equals(searchPluginName))) { - // install latest version - pluginsService.installPlugin(searchPluginName, null); - } - // TODO rebuild search index? - } catch (Exception e) { - // not installed, not to be upgraded - // OR upgrade failed for unknown reason - logger.error("Error occurred while Updating 'Search Plugin'", e); - } - } - - private void updateMicaConfig() throws JSONException { - Document micaConfig = getDBObjectSafely("micaConfig"); - // delete field anonymousCanCreateCart to reset to default - if (null != micaConfig) { - micaConfig.remove("opal"); - mongoTemplate.save(micaConfig, "micaConfig"); - } - } - - private Document getDBObjectSafely(String collectionName) { - if (mongoTemplate.collectionExists(collectionName)) { - MongoCollection existingCollection = mongoTemplate.getCollection(collectionName); - return existingCollection.find().first(); - } - - return null; - } - -} diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java index b1bf198ed1..cde8a72a37 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java @@ -39,7 +39,7 @@ public String getDescription() { @Override public Version getAppliesTo() { - return new Version(5, 1, 0); + return new Version(5, 2, 0); } @Override From 06e4d212fef8ddba57ed9d326f5527f18cdf13b1 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 20 Jan 2023 08:50:05 +0100 Subject: [PATCH 39/44] StudyTableSource does not need to return the table name, this is implementation specific --- .../java/org/obiba/mica/core/source/ExcelTableSource.java | 5 ----- .../java/org/obiba/mica/core/source/OpalTableSource.java | 1 - .../org/obiba/mica/core/source/ExcelTableSourceTest.java | 3 --- .../java/org/obiba/mica/spi/tables/StudyTableSource.java | 7 ------- 4 files changed, 16 deletions(-) diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java index 3882a6d30d..f3221ee35b 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -65,11 +65,6 @@ public String getPath() { return path; } - @Override - public String getTable() { - return table; - } - @Override public ValueTable getValueTable() { ensureInitialized(); diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java index ba0dc9be06..fb743d7d76 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -48,7 +48,6 @@ public void setProject(String project) { this.project = project; } - @Override public String getTable() { return table; } diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java index 08cbc329ef..f34d50ac7c 100644 --- a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java @@ -29,14 +29,11 @@ public void test_is_for() { public void test_urn_parse() { ExcelTableSource source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx:MyTable"); assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); - assertThat(source.getTable()).isEqualTo("MyTable"); source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx"); assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); - assertThat(source.getTable()).isNullOrEmpty(); source = ExcelTableSource.fromURN("urn:file:/path/to/MyProject.xlsx:MyTable"); assertThat(source.getPath()).isEqualTo("/path/to/MyProject.xlsx"); - assertThat(source.getTable()).isEqualTo("MyTable"); } } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java index 3e29028796..deece69d60 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java @@ -19,13 +19,6 @@ */ public interface StudyTableSource { - /** - * Get the associated table name. - * - * @return - */ - String getTable(); - /** * Get the {@link ValueTable} implementing the data dictionary and the data values. * From 84be7b2b31673309304c13880e5edbc68dd149a0 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 20 Jan 2023 08:52:44 +0100 Subject: [PATCH 40/44] Modifying the meta txonomy requires admin role --- .../org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java index 52dd9ac737..42f26b84f9 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java @@ -12,7 +12,9 @@ import com.codahale.metrics.annotation.Timed; +import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.service.TaxonomiesService; +import org.obiba.mica.security.Roles; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.taxonomy.Dtos; @@ -37,6 +39,7 @@ public Opal.TaxonomyDto getTaxonomy() { @PUT @Path("/{target}/{taxonomy}/_move") + @RequiresRoles(Roles.MICA_ADMIN) public Response up(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("dir") String dir) { taxonomiesService.moveTaxonomy(TaxonomyTarget.fromId(target), taxonomy, "up".equals(dir)); return Response.ok().build(); @@ -44,6 +47,7 @@ public Response up(@PathParam("target") @DefaultValue("variable") String target, @PUT @Path("/{target}/{taxonomy}/_attribute") + @RequiresRoles(Roles.MICA_ADMIN) public Response hide(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("name") String name, @QueryParam("value") String value) { taxonomiesService.setTaxonomyAttribute(TaxonomyTarget.fromId(target), taxonomy, name, value); return Response.ok().build(); From 2f04954beff9747d28cf81792980771d6c2495c2 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 20 Jan 2023 10:24:32 +0100 Subject: [PATCH 41/44] Plugin management requires admin role --- .../java/org/obiba/mica/micaConfig/rest/PluginResource.java | 3 +++ .../java/org/obiba/mica/micaConfig/rest/PluginsResource.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java index f969f74cee..eef19a8fac 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java @@ -12,7 +12,9 @@ import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.security.Roles; import org.obiba.mica.web.model.MicaPlugins; import org.obiba.mica.web.model.PluginDtos; import org.springframework.context.annotation.Scope; @@ -26,6 +28,7 @@ @Scope("request") @Path("/config/plugin/{name}") @RequiresAuthentication +@RequiresRoles(Roles.MICA_ADMIN) public class PluginResource { @Inject diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java index 28917508aa..6b425e41ab 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java @@ -13,7 +13,9 @@ import com.google.common.base.Strings; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.security.Roles; import org.obiba.mica.web.model.MicaPlugins; import org.obiba.mica.web.model.PluginDtos; import org.springframework.stereotype.Component; @@ -29,6 +31,7 @@ @Component @Path("/config/plugins") @RequiresAuthentication +@RequiresRoles(Roles.MICA_ADMIN) public class PluginsResource { @Inject From 79377326ae2aeb9af5c129f116f41d5586eda13c Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Fri, 20 Jan 2023 10:55:06 +0100 Subject: [PATCH 42/44] Clean Opal projects web services --- .../network/rest/DraftNetworkResource.java | 23 +------------------ .../mica/study/rest/StudyStateResource.java | 2 +- .../webapp/app/dataset/dataset-controller.js | 12 +++++----- .../webapp/app/dataset/dataset-service.js | 4 ++-- .../webapp/app/network/network-service.js | 7 ------ 5 files changed, 10 insertions(+), 38 deletions(-) diff --git a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java index 758b94bd10..929dff76b4 100644 --- a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java @@ -11,8 +11,6 @@ package org.obiba.mica.network.rest; import com.google.common.base.Strings; - -import org.apache.commons.compress.utils.Lists; import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; import org.obiba.mica.NoSuchEntityException; @@ -21,7 +19,6 @@ import org.obiba.mica.core.service.AbstractGitPersistableService; import org.obiba.mica.file.Attachment; import org.obiba.mica.file.rest.FileResource; -import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.NoSuchNetworkException; import org.obiba.mica.network.domain.Network; import org.obiba.mica.network.domain.NetworkState; @@ -29,7 +26,6 @@ import org.obiba.mica.security.rest.SubjectAclResource; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Projects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -43,13 +39,11 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; -import java.net.URISyntaxException; import java.util.HashMap; -import java.util.List; import java.util.Map; /** - * REST controller for managing draft Study. + * REST controller for managing draft Network. */ @Component @Scope("request") @@ -66,9 +60,6 @@ public class DraftNetworkResource extends AbstractGitPersistableResource projects() throws URISyntaxException { - checkPermission("/draft/network", "VIEW"); - try { - return opalService.getProjectDtos(networkService.findById(id).getOpal()); - } catch (Exception e) { - log.warn("Failed at retrieving opal projects: {}", e.getMessage()); - return Lists.newArrayList(); - } - } - @Override protected String getId() { return id; diff --git a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java index 8f74a31bcf..30440d7aca 100644 --- a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java @@ -70,7 +70,7 @@ public Mica.StudySummaryDto get() { } @GET - @Path("/projects") + @Path("/opal-projects") public List projects() throws URISyntaxException { subjectAclService.checkPermission(resource, "VIEW", id); String opalUrl = Optional.ofNullable(studyService.findStudy(id)).map(BaseStudy::getOpal).orElse(null); diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js index 074e1d49b6..2cddbef6a7 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js @@ -37,7 +37,7 @@ mica.dataset 'MicaConfigResource', 'FormServerValidation', 'StudyStatesResource', - 'StudyStateProjectsResource', + 'StudyStateOpalProjectsResource', 'FormDirtyStateObserver', 'EntityFormResource', 'SfOptionsService', @@ -57,7 +57,7 @@ mica.dataset MicaConfigResource, FormServerValidation, StudyStatesResource, - StudyStateProjectsResource, + StudyStateOpalProjectsResource, FormDirtyStateObserver, EntityFormResource, SfOptionsService, @@ -175,7 +175,7 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise + StudyStateOpalProjectsResource.query({id: $scope.selected.study.id}).$promise .then(function (projects) { $scope.projects = projects; var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === $scope.studyTable.project; })[0]; @@ -849,7 +849,7 @@ mica.dataset '$filter', 'MicaConfigResource', 'StudyStatesResource', - 'StudyStateProjectsResource', + 'StudyStateOpalProjectsResource', 'LocalizedValues', 'LocalizedSchemaFormService', 'isCommentsRequiredOnDocumentSave', @@ -862,7 +862,7 @@ mica.dataset $filter, MicaConfigResource, StudyStatesResource, - StudyStateProjectsResource, + StudyStateOpalProjectsResource, LocalizedValues, LocalizedSchemaFormService, isCommentsRequiredOnDocumentSave, @@ -958,7 +958,7 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise + StudyStateOpalProjectsResource.query({id: $scope.selected.study.id}).$promise .then(function (projects) { $scope.projects = projects; var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === table.project; })[0]; diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js index 4f4da3916d..bf26039542 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js @@ -48,9 +48,9 @@ mica.dataset }); }]) - .factory('StudyStateProjectsResource', ['$resource', + .factory('StudyStateOpalProjectsResource', ['$resource', function ($resource) { - return $resource(contextPath + '/ws/draft/study-state/:id/projects', {}, { + return $resource(contextPath + '/ws/draft/study-state/:id/opal-projects', {}, { 'get': {method: 'GET', params: {id: '@id'}} }); }]) diff --git a/mica-webapp/src/main/webapp/app/network/network-service.js b/mica-webapp/src/main/webapp/app/network/network-service.js index 0798513b3c..d74332c000 100644 --- a/mica-webapp/src/main/webapp/app/network/network-service.js +++ b/mica-webapp/src/main/webapp/app/network/network-service.js @@ -37,13 +37,6 @@ mica.network }); }]) - .factory('DraftNetworkProjectsResource', ['$resource', - function ($resource) { - return $resource(contextPath + '/ws/draft/network/:id/projects', {}, { - 'get': {method: 'GET', errorHandler: true} - }); - }]) - .factory('DraftNetworkPermissionsResource', ['$resource', function ($resource) { return $resource(contextPath + '/ws/draft/network/:id/permissions', {}, { From 99d259126013aa0ff6ef61bd0fc24debfe9608ce Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Thu, 26 Jan 2023 12:54:46 +0100 Subject: [PATCH 43/44] Helper class to read a taxonomy in YAML format from a stream --- .../AbstractTaxonomiesProviderService.java | 34 +++++++++++++++++++ .../taxonomies/TaxonomyImportException.java | 19 +++++++++++ 2 files changed, 53 insertions(+) create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java index 44c55b9c29..b8aaf31d90 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java @@ -10,6 +10,13 @@ package org.obiba.mica.spi.taxonomies; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.support.yaml.TaxonomyYaml; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; import java.util.Properties; public abstract class AbstractTaxonomiesProviderService implements TaxonomiesProviderService { @@ -42,4 +49,31 @@ public void start() { public void stop() { this.running = false; } + + /** + * Read a taxonomy in YAML format from a URL stream. + * + * @param uri + * @return + * @throws TaxonomyImportException + */ + protected Taxonomy readTaxonomy(URL uri) throws TaxonomyImportException { + try (InputStream input = uri.openStream()) { + return readTaxonomy(input); + } catch (Exception e) { + throw new TaxonomyImportException(e); + } + } + + /** + * Read a taxonomy in YAML format from a stream. + * + * @param input + * @return + * @throws TaxonomyImportException + */ + protected Taxonomy readTaxonomy(InputStream input) { + TaxonomyYaml yaml = new TaxonomyYaml(); + return yaml.load(input); + } } diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java new file mode 100644 index 0000000000..84d15c5c31 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +public class TaxonomyImportException extends RuntimeException { + + public TaxonomyImportException(Throwable error) { + super(error); + } + +} From d7e500f85ee7ed1dc925686e9786775117a7c846 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Mon, 30 Jan 2023 21:10:01 +0100 Subject: [PATCH 44/44] Other study table source handled in the admin --- mica-webapp/src/main/resources/i18n/en.json | 10 ++++++- mica-webapp/src/main/resources/i18n/fr.json | 10 ++++++- .../webapp/app/dataset/dataset-controller.js | 14 +++++++--- .../webapp/app/dataset/dataset-service.js | 16 +++++++++--- .../views/collected-dataset-view-details.html | 22 ++++++++++++++++ .../views/harmonization-dataset-form.html | 23 ++++++++++++++++ .../harmonization-study-tables-form.html | 9 +++---- .../harmonized-dataset-view-details.html | 21 +++++++++++++++ .../dataset/views/opal-table-modal-form.html | 26 +++++++++++++++++++ 9 files changed, 137 insertions(+), 14 deletions(-) diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index 1b98ef4b0b..61508c24e6 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -967,6 +967,13 @@ "title": "File Source", "info": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the dataset's folder, then in the associated study's folder. The table name is optional if there is only one described in the file.", "info-schema": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the harmonization protocol's folder, then in the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." + }, + "other": { + "title": "Other Source", + "info": "Generic study table source definition.", + "info-schema": "Generic data schema source definition.", + "nid": "Namespace identifier", + "nss": "Specific source name in the namespace" } }, "delete-dialog": { @@ -991,7 +998,8 @@ "info": "A study table defines which study data collection event it applies to and which Opal table the variables and data summaries are to be extracted from.", "source": { "opal": "Opal source", - "file": "File source" + "file": "File source", + "other": "Other source" } }, "harmonization-study-table": { diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index ed8f76d6ca..aba446266b 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -967,6 +967,13 @@ "title": "Fichier source", "info": "Fichier duquel les variables de la table seront extraites. Si le chemin est relatif, le fichier est d'abord cherché dans le dossier de l'ensemble de données, puis dans celui de l'étude associée. Le nom de la table est optionel si le fichier ne décrit qu'une table.", "info-schema": "Fichier duquel les variables de la table seront extraites. Le chemin du fichier peut-être relatif, le fichier est d'abord cherché dans le dossier du protocole d'harmonisation, puis dans celui de l'initiative d'harmonisation associée. Le nom de la table est optionel si le fichier ne décrit qu'une table." + }, + "other": { + "title": "Autre source", + "info": "Définition générique de la source de la table.", + "info-schema": "Définition générique de la source du schéma de données.", + "nid": "Identifiant de l'espace de nommage", + "nss": "Nom spécifique de la source dans l'espace de nommage" } }, "delete-dialog": { @@ -991,7 +998,8 @@ "info": "La table d'étude définie à quel événement de collecte elle s'applique et à partir de quelle table Opal il faut extraire les variables et les données sommaires.", "source": { "opal": "Opal source", - "file": "Fichier source" + "file": "Fichier source", + "other": "Autre source" } }, "harmonization-study-table": { diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js index 2cddbef6a7..af430f2d90 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js @@ -411,7 +411,8 @@ mica.dataset $scope.NAMESPACES = [ 'opal', - 'file' + 'file', + 'other' ]; }]) @@ -1002,7 +1003,7 @@ mica.dataset populationId: populationId, dataCollectionEventId: dceId }); - } else { + } else if ($scope.table.namespace === 'opal') { angular.extend($scope.table, { studyId: $scope.selected.study.id, populationId: populationId, @@ -1010,6 +1011,12 @@ mica.dataset project: $scope.selected.project.name, table: $scope.selected.project.table }); + } else { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId + }); } $scope.table.name = LocalizedValues.objectToArray($scope.table.model.name); @@ -1031,7 +1038,8 @@ mica.dataset $scope.NAMESPACES = [ 'opal', - 'file' + 'file', + 'other' ]; }]); diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js index bf26039542..5887ba4139 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js @@ -437,12 +437,16 @@ mica.dataset if (result.namespace === 'file') { result.source = 'urn:file:' + result.path + (result.table ? ':' + result.table : ''); - delete result.path; - } else { + } else if (result.namespace === 'opal') { result.source = 'urn:opal:' + result.project + '.' + result.table; - delete result.project; + } else { + result.source = 'urn:' + result.nid + ':' + result.nss; } + delete result.path; + delete result.project; delete result.table; + delete result.nid; + delete result.nss; delete result.namespace; return result; @@ -481,7 +485,13 @@ mica.dataset result.namespace = 'file'; result.path = id.indexOf(':') > 0 ? id.substring(0, id.indexOf(':')) : id; result.table = id.indexOf(':') > 0 ? id.substring(id.indexOf(':') + 1) : undefined; + } else { + result.namespace = 'other'; } + const id = result.source.replace('urn:', ''); + const sepIdx = id.indexOf(':'); + result.nid = id.substring(0, sepIdx); + result.nss = id.substring(sepIdx + 1); } else { result.namespace = 'opal'; } diff --git a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html index d86f75fbcc..1d7b7ef6da 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html @@ -133,6 +133,28 @@

    dataset.datasource.file.title

    +
    +

    dataset.datasource.other.title

    +

    dataset.datasource.other.info

    + + + + + + + + + + + + +
    dataset.datasource.other.nid + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.nid}} +
    dataset.datasource.other.nss + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.nss}} +
    +
    +
    diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html index fd8e28989c..769912ad73 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html @@ -107,6 +107,29 @@

    dataset.data-schema.title

    +
    +

    dataset.datasource.other.info-schema

    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    +

    dataset.harmonization-study.title

    dataset.harmonized-tables.title study.label type - dataset.project / path - dataset.table + dataset.datasource.title dataset.table-name dataset.table-description actions @@ -46,10 +45,8 @@

    dataset.harmonized-tables.title

    {{(wrapper.type === 'studyTable' ? 'search.study.individual' : 'search.study.harmonization') | translate}} - {{wrapper.table.namespace === 'opal' ? wrapper.table.project : wrapper.table.path}} - - - {{wrapper.table.table}} + {{wrapper.table.nid}} + {{wrapper.table.nss}}
      diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html index b890135de3..b9d24a3536 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html @@ -60,6 +60,27 @@

      dataset.datasource.file.title

    +
    +

    dataset.datasource.other.title

    + + + + + + + + + + + + +
    dataset.datasource.other.nid + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.nid}} +
    dataset.datasource.other.nss + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.nss}} +
    +
    +

    dataset.harmonization-study.title

    dataset.harmonization-study.info

    diff --git a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html index 31205b2615..9716140040 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html @@ -90,6 +90,32 @@

    dataset.datasource.title

    +
    +

    dataset.datasource.other.info

    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    +

    dataset.options