From 2101d87ab2990e938d86f4e37b4fb2935fdb61f9 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 1 Feb 2023 16:38:51 +0100 Subject: [PATCH] Mica without Opal (#4393) * #4376 WIP. StudyTableSource interface added. New property sourceURN replaces opal project/table in dataset's study tables and variables. Requires ES index config update. * sourceURN property/getter/setter renamed to source * source package added, with unit tests * Excel table source added, reads excel from Mica's file system * Edit/create study tables with opal or file source. WIP. * File path can be copied. Study table's file source can be relative to the dataset's folder * Study table context added. Opal dtos (aggregations) are not exposed in the source plugin interface. * Deferred init of excel datasource. Less verbose log messages when study table source operation is not supported * SPI mica-source testing * Deferred attachment stream extraction * Handle disposable table sources * Mica source plugin could be an Initialisable * #4381 Variable taxonomies service added to support other taxonomies providers, in addition to the opal one * mica-source plugin type renamed to mica-tables. Ne plugin type: mica-taxonomies for loading taxonomies provider services. * Taxonomy of taxonomies is configurable. Added attribute to have a taxonomy declared but not visible. * Variable taxonomies can be read from local folder: MICA_HOME/conf/taxonomies/variable * AbstractTaxonomiesProviderService added * Caching of taxonomies refactored, not functional yet * Variable taxonomies are cached * Fix dataset taxonomy init * Opal taxonomies cache replaced by generic variable taxonomies cache. Variable taxonomies are decorated with variable attributes * Code cleaning: Opal service helper merged into opal service * classifications administration * make sure variable taxo title/description in the meta taxo matches the one of the original taxo * hide instead of remove a target vocabulary if type is disabled * Plugins REST API added * Plugin REST API added * Service Plugin REST API added * Update the search plugin in the upgrade procedure * ng-obiba-mica version * No server error when opal projects cannot be retrieved * Fix some wording and showVariableStatistics template setting added * Fix some wording and showVariableStatistics template setting added * Fix harmonized variable id encoding for url * Fix harmonized variable id encoding for url for reverse proxy * NoSuchTableSourceException added with their web service mapper * ng-obiba-mica latest release * prepare for upgrade to 5.2 * StudyTableSource does not need to return the table name, this is implementation specific * Modifying the meta txonomy requires admin role * Plugin management requires admin role * Clean Opal projects web services * Helper class to read a taxonomy in YAML format from a stream * Other study table source handled in the admin --- mica-core/pom.xml | 4 + .../service/DataAccessEntityService.java | 4 +- .../mica/config/UpgradeConfiguration.java | 6 +- .../mica/core/domain/BaseStudyTable.java | 82 ++- .../org/obiba/mica/core/domain/OpalTable.java | 88 --- .../obiba/mica/core/domain/StudyTable.java | 6 +- .../StudyTableSourceServiceRegistry.java | 169 ++++++ .../mica/core/source/ExcelTableSource.java | 95 +++ .../mica/core/source/OpalTableSource.java | 131 +++++ .../mica/core/source/support/OpalDtos.java | 280 +++++++++ .../source}/support/QueryTermsUtil.java | 28 +- ...rceReader.java => YamlResourceReader.java} | 12 +- .../mica/core/upgrade/Mica510upgrade.java | 40 -- .../mica/core/upgrade/Mica520upgrade.java | 99 ++++ .../obiba/mica/dataset/domain/Dataset.java | 5 +- .../mica/dataset/domain/DatasetCategory.java | 5 +- .../mica/dataset/domain/DatasetVariable.java | 122 ++-- .../dataset/domain/HarmonizationDataset.java | 15 +- .../service/CollectedDatasetService.java | 58 +- .../mica/dataset/service/DatasetService.java | 187 +----- .../service/HarmonizedDatasetService.java | 187 ++---- .../service/InvalidDatasetException.java | 30 - .../dataset/service/VariableSetService.java | 47 +- .../mica/micaConfig/domain/MicaConfig.java | 12 +- ...va => VariableTaxonomiesUpdatedEvent.java} | 18 +- .../mica/micaConfig/service/CacheService.java | 29 +- .../micaConfig/service/MicaConfigService.java | 29 - .../service/MicaConfigurationProvider.java | 12 +- .../mica/micaConfig/service/OpalService.java | 250 +++----- .../micaConfig/service/PluginsService.java | 285 +++++++-- .../micaConfig/service/TaxonomiesService.java | 539 ++++++++++++++++++ .../service/TaxonomyConfigService.java | 6 +- .../micaConfig/service/TaxonomyService.java | 327 ----------- .../service/VariableTaxonomiesService.java | 166 ++++++ .../service/helper/OpalServiceHelper.java | 107 ---- .../obiba/mica/study/domain/BaseStudy.java | 4 +- .../org/obiba/mica/web/model/DatasetDtos.java | 403 ++++--------- .../java/org/obiba/mica/web/model/Dtos.java | 20 +- .../obiba/mica/web/model/MicaConfigDtos.java | 3 +- .../org/obiba/mica/web/model/PluginDtos.java | 78 +++ mica-core/src/main/resources/ehcache.xml | 10 +- .../core/source/ExcelTableSourceTest.java | 39 ++ .../mica/core/source/OpalTableSourceTest.java | 25 + ...rTest.java => YamlResourceReaderTest.java} | 6 +- .../service/CollectedDatasetServiceTest.java | 4 +- .../service/HarmonizedDatasetServiceTest.java | 5 +- .../support/HarmonizedDatasetHelperTest.java | 6 +- .../obiba/mica/web/model/DatasetDtosTest.java | 4 +- ...DatasourceNotAvailableExceptionMapper.java | 2 +- ...oSuchStudyTableSourceExceptionMapper.java} | 12 +- .../rest/NoSuchValueTableExceptionMapper.java | 2 +- .../rest/PublishedDatasetResource.java | 6 +- .../DraftCollectedDatasetResource.java | 17 - ...DraftCollectedDatasetVariableResource.java | 21 +- .../entity/rql/RQLCriteriaOpalConverter.java | 4 +- .../rest/entity/rql/RQLFieldReferences.java | 2 +- ...raftDataschemaDatasetVariableResource.java | 35 -- .../DraftHarmonizedDatasetResource.java | 23 - ...raftHarmonizedDatasetVariableResource.java | 34 +- .../DraftDatasetVariableResource.java | 3 +- .../micaConfig/rest/MicaConfigResource.java | 45 +- .../mica/micaConfig/rest/PluginResource.java | 84 +++ .../mica/micaConfig/rest/PluginsResource.java | 74 +++ .../network/rest/DraftNetworkResource.java | 21 +- .../mica/study/rest/StudyStateResource.java | 15 +- .../rest/AbstractTaxonomyResource.java | 139 +++++ .../taxonomy/rest/MetaTaxonomyResource.java | 56 ++ .../taxonomy/rest/TaxonomiesResource.java | 29 +- .../mica/taxonomy/rest/TaxonomyResource.java | 27 +- .../obiba/mica/web/rest/CacheResource.java | 7 +- .../rest/IllegalArgumentExceptionMapper.java | 2 +- .../AbstractPublishedDatasetResource.java | 89 ++- ...ishedCollectedDatasetVariableResource.java | 31 +- .../harmonization/CsvContingencyWriter.java | 5 +- .../CsvHarmonizationVariablesWriter.java | 5 +- ...shedDataschemaDatasetVariableResource.java | 97 +--- .../PublishedHarmonizedDatasetResource.java | 7 +- ...shedHarmonizedDatasetVariableResource.java | 73 +-- .../PublishedDatasetVariableResource.java | 4 +- .../mica/search/CoverageQueryExecutor.java | 13 +- .../obiba/mica/search/JoinQueryExecutor.java | 16 +- ...ConfigurationTaxonomyMetaDataProvider.java | 4 +- .../DatasetTaxonomyMetaDataProvider.java | 2 +- .../NetworkTaxonomyMetaDataProvider.java | 2 +- .../StudyTaxonomyMetaDataProvider.java | 2 +- .../TaxonomyAggregationMetaDataProvider.java | 34 +- .../VariableTaxonomyMetaDataProvider.java | 2 +- .../search/queries/AbstractDocumentQuery.java | 4 +- .../mica/search/queries/DatasetQuery.java | 4 +- .../mica/search/queries/NetworkQuery.java | 4 +- .../obiba/mica/search/queries/StudyQuery.java | 4 +- .../mica/search/queries/VariableQuery.java | 26 +- .../obiba/mica/taxonomy/TaxonomyIndexer.java | 34 +- .../rest/AbstractTaxonomySearchResource.java | 95 +-- .../rest/TaxonomiesSearchResource.java | 49 +- .../mica/spi/search/SearchEngineService.java | 2 +- .../spi/tables/AbstractStudyTableSource.java | 50 ++ .../AbstractStudyTableSourceService.java | 48 ++ .../org/obiba/mica/spi/tables/ICategory.java | 32 ++ .../org/obiba/mica/spi/tables/IDataset.java | 29 + .../org/obiba/mica/spi/tables/IStudy.java | 35 ++ .../org/obiba/mica/spi/tables/IVariable.java | 55 ++ .../NoSuchStudyTableFileSourceException.java | 21 + .../NoSuchStudyTableSourceException.java | 19 + .../mica/spi/tables/StudyTableContext.java | 38 ++ .../mica/spi/tables/StudyTableFileSource.java | 32 ++ .../tables/StudyTableFileStreamProvider.java | 27 + .../mica/spi/tables/StudyTableSource.java | 74 +++ .../spi/tables/StudyTableSourceService.java | 35 ++ .../tables/StudyTableSourceServiceLoader.java | 28 + .../AbstractTaxonomiesProviderService.java | 79 +++ .../taxonomies/TaxonomiesProviderService.java | 25 + .../TaxonomiesProviderServiceLoader.java | 28 + .../taxonomies/TaxonomyImportException.java | 19 + mica-web-model/src/main/protobuf/Mica.proto | 13 +- .../src/main/protobuf/MicaPlugins.proto | 49 ++ mica-webapp/bower.json | 2 +- .../controller/DatasetAnalysisController.java | 35 +- .../web/controller/VariableController.java | 31 +- .../src/main/resources/_templates/dataset.ftl | 2 +- .../resources/_templates/libs/settings.ftl | 1 + .../_templates/libs/variable-scripts.ftl | 2 +- .../main/resources/_templates/variable.ftl | 50 +- mica-webapp/src/main/resources/i18n/en.json | 58 +- mica-webapp/src/main/resources/i18n/fr.json | 58 +- .../main/webapp/app/admin/admin-controller.js | 105 +++- .../src/main/webapp/app/admin/admin-router.js | 7 + .../webapp/app/admin/views/admin-view.html | 28 +- .../main/webapp/app/admin/views/caching.html | 6 +- .../app/admin/views/classifications.html | 37 ++ .../webapp/app/config/views/config-form.html | 3 +- .../webapp/app/dataset/dataset-controller.js | 141 +++-- .../webapp/app/dataset/dataset-service.js | 68 ++- .../views/collected-dataset-view-details.html | 78 ++- .../views/dataset-study-table-form.html | 4 +- .../views/harmonization-dataset-form.html | 113 +++- .../harmonization-study-tables-form.html | 9 +- .../harmonized-dataset-view-details.html | 81 ++- .../dataset/views/opal-table-modal-form.html | 110 +++- .../app/file-system/file-system-controller.js | 13 +- .../views/documents-table-template.html | 5 + .../views/file-system-template.html | 5 + .../webapp/app/network/network-service.js | 7 - .../src/main/webapp/app/search/search.js | 7 - .../views/classifications-view-header.html | 19 - .../app/search/views/search-view-header.html | 19 - mica-webapp/src/main/webapp/app/services.js | 2 +- .../src/main/webapp/assets/js/mica-search.js | 14 +- .../main/webapp/assets/js/mica-variable.js | 2 +- pom.xml | 7 + 150 files changed, 4633 insertions(+), 2587 deletions(-) delete mode 100644 mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java rename mica-core/src/main/java/org/obiba/mica/{dataset/service => core/source}/support/QueryTermsUtil.java (77%) rename mica-core/src/main/java/org/obiba/mica/core/support/{YamlClassPathResourceReader.java => YamlResourceReader.java} (53%) delete mode 100644 mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java create mode 100644 mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java delete mode 100644 mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java rename mica-core/src/main/java/org/obiba/mica/micaConfig/event/{OpalTaxonomiesUpdatedEvent.java => VariableTaxonomiesUpdatedEvent.java} (51%) create mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java delete mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java create mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java delete mode 100644 mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java create mode 100644 mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java create mode 100644 mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java create mode 100644 mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java rename mica-core/src/test/java/org/obiba/mica/core/support/{YamlClassPathResourceReaderTest.java => YamlResourceReaderTest.java} (80%) rename mica-rest/src/main/java/org/obiba/mica/dataset/rest/{InvalidDatasetExceptionMapper.java => NoSuchStudyTableSourceExceptionMapper.java} (61%) create mode 100644 mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java create mode 100644 mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java create mode 100644 mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java create mode 100644 mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableFileSourceException.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java create mode 100644 mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java create mode 100644 mica-web-model/src/main/protobuf/MicaPlugins.proto create mode 100644 mica-webapp/src/main/webapp/app/admin/views/classifications.html delete mode 100644 mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html delete mode 100644 mica-webapp/src/main/webapp/app/search/views/search-view-header.html diff --git a/mica-core/pom.xml b/mica-core/pom.xml index fb0bc56112..962b8e1cbb 100644 --- a/mica-core/pom.xml +++ b/mica-core/pom.xml @@ -34,6 +34,10 @@ org.obiba.mica mica-web-model + + org.obiba.magma + magma-datasource-excel + com.codahale.metrics metrics-core diff --git a/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java b/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java index 87f901ecd4..c84d115293 100644 --- a/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java +++ b/mica-core/src/main/java/org/obiba/mica/access/service/DataAccessEntityService.java @@ -47,7 +47,7 @@ import org.obiba.mica.core.service.MailService; import org.obiba.mica.core.service.SchemaFormContentFileService; import org.obiba.mica.core.support.IdentifierGenerator; -import org.obiba.mica.core.support.YamlClassPathResourceReader; +import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.dataset.service.VariableSetService; import org.obiba.mica.micaConfig.domain.DataAccessConfig; import org.obiba.mica.micaConfig.service.DataAccessConfigService; @@ -396,7 +396,7 @@ protected void setAndLogStatus(T request, DataAccessEntityStatus to) { protected String generateId() { DataAccessConfig dataAccessConfig = dataAccessConfigService.getOrCreateConfig(); - Object exclusions = YamlClassPathResourceReader.read(EXCLUSION_IDS_YAML_RESOURCE_PATH, Map.class).get("exclusions"); + Object exclusions = YamlResourceReader.readClassPath(EXCLUSION_IDS_YAML_RESOURCE_PATH, Map.class).get("exclusions"); IdentifierGenerator.Builder builder = IdentifierGenerator.newBuilder().prefix(dataAccessConfig.getIdPrefix()) .size(dataAccessConfig.getIdLength()); diff --git a/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java b/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java index bafe87dea5..ad306e8e50 100644 --- a/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java +++ b/mica-core/src/main/java/org/obiba/mica/config/UpgradeConfiguration.java @@ -16,7 +16,7 @@ import org.obiba.mica.core.upgrade.Mica460Upgrade; import org.obiba.mica.core.upgrade.Mica500Upgrade; -import org.obiba.mica.core.upgrade.Mica510upgrade; +import org.obiba.mica.core.upgrade.Mica520upgrade; import org.obiba.mica.core.upgrade.MicaVersionModifier; import org.obiba.mica.core.upgrade.RuntimeVersionProvider; import org.obiba.runtime.upgrade.UpgradeManager; @@ -31,8 +31,8 @@ public class UpgradeConfiguration { private List upgradeSteps; - public UpgradeConfiguration(Mica460Upgrade mica460Upgrade, Mica500Upgrade mica500Upgrade, Mica510upgrade mica510upgrade) { - upgradeSteps = Arrays.asList(mica460Upgrade, mica500Upgrade, mica510upgrade); + public UpgradeConfiguration(Mica460Upgrade mica460Upgrade, Mica500Upgrade mica500Upgrade, Mica520upgrade mica520upgrade) { + upgradeSteps = Arrays.asList(mica460Upgrade, mica500Upgrade, mica520upgrade); } @Bean diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java index b85fb007f5..51b9cb5169 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/BaseStudyTable.java @@ -12,10 +12,9 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Strings; +import org.obiba.mica.core.source.OpalTableSource; -import java.io.Serializable; - -public class BaseStudyTable extends OpalTable { +public class BaseStudyTable { protected String studyId; @@ -23,6 +22,22 @@ public class BaseStudyTable extends OpalTable { protected int populationWeight; + private LocalizedString name; + + private LocalizedString description; + + private LocalizedString additionalInformation; + + private int weight; + + private String source; + + // legacy + private String project; + + // legacy + private String table; + public String getStudyId() { return studyId; } @@ -72,13 +87,66 @@ public void setPopulationWeight(int populationWeight) { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("project", getProject()).add("table", getTable()) + return MoreObjects.toStringHelper(this).add("source", getSource()) .add("studyId", getStudyId()).add("populationId", getPopulationId()) .toString(); } - @Override - protected String getEntityId() { - return studyId; + public void setName(LocalizedString name) { + this.name = name; + } + + public LocalizedString getName() { + return name; + } + + public void setDescription(LocalizedString description) { + this.description = description; + } + + public LocalizedString getDescription() { + return description; + } + + public LocalizedString getAdditionalInformation() { + return additionalInformation; + } + + public void setAdditionalInformation(LocalizedString additionalInformation) { + this.additionalInformation = additionalInformation; + } + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } + + public boolean isFor(String studyId, String source) { + return this.studyId.equals(studyId) && getSource().equals(source); + } + + public void setSource(String source) { + this.source = source; + } + + public String getSource() { + // legacy + if (Strings.isNullOrEmpty(source)) { + this.source = OpalTableSource.newSource(project, table).getURN(); + } + return source; + } + + @Deprecated + public void setProject(String project) { + this.project = project; + } + + @Deprecated + public void setTable(String table) { + this.table = table; } } diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java deleted file mode 100644 index 9b17df54ce..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/OpalTable.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.core.domain; - -import javax.validation.constraints.NotNull; - -/** - * Represents a table in opal. - */ -public abstract class OpalTable { - - @NotNull - private String project; - - @NotNull - private String table; - - private LocalizedString name; - - private LocalizedString description; - - private LocalizedString additionalInformation; - - private int weight; - - public String getProject() { - return project; - } - - public void setProject(String project) { - this.project = project; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public void setName(LocalizedString name) { - this.name = name; - } - - public LocalizedString getName() { - return name; - } - - public void setDescription(LocalizedString description) { - this.description = description; - } - - public LocalizedString getDescription() { - return description; - } - - public LocalizedString getAdditionalInformation() { - return additionalInformation; - } - - public void setAdditionalInformation(LocalizedString additionalInformation) { - this.additionalInformation = additionalInformation; - } - - public int getWeight() { - return weight; - } - - public void setWeight(int weight) { - this.weight = weight; - } - - public boolean isFor(String entityId, String project, String table) { - return getEntityId().equals(entityId) && getProject().equals(project) && getTable().equals(table); - } - - protected abstract String getEntityId(); - -} diff --git a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java index 14ec461824..2da6510811 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java +++ b/mica-core/src/main/java/org/obiba/mica/core/domain/StudyTable.java @@ -10,10 +10,10 @@ package org.obiba.mica.core.domain; -import java.io.Serializable; - import com.google.common.base.MoreObjects; +import java.io.Serializable; + /** * Represents a opal table that is associated to a {@link org.obiba.mica.study.domain.Study} * {@link org.obiba.mica.study.domain.Population} {@link org.obiba.mica.study.domain.DataCollectionEvent}. @@ -63,7 +63,7 @@ public boolean appliesTo(String studyId, String populationId, String dataCollect @Override public String toString() { - return MoreObjects.toStringHelper(this).add("project", getProject()).add("table", getTable()) + return MoreObjects.toStringHelper(this).add("source", getSource()) .add("dceId", getDataCollectionEventUId()).toString(); } diff --git a/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java new file mode 100644 index 0000000000..4ca867e5d1 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/service/StudyTableSourceServiceRegistry.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.service; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import org.apache.commons.math3.util.Pair; +import org.obiba.magma.NoSuchValueTableException; +import org.obiba.magma.support.Disposables; +import org.obiba.magma.support.Initialisables; +import org.obiba.mica.core.source.ExcelTableSource; +import org.obiba.mica.core.source.OpalTableSource; +import org.obiba.mica.dataset.domain.StudyDataset; +import org.obiba.mica.file.AttachmentState; +import org.obiba.mica.file.FileStoreService; +import org.obiba.mica.file.service.FileSystemService; +import org.obiba.mica.micaConfig.service.MicaConfigService; +import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.spi.tables.*; +import org.obiba.mica.study.domain.Study; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import java.io.InputStream; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +@Service +public class StudyTableSourceServiceRegistry { + + private static final Logger log = LoggerFactory.getLogger(StudyTableSourceServiceRegistry.class); + + @Inject + private MicaConfigService micaConfigService; + + @Inject + private PluginsService pluginsService; + + @Inject + private OpalService opalService; + + @Inject + protected FileSystemService fileSystemService; + + @Inject + private FileStoreService fileStoreService; + + private Cache sourcesCache; + + @PostConstruct + public void initialize() { + sourcesCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.MINUTES) + .removalListener((RemovalListener) notification -> { + Disposables.silentlyDispose(notification.getValue()); + }) + .build(); + } + + @PreDestroy + public void close() { + sourcesCache.cleanUp(); + } + + public synchronized StudyTableSource makeStudyTableSource(IDataset dataset, IStudy study, String source) { + StudyTableContext context = new StudyTableContext(dataset, study, micaConfigService.getConfig().getPrivacyThreshold()); + + String cacheKey = String.format("%s::%s::%s", dataset.getId(), study.getId(), source); + try { + return sourcesCache.get(cacheKey, () -> makeStudyTableSourceInternal(context, source)); + } catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); + } + } + private StudyTableSource makeStudyTableSourceInternal(StudyTableContext context, String source) { + if (OpalTableSource.isFor(source)) { + OpalTableSource tableSource = OpalTableSource.fromURN(source); + tableSource.setStudyTableContext(context); + tableSource.setOpalService(opalService); + return tableSource; + } + if (ExcelTableSource.isFor(source)) { + ExcelTableSource tableSource = ExcelTableSource.fromURN(source); + tableSource.setStudyTableContext(context); + tableSource.setStudyTableFileStreamProvider(new AttachmentStreamProvider(context, tableSource.getPath())); + return tableSource; + } + Optional serviceOptional = pluginsService.getStudyTableSourceServices().stream() + .filter(service -> service.isFor(source)).findFirst(); + if (serviceOptional.isPresent()) { + StudyTableSource tableSource = serviceOptional.get().makeSource(source); + tableSource.setStudyTableContext(context); + if (tableSource instanceof StudyTableFileSource) { + StudyTableFileSource fileSource = (StudyTableFileSource)tableSource; + fileSource.setStudyTableFileStreamProvider(new AttachmentStreamProvider(context, fileSource.getPath())); + } + Initialisables.initialise(tableSource); + return tableSource; + } + throw new NoSuchElementException("Missing study-table-source plugin to handle source: " + source); + } + + + /** + * Get the input stream from an {@link AttachmentState} object. + */ + private class AttachmentStreamProvider implements StudyTableFileStreamProvider { + + private final StudyTableContext context; + private final String path; + + private AttachmentStreamProvider(StudyTableContext context, String path) { + this.context = context; + this.path = path; + } + + @Override + public InputStream getInputStream() { + String fullPath = path; + Optional attachmentState; + if (!fullPath.startsWith("/")) { + // not a full path, then it may be relative to the dataset's folder + fullPath = String.format("/%s-dataset/%s/%s", (context.getDataset() instanceof StudyDataset ? "collected" : "harmonized"), context.getDataset().getId(), path); + attachmentState = getAttachmentState(fullPath); + // not found, then try a path relative to the study's folder + if (!attachmentState.isPresent()) { + fullPath = String.format("/%s-study/%s/%s", (context.getStudy() instanceof Study ? "individual" : "harmonization"), context.getStudy().getId(), path); + attachmentState = getAttachmentState(fullPath); + } + } else { + attachmentState = getAttachmentState(fullPath); + } + if (attachmentState.isPresent()) { + return fileStoreService.getFile(attachmentState.get().getAttachment().getFileReference()); + } else { + throw new NoSuchStudyTableFileSourceException("No such file at " + fullPath); + } + } + + private Optional getAttachmentState(String fullPath) { + log.info("Reading study table from file: {}", fullPath); + Pair pathName = FileSystemService.extractPathName(fullPath); + try { + AttachmentState state = fileSystemService.getAttachmentState(pathName.getKey(), pathName.getValue(), false); + return Optional.of(state); + } catch (Exception e) { + return Optional.empty(); + } + } + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java new file mode 100644 index 0000000000..f3221ee35b --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/source/ExcelTableSource.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import org.obiba.magma.ValueTable; +import org.obiba.magma.datasource.excel.ExcelDatasource; +import org.obiba.magma.support.Initialisables; +import org.obiba.mica.spi.tables.AbstractStudyTableSource; +import org.obiba.mica.spi.tables.StudyTableFileSource; +import org.obiba.mica.spi.tables.StudyTableFileStreamProvider; + +import javax.validation.constraints.NotNull; +import java.util.List; + +public class ExcelTableSource extends AbstractStudyTableSource implements StudyTableFileSource { + + @NotNull + private String path; + + private String table; + + private boolean initialized; + + private ExcelDatasource excelDatasource; + + private StudyTableFileStreamProvider fileStreamProvider; + + public static boolean isFor(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) + return false; + List tokens = Splitter.on(":").splitToList(source); + return tokens.size() > 2 && tokens.get(2).toLowerCase().endsWith(".xlsx"); + } + + public static ExcelTableSource fromURN(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:file:")) + throw new IllegalArgumentException("Not a valid Excel table source URN: " + source); + + String fullName = source.replace("urn:file:", ""); + int sep = fullName.lastIndexOf(":"); + String file = sep > 0 ? fullName.substring(0, sep) : fullName; + String table = sep > 0 ? fullName.substring(sep + 1) : null; + return ExcelTableSource.newSource(file, table); + } + + private static ExcelTableSource newSource(String path, String table) { + ExcelTableSource source = new ExcelTableSource(); + source.path = path; + source.table = table; + return source; + } + + @Override + public String getPath() { + return path; + } + + @Override + public ValueTable getValueTable() { + ensureInitialized(); + return Strings.isNullOrEmpty(table) ? + excelDatasource.getValueTables().stream().findFirst().get() : + excelDatasource.getValueTable(table); + } + + @Override + public String getURN() { + return Strings.isNullOrEmpty(table) ? String.format("urn:file:%s", path) : String.format("urn:file:%s:%s", path, table); + } + + @Override + public void setStudyTableFileStreamProvider(StudyTableFileStreamProvider provider) { + this.fileStreamProvider = provider; + // deferred init + this.initialized = false; + } + + private void ensureInitialized() { + if (!initialized) { + excelDatasource = new ExcelDatasource(path, fileStreamProvider.getInputStream()); + Initialisables.initialise(excelDatasource); + initialized = true; + } + } +} diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java new file mode 100644 index 0000000000..fb743d7d76 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/source/OpalTableSource.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import com.google.common.base.Strings; +import org.obiba.magma.ValueTable; +import org.obiba.mica.core.source.support.OpalDtos; +import org.obiba.mica.core.source.support.QueryTermsUtil; +import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.spi.tables.AbstractStudyTableSource; +import org.obiba.mica.spi.tables.IVariable; +import org.obiba.mica.web.model.Mica; +import org.obiba.opal.rest.client.magma.RestDatasource; +import org.obiba.opal.rest.client.magma.RestValueTable; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +import javax.validation.constraints.NotNull; + +/** + * Connector to an Opal server, to retrieve value table and summary statistics. + */ +public class OpalTableSource extends AbstractStudyTableSource { + + private OpalService opalService; + + private String opalUrl; + + @NotNull + private String project; + + @NotNull + private String table; + + public String getProject() { + return project; + } + + public void setProject(String project) { + this.project = project; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + @Override + public ValueTable getValueTable() { + return getDatasource().getValueTable(table); + } + + @Override + public boolean providesContingency() { + return true; + } + + @Override + public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { + Search.QueryTermsDto query = QueryTermsUtil.getContingencyQuery(variable, crossVariable); + Search.QueryResultDto results = getRestValueTable().getFacets(query); + return OpalDtos.asDto(variable, crossVariable, getContext().getPrivacyThreshold(), results); + } + + @Override + public boolean providesVariableSummary() { + return true; + } + + @Override + public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { + RestValueTable.RestVariableValueSource variableValueSource = (RestValueTable.RestVariableValueSource) getRestValueTable().getVariableValueSource(variableName); + Math.SummaryStatisticsDto results = variableValueSource.getSummary(); + return OpalDtos.asDto(results); + } + + @Override + public String getURN() { + return String.format("urn:opal:%s.%s", project, table); + } + + public static boolean isFor(String source) { + return !Strings.isNullOrEmpty(source) && source.startsWith("urn:opal:"); + } + + public static OpalTableSource newSource(String project, String table) { + OpalTableSource source = new OpalTableSource(); + source.setProject(project); + source.setTable(table); + return source; + } + + public static OpalTableSource fromURN(String source) { + if (Strings.isNullOrEmpty(source) || !source.startsWith("urn:opal:")) + throw new IllegalArgumentException("Not a valid Opal table source URN: " + source); + + String fullName = toTableName(source); + int sep = fullName.indexOf("."); + String project = fullName.substring(0, sep); + String table = fullName.substring(sep + 1); + return OpalTableSource.newSource(project, table); + } + + public static String toTableName(String source) { + return source.replace("urn:opal:", ""); + } + + public void setOpalService(OpalService opalService) { + this.opalService = opalService; + this.opalUrl = getContext().getStudy().getOpal(); + } + + private RestDatasource getDatasource() { + return opalService.getDatasource(opalUrl, project); + } + + private RestValueTable getRestValueTable() { + return (RestValueTable) getDatasource().getValueTable(table); + } +} diff --git a/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java new file mode 100644 index 0000000000..6151e910fc --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/OpalDtos.java @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source.support; + +import com.google.common.collect.Lists; +import org.obiba.magma.type.BooleanType; +import org.obiba.mica.spi.tables.ICategory; +import org.obiba.mica.spi.tables.IVariable; +import org.obiba.mica.web.model.Mica; +import org.obiba.opal.web.model.Math; +import org.obiba.opal.web.model.Search; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; + +public class OpalDtos { + + // + // Search.QueryResultDto + // + + public static Mica.DatasetVariableContingencyDto asDto(IVariable variable, IVariable crossVariable, int privacyThreshold, Search.QueryResultDto results) { + Mica.DatasetVariableContingencyDto.Builder crossDto = Mica.DatasetVariableContingencyDto.newBuilder(); + Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + + if (results == null) { + allAggBuilder.setN(0); + allAggBuilder.setTotal(0); + crossDto.setAll(allAggBuilder); + return crossDto.build(); + } + + allAggBuilder.setTotal(results.getTotalHits()); + crossDto.setPrivacyThreshold(privacyThreshold); + boolean privacyChecks = !crossVariable.hasCategories() || validatePrivacyThreshold(results, privacyThreshold); + boolean totalPrivacyChecks = validateTotalPrivacyThreshold(results, privacyThreshold); + + // add facet results in the same order as the variable categories + List catNames = variable.getValueType().equals(BooleanType.get().getName()) ? + Lists.newArrayList("true", "false") : variable.getCategoryNames(); + catNames.forEach(catName -> results.getFacetsList().stream() + .filter(facet -> facet.hasFacet() && catName.equals(facet.getFacet())).forEach(facet -> { + boolean privacyCheck = privacyChecks && checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold); + Mica.DatasetVariableAggregationDto.Builder aggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + aggBuilder.setTotal(totalPrivacyChecks ? results.getTotalHits() : 0); + aggBuilder.setTerm(facet.getFacet()); + ICategory category = variable.getCategory(facet.getFacet()); + aggBuilder.setMissing(category != null && category.isMissing()); + addSummaryStatistics(crossVariable, aggBuilder, facet, privacyCheck, totalPrivacyChecks); + crossDto.addAggregations(aggBuilder); + })); + + // add total facet for all variable categories + results.getFacetsList().stream().filter(facet -> facet.hasFacet() && "_total".equals(facet.getFacet())) + .forEach(facet -> { + boolean privacyCheck = privacyChecks && facet.getFilters(0).getCount() >= privacyThreshold; + addSummaryStatistics(crossVariable, allAggBuilder, facet, privacyCheck, totalPrivacyChecks); + }); + + crossDto.setAll(allAggBuilder); + + return crossDto.build(); + + } + + private static boolean checkPrivacyThreshold(int count, int threshold) { + return count == 0 || count >= threshold; + } + + private static boolean validateTotalPrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { + return results.getFacetsList().stream() + .allMatch(facet -> checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold)); + } + + private static boolean validatePrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { + return results.getFacetsList().stream().map(Search.FacetResultDto::getFrequenciesList).flatMap(Collection::stream) + .allMatch(freq -> checkPrivacyThreshold(freq.getCount(), privacyThreshold)); + } + + private static void addSummaryStatistics(IVariable crossVariable, + Mica.DatasetVariableAggregationDto.Builder aggBuilder, Search.FacetResultDto facet, boolean privacyCheck, + boolean totalPrivacyCheck) { + aggBuilder.setN(totalPrivacyCheck ? facet.getFilters(0).getCount() : -1); + if (!privacyCheck) return; + + List catNames = crossVariable.getValueType().equals(BooleanType.get().getName()) ? + Lists.newArrayList("1", "0") : crossVariable.getCategoryNames(); + // order results as the order of cross variable categories + catNames.forEach(catName -> facet.getFrequenciesList().stream().filter(freq -> catName.equals(freq.getTerm())) + .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq)))); + // observed terms, not described by categories + facet.getFrequenciesList().stream().filter(freq -> !catNames.contains(freq.getTerm())) + .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq))); + + if (facet.hasStatistics()) { + aggBuilder.setStatistics(asDto(facet.getStatistics())); + } + } + + private static Mica.FrequencyDto.Builder asDto(IVariable crossVariable, + Search.FacetResultDto.TermFrequencyResultDto result) { + if (crossVariable.getValueType().equals(BooleanType.get().getName())) { + // for some reason 0/1 is returned instead of false/true + return Mica.FrequencyDto.newBuilder() + .setValue("1".equals(result.getTerm()) ? "true" : "false") + .setCount(result.getCount()) + .setMissing(false); + } else if (crossVariable.getCategory(result.getTerm()) != null) { + ICategory category = crossVariable.getCategory(result.getTerm()); + return Mica.FrequencyDto.newBuilder() + .setValue(result.getTerm()) + .setCount(result.getCount()) + .setMissing(category != null && category.isMissing()); + } else { + // observed value, not described by a category + return Mica.FrequencyDto.newBuilder() + .setValue(result.getTerm()) + .setCount(result.getCount()) + .setMissing(false); + } + } + + private static Mica.StatisticsDto.Builder asDto(Search.FacetResultDto.StatisticalResultDto result) { + return Mica.StatisticsDto.newBuilder() // + .setMin(result.getMin()) // + .setMax(result.getMax()) // + .setMean(result.getMean()) // + .setSum(result.getTotal()) // + .setSumOfSquares(result.getSumOfSquares()) // + .setVariance(result.getVariance()) // + .setStdDeviation(result.getStdDeviation()); + } + + // + // SummaryStatisticsDto methods + // + + public static Mica.DatasetVariableAggregationDto asDto(@Nullable Math.SummaryStatisticsDto summary) { + + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + + if (summary == null) return aggDto.setTotal(0).setN(0).build(); + + if (summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { + aggDto = asDto(summary.getExtension(Math.CategoricalSummaryDto.categorical)); + } else if (summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { + aggDto = asDto(summary.getExtension(Math.ContinuousSummaryDto.continuous)); + } else if (summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { + aggDto = asDto(summary.getExtension(Math.DefaultSummaryDto.defaultSummary)); + } else if (summary.hasExtension(Math.TextSummaryDto.textSummary)) { + aggDto = asDto(summary.getExtension(Math.TextSummaryDto.textSummary)); + } else if (summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { + aggDto = asDto(summary.getExtension(Math.GeoSummaryDto.geoSummary)); + } else if (summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { + aggDto = asDto(summary.getExtension(Math.BinarySummaryDto.binarySummary)); + } + + return aggDto.build(); + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.CategoricalSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList(), + summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.DefaultSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.TextSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList(), + summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.GeoSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.BinarySummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); + addFrequenciesDto(aggDto, summary.getFrequenciesList()); + return aggDto; + } + + private static Mica.FrequencyDto.Builder asDto(Math.FrequencyDto freq) { + return Mica.FrequencyDto.newBuilder().setValue(freq.getValue()).setCount(Long.valueOf(freq.getFreq()).intValue()) + .setMissing(freq.getMissing()); + } + + private static Mica.IntervalFrequencyDto.Builder asDto(Math.IntervalFrequencyDto inter) { + return Mica.IntervalFrequencyDto.newBuilder().setCount((int) inter.getFreq()) + .setLower(inter.getLower()).setUpper(inter.getUpper()); + } + + private static void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, + List frequencies) { + addFrequenciesDto(aggDto, frequencies, 0); + } + + private static void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, List frequencies, + int otherFrequency) { + int n = otherFrequency; + if (frequencies != null) { + for (Math.FrequencyDto freq : frequencies) { + aggDto.addFrequencies(asDto(freq)); + if (!freq.getMissing()) n += freq.getFreq(); + } + } + if (otherFrequency > 0) + aggDto.addFrequencies(Mica.FrequencyDto.newBuilder().setValue("???").setCount(otherFrequency) + .setMissing(false)); + aggDto.setN(n); + } + + private static Mica.DatasetVariableAggregationDto.Builder asDto(Math.ContinuousSummaryDto summary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); + Math.DescriptiveStatsDto stats = summary.getSummary(); + + aggDto.setN(Long.valueOf(stats.getN()).intValue()); + + Mica.StatisticsDto.Builder builder = Mica.StatisticsDto.newBuilder(); + + if (stats.hasSum()) builder.setSum(Double.valueOf(stats.getSum()).floatValue()); + if (stats.hasMin() && stats.getMin() != Double.POSITIVE_INFINITY) + builder.setMin(Double.valueOf(stats.getMin()).floatValue()); + if (stats.hasMax() && stats.getMax() != Double.NEGATIVE_INFINITY) + builder.setMax(Double.valueOf(stats.getMax()).floatValue()); + if (stats.hasMean() && !Double.isNaN(stats.getMean())) + builder.setMean(Double.valueOf(stats.getMean()).floatValue()); + if (stats.hasSumsq() && !Double.isNaN(stats.getSumsq())) + builder.setSumOfSquares(Double.valueOf(stats.getSumsq()).floatValue()); + if (stats.hasVariance() && !Double.isNaN(stats.getVariance())) + builder.setVariance(Double.valueOf(stats.getVariance()).floatValue()); + if (stats.hasStdDev() && !Double.isNaN(stats.getStdDev())) + builder.setStdDeviation(Double.valueOf(stats.getStdDev()).floatValue()); + + aggDto.setStatistics(builder); + + if (summary.getFrequenciesCount() > 0) { + summary.getFrequenciesList().forEach(freq -> aggDto.addFrequencies(asDto(freq))); + } + + if (summary.getIntervalFrequencyCount() > 0) { + summary.getIntervalFrequencyList().forEach(inter -> aggDto.addIntervalFrequencies(asDto(inter))); + } + + int total = 0; + if (summary.getFrequenciesCount() > 0) { + for (Math.FrequencyDto freq : summary.getFrequenciesList()) { + total += freq.getFreq(); + } + } + aggDto.setTotal(total); + + return aggDto; + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java similarity index 77% rename from mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java rename to mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java index d9fc80248f..80fe307282 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/support/QueryTermsUtil.java +++ b/mica-core/src/main/java/org/obiba/mica/core/source/support/QueryTermsUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 OBiBa. All rights reserved. + * Copyright (c) 2022 OBiBa. All rights reserved. * * This program and the accompanying materials * are made available under the terms of the GNU Public License v3.0. @@ -8,23 +8,23 @@ * along with this program. If not, see . */ -package org.obiba.mica.dataset.service.support; - -import java.util.List; +package org.obiba.mica.core.source.support; +import com.google.common.collect.Lists; import org.obiba.magma.type.BooleanType; -import org.obiba.mica.dataset.domain.DatasetVariable; +import org.obiba.mica.spi.tables.IVariable; import org.obiba.opal.web.model.Search; -import com.google.common.collect.Lists; +import java.util.List; public class QueryTermsUtil { public static final String TOTAL_FACET = "_total"; - private QueryTermsUtil() {} + private QueryTermsUtil() { + } - public static Search.QueryTermsDto getContingencyQuery(DatasetVariable variable, DatasetVariable crossVariable) { + public static Search.QueryTermsDto getContingencyQuery(IVariable variable, IVariable crossVariable) { Search.QueryTermsDto.Builder queries = Search.QueryTermsDto.newBuilder(); // for each category, make a facet query @@ -40,7 +40,7 @@ public static Search.QueryTermsDto getContingencyQuery(DatasetVariable variable, // Private methods // - private static Search.QueryTermDto getQueryTerm(DatasetVariable variable, DatasetVariable crossVariable, String facetName) { + private static Search.QueryTermDto getQueryTerm(IVariable variable, IVariable crossVariable, String facetName) { Search.QueryTermDto.Builder query = Search.QueryTermDto.newBuilder(); query.setFacet(facetName); @@ -51,7 +51,7 @@ private static Search.QueryTermDto getQueryTerm(DatasetVariable variable, Datase return query.build(); } - private static Search.QueryTermDto getTotalTerm(DatasetVariable variable, DatasetVariable crossVariable) { + private static Search.QueryTermDto getTotalTerm(IVariable variable, IVariable crossVariable) { Search.QueryTermDto.Builder query = Search.QueryTermDto.newBuilder(); query.setFacet(TOTAL_FACET); @@ -80,13 +80,13 @@ private static Search.LogicalTermDto getLogicalTermDto(String variableName, List return term.build(); } - private static List getCategories(DatasetVariable variable) { + private static List getCategories(IVariable variable) { List categories = Lists.newArrayList(); - if(variable.getValueType().equals(BooleanType.get().getName())) { + if (variable.getValueType().equals(BooleanType.get().getName())) { categories.add("true"); categories.add("false"); - } else if(variable.hasCategories()) { - variable.getCategories().forEach(c -> categories.add(c.getName())); + } else if (variable.hasCategories()) { + categories = variable.getCategoryNames(); } return categories; } diff --git a/mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java b/mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java similarity index 53% rename from mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java rename to mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java index 3d61ea3661..f3f1fac398 100644 --- a/mica-core/src/main/java/org/obiba/mica/core/support/YamlClassPathResourceReader.java +++ b/mica-core/src/main/java/org/obiba/mica/core/support/YamlResourceReader.java @@ -4,8 +4,9 @@ import org.springframework.core.io.ClassPathResource; import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.core.io.FileSystemResource; -public class YamlClassPathResourceReader { +public class YamlResourceReader { private static ObjectMapper mapper; @@ -13,10 +14,17 @@ public class YamlClassPathResourceReader { mapper = new ObjectMapper(); } - public static T read(String resourcePath, Class resultClass) { + public static T readClassPath(String resourcePath, Class resultClass) { YamlMapFactoryBean factory = new YamlMapFactoryBean(); factory.setResources(new ClassPathResource(resourcePath)); return mapper.convertValue(factory.getObject(), resultClass); } + + public static T readFile(String resourcePath, Class resultClass) { + YamlMapFactoryBean factory = new YamlMapFactoryBean(); + factory.setResources(new FileSystemResource(resourcePath)); + + return mapper.convertValue(factory.getObject(), resultClass); + } } diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java deleted file mode 100644 index ead2ffbccd..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica510upgrade.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.obiba.mica.core.upgrade; - -import javax.inject.Inject; - -import org.obiba.mica.micaConfig.service.CacheService; -import org.obiba.runtime.Version; -import org.obiba.runtime.upgrade.UpgradeStep; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -@Component -public class Mica510upgrade implements UpgradeStep { - - private static final Logger logger = LoggerFactory.getLogger(Mica510upgrade.class); - - private CacheService cacheService; - - @Inject - public Mica510upgrade(CacheService cacheService) { - this.cacheService = cacheService; - } - - @Override - public String getDescription() { - return "Clear caches for 5.1.0"; - } - - @Override - public Version getAppliesTo() { - return new Version(5, 1, 0); - } - - @Override - public void execute(Version currentVersion) { - logger.info("Executing Mica upgrade to version 5.1.0"); - cacheService.clearAllCaches(); - } - -} diff --git a/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java new file mode 100644 index 0000000000..cde8a72a37 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/core/upgrade/Mica520upgrade.java @@ -0,0 +1,99 @@ +package org.obiba.mica.core.upgrade; + +import javax.inject.Inject; + +import com.mongodb.client.MongoCollection; +import org.bson.Document; +import org.json.JSONException; +import org.obiba.mica.micaConfig.service.CacheService; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.runtime.Version; +import org.obiba.runtime.upgrade.UpgradeStep; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Component; + +@Component +public class Mica520upgrade implements UpgradeStep { + + private static final Logger logger = LoggerFactory.getLogger(Mica520upgrade.class); + + private final CacheService cacheService; + + private final MongoTemplate mongoTemplate; + + private final PluginsService pluginsService; + + @Inject + public Mica520upgrade(CacheService cacheService, MongoTemplate mongoTemplate, PluginsService pluginsService) { + this.cacheService = cacheService; + this.mongoTemplate = mongoTemplate; + this.pluginsService = pluginsService; + } + + @Override + public String getDescription() { + return "Clear caches, update config and search plugin for 5.2.0"; + } + + @Override + public Version getAppliesTo() { + return new Version(5, 2, 0); + } + + @Override + public void execute(Version currentVersion) { + logger.info("Executing Mica upgrade to version 5.2.0"); + + try { + logger.info("Updating 'Mica Config'..."); + updateMicaConfig(); + } catch (JSONException e) { + logger.error("Error occurred while Updating 'Mica Config'"); + } + + logger.info("Clearing all caches..."); + cacheService.clearAllCaches(); + + logger.info("Updating search plugin..."); + try { + updateSearchPlugin(); + // TODO rebuild search index? + } catch (Exception e) { + // not installed, not to be upgraded + // OR upgrade failed for unknown reason + logger.error("Error occurred while updating 'Search Plugin' to its latest version", e); + } + } + + private void updateMicaConfig() throws JSONException { + Document micaConfig = getDBObjectSafely("micaConfig"); + // delete field opal as it was not used + if (null != micaConfig) { + micaConfig.remove("opal"); + mongoTemplate.save(micaConfig, "micaConfig"); + } + } + + private void updateSearchPlugin() { + String searchPluginName = pluginsService.getSearchPluginName(); + // check it is installed + pluginsService.getInstalledPlugin(searchPluginName); + // check it is updatable + if (pluginsService.getUpdatablePlugins().stream().anyMatch(pkg -> pkg.getName().equals(searchPluginName))) { + // install latest version + pluginsService.installPlugin(searchPluginName, null); + } + } + + private Document getDBObjectSafely(String collectionName) { + if (mongoTemplate.collectionExists(collectionName)) { + MongoCollection existingCollection = mongoTemplate.getCollection(collectionName); + return existingCollection.find().first(); + } + + return null; + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java index b1aabd45fe..6321183b41 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/Dataset.java @@ -24,13 +24,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import org.obiba.mica.spi.tables.IDataset; import java.util.Map; /** - * Proxy to Opal tables. + * Proxy to value tables. */ -public abstract class Dataset extends AbstractModelAware implements AttributeAware, Indexable { +public abstract class Dataset extends AbstractModelAware implements AttributeAware, Indexable, IDataset { private static final long serialVersionUID = -3328963766855899217L; diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java index b138f16f4c..62e796c87e 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetCategory.java @@ -16,8 +16,9 @@ import org.obiba.mica.core.domain.Attribute; import org.obiba.mica.core.domain.AttributeAware; import org.obiba.mica.core.domain.Attributes; +import org.obiba.mica.spi.tables.ICategory; -public class DatasetCategory implements AttributeAware { +public class DatasetCategory implements AttributeAware, ICategory { private String name; @@ -37,10 +38,12 @@ public DatasetCategory(Category category) { } } + @Override public String getName() { return name; } + @Override public boolean isMissing() { return missing; } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java index c4884ebd59..f5ecfcf08b 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/DatasetVariable.java @@ -13,10 +13,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import com.google.common.collect.Sets; +import org.apache.commons.compress.utils.Lists; import org.obiba.magma.Variable; import org.obiba.magma.support.VariableNature; import org.obiba.mica.core.domain.*; import org.obiba.mica.spi.search.Indexable; +import org.obiba.mica.spi.tables.IVariable; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -29,7 +31,7 @@ import java.util.stream.Stream; -public class DatasetVariable implements Indexable, AttributeAware { +public class DatasetVariable implements Indexable, AttributeAware, IVariable { private static final long serialVersionUID = -141834508275072637L; @@ -89,9 +91,7 @@ public enum OpalTableType { private int index; - private String project; - - private String table; + private String source; private OpalTableType opalTableType; @@ -132,25 +132,17 @@ public DatasetVariable(HarmonizationDataset dataset, Variable variable) { setContainerId(table.getStudyId()); } - public DatasetVariable(HarmonizationDataset dataset, Variable variable, OpalTable opalTable) { + public DatasetVariable(HarmonizationDataset dataset, Variable variable, BaseStudyTable studyTable) { this(dataset, Type.Harmonized, variable); - if (opalTable instanceof BaseStudyTable) { - studyId = ((BaseStudyTable)opalTable).getStudyId(); - setContainerId(studyId); - opalTableType = opalTable instanceof StudyTable ? OpalTableType.Study : OpalTableType.Harmonization; + studyId = studyTable.getStudyId(); + setContainerId(studyId); + opalTableType = studyTable instanceof StudyTable ? OpalTableType.Study : OpalTableType.Harmonization; - if (opalTable instanceof HarmonizationStudyTable) { - populationId = ((HarmonizationStudyTable) opalTable).getPopulationUId(); - dceId = ((HarmonizationStudyTable) opalTable).getDataCollectionEventUId(); - } else { - populationId = ((BaseStudyTable) opalTable).getPopulationUId(); - dceId = ((BaseStudyTable) opalTable).getDataCollectionEventUId(); - } - } + populationId = studyTable.getPopulationUId(); + dceId = studyTable.getDataCollectionEventUId(); - project = opalTable.getProject(); - table = opalTable.getTable(); + source = studyTable.getSource(); } private DatasetVariable(Dataset dataset, Type type, Variable variable) { @@ -187,7 +179,7 @@ public String getId() { if (Type.Harmonized == variableType) { String entityId = studyId; String tableType = opalTableType == OpalTableType.Study ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + project + ID_SEPARATOR + table; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + source; } return id; @@ -202,8 +194,7 @@ public void setId(String id) { opalTableType = Strings.isNullOrEmpty(tableType) ? null : OpalTableType.valueOf(tableType); if (resolver.hasStudyId()) studyId = resolver.getStudyId(); - if (resolver.hasProject()) project = resolver.getProject(); - if (resolver.hasTable()) table = resolver.getTable(); + if (resolver.hasSource()) source = resolver.getSource(); } public String getDatasetId() { @@ -250,6 +241,7 @@ public String getUnit() { return unit; } + @Override public String getValueType() { return valueType; } @@ -270,6 +262,7 @@ public String getOccurrenceGroup() { return occurrenceGroup; } + @Override public boolean hasCategories() { return categories != null && !categories.isEmpty(); } @@ -278,6 +271,13 @@ public List getCategories() { return categories; } + @Override + public List getCategoryNames() { + if (!hasCategories()) return Lists.newArrayList(); + return categories.stream().map(DatasetCategory::getName).collect(Collectors.toList()); + } + + @Override public DatasetCategory getCategory(String name) { if (!hasCategories()) return null; @@ -327,12 +327,8 @@ public int getIndex() { return index; } - public String getProject() { - return project; - } - - public String getTable() { - return table; + public String getSource() { + return source; } public OpalTableType getOpalTableType() { @@ -429,7 +425,7 @@ private String cleanStringForSearch(String string) { } public static class IdEncoderDecoder { - private static Pattern encodePattern = Pattern.compile("&|\\||\\(|\\)|=|<|>|,"); + private static Pattern encodePattern = Pattern.compile("&|\\||\\(|\\)|=|<|>|,|/"); // Use underscore to make sure the RQLParser does not try to decode private static final Map encodeMap = Stream.of(new String[][] { { "&", "_26" }, @@ -439,10 +435,11 @@ public static class IdEncoderDecoder { { "=", "_3d" }, { "<", "_3c" }, { ">", "_3e" }, - { ",", "_2c" } + { ",", "_2c" }, + { "/", "_2f" } }).collect(Collectors.toMap(data -> data[0], data -> data[1])); - private static Pattern decodePattern = Pattern.compile("(_26|_7c|_28|_29|_3d|_3c|_3e|_2c)"); + private static Pattern decodePattern = Pattern.compile("(_26|_7c|_28|_29|_3d|_3c|_3e|_2c|_2f)"); private static final Map decodeMap = Stream.of(new String[][] { { "_26", "&" }, { "_7c" , "|"}, @@ -451,7 +448,8 @@ public static class IdEncoderDecoder { { "_3d" , "="}, { "_3c" , "<"}, { "_3e" , ">"}, - { "_2c" , ","} + { "_2c" , ","}, + { "_2f" , "/"} }).collect(Collectors.toMap(data -> data[0], data -> data[1])); public static String encode(String value) { @@ -483,7 +481,7 @@ private static String encodeDecode(Map map, Pattern pattern, Str public static class IdResolver { - private final String id; + private String id; private final Type type; @@ -493,12 +491,10 @@ public static class IdResolver { private final String studyId; - private final String project; - - private final String table; - private final String tableType; + private final String source; + public static IdResolver from(String id) { return new IdResolver(id); } @@ -507,31 +503,33 @@ public static IdResolver from(String datasetId, String variableName, Type variab return from(encode(datasetId, variableName, variableType, null)); } + public static String encode(String datasetId, String variableName, Type variableType) { + return encode(datasetId, variableName, variableType, null, null, null); + } + public static String encode(String datasetId, String variableName, Type variableType, String studyId, - String project, String table, String tableType) { + String source, String tableType) { String id = datasetId + ID_SEPARATOR + IdEncoderDecoder.encode(variableName) + ID_SEPARATOR + variableType; String entityId; if (Type.Harmonized == variableType) { entityId = studyId; - id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + project + ID_SEPARATOR + table; + id = id + ID_SEPARATOR + tableType + ID_SEPARATOR + entityId + ID_SEPARATOR + source; } return id; } - public static String encode(String datasetId, String variableName, Type variableType, OpalTable opalTable) { - String tableType = opalTable instanceof StudyTable ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; - BaseStudyTable studyTable = (BaseStudyTable)opalTable; + public static String encode(String datasetId, String variableName, Type variableType, BaseStudyTable studyTable) { + String tableType = studyTable instanceof StudyTable ? OPAL_STUDY_TABLE_PREFIX : OPAL_HARMONIZATION_TABLE_PREFIX; return studyTable == null - ? encode(datasetId, variableName, variableType, null, null, null, null) + ? encode(datasetId, variableName, variableType) : encode(datasetId, variableName, variableType, studyTable.getStudyId(), - opalTable.getProject(), - opalTable.getTable(), + studyTable.getSource(), tableType); } @@ -540,7 +538,9 @@ private IdResolver(String id) { this.id = IdEncoderDecoder.encode(id); boolean encoded = !this.id.equals(id); - String[] tokens = id.split(ID_SEPARATOR); + String[] parts = id.split(":urn:"); + + String[] tokens = parts[0].split(ID_SEPARATOR); if (tokens.length < 3) throw new IllegalArgumentException("Not a valid dataset variable ID: " + id); datasetId = tokens[0]; @@ -550,8 +550,16 @@ private IdResolver(String id) { tableType = tokens.length > 3 ? tokens[3] : null; studyId = tokens.length > 4 ? tokens[4] : null; - project = tokens.length > 5 ? tokens[5] : null; - table = tokens.length > 6 ? tokens[6] : null; + if (parts.length>1) { + source = "urn:" + parts[1]; + } else if (tokens.length > 6) { + // legacy + source = String.format("urn:opal:%s.%s", tokens[5], tokens[6]); + // need to rewrite id + this.id = IdEncoderDecoder.encode(encode(datasetId, name, type,studyId, source, tableType)); + } else { + source = null; + } } public String getId() { @@ -578,26 +586,18 @@ public boolean hasStudyId() { return !Strings.isNullOrEmpty(studyId); } - public String getProject() { - return project; - } - - public boolean hasProject() { - return !Strings.isNullOrEmpty(project); - } - - public String getTable() { - return table; + public boolean hasSource() { + return !Strings.isNullOrEmpty(source); } - public boolean hasTable() { - return !Strings.isNullOrEmpty(table); + public String getSource() { + return source; } @Override public String toString() { String tableType = type == Type.Dataschema ? OPAL_HARMONIZATION_TABLE_PREFIX : OPAL_STUDY_TABLE_PREFIX; - return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + project + ", " + table + "]"; + return "[" + datasetId + "," + name + "," + type + ", " + tableType + ", " + studyId + ", " + source + "]"; } public String getTableType() { diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java b/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java index 124cc5b0c2..6ee28b065f 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/domain/HarmonizationDataset.java @@ -10,21 +10,18 @@ package org.obiba.mica.dataset.domain; -import java.io.Serializable; -import java.util.*; -import java.util.stream.Collectors; - -import javax.validation.constraints.NotNull; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; - import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTable; import org.obiba.mica.core.domain.StudyTable; +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.*; +import java.util.stream.Collectors; + /** * Dataset that relies on Study Opal servers summaries. */ @@ -106,6 +103,6 @@ public Map parts() { @JsonIgnore public List getBaseStudyTables() { return Lists.newArrayList(Iterables.concat(getStudyTables(), getHarmonizationTables())).stream()// - .sorted(Comparator.comparingInt(OpalTable::getWeight)).collect(Collectors.toList()); + .sorted(Comparator.comparingInt(BaseStudyTable::getWeight)).collect(Collectors.toList()); } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java index 29511f5a61..d669a068fd 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/CollectedDatasetService.java @@ -15,9 +15,10 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; -import org.joda.time.DateTime; import org.obiba.magma.NoSuchValueTableException; import org.obiba.magma.NoSuchVariableException; +import org.obiba.magma.ValueTable; +import org.obiba.magma.support.Disposables; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.AbstractGitPersistable; import org.obiba.mica.core.domain.PublishCascadingScope; @@ -35,12 +36,12 @@ import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; import org.obiba.mica.dataset.event.DatasetUpdatedEvent; -import org.obiba.mica.dataset.service.support.QueryTermsUtil; import org.obiba.mica.file.FileUtils; import org.obiba.mica.file.service.FileSystemService; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.DataCollectionEvent; @@ -50,8 +51,7 @@ import org.obiba.mica.study.service.IndividualStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestValueTable; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -67,6 +67,7 @@ import javax.inject.Inject; import javax.validation.Valid; import javax.validation.constraints.NotNull; +import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -74,8 +75,6 @@ import static java.util.stream.Collectors.toList; -import java.time.LocalDateTime; - @Service @Validated public class CollectedDatasetService extends DatasetService { @@ -410,10 +409,8 @@ public List processVariablesForStudyDataset(StudyDataset datase } @Override - @NotNull - protected RestValueTable getTable(@NotNull StudyDataset dataset) throws NoSuchValueTableException { - StudyTable studyTable = dataset.getSafeStudyTable(); - return execute(studyTable, datasource -> (RestValueTable) datasource.getValueTable(studyTable.getTable())); + protected ValueTable getValueTable(@NotNull StudyDataset dataset) throws NoSuchValueTableException { + return getStudyTableSource(dataset, dataset.getSafeStudyTable()).getValueTable(); } @Override @@ -428,30 +425,25 @@ public Iterable getDatasetVariables(StudyDataset dataset) { @Override public DatasetVariable getDatasetVariable(StudyDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getVariableValueSource(dataset, variableName).getVariable()); + return new DatasetVariable(dataset, getValueTable(dataset).getVariable(variableName)); } @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName") - public SummaryStatisticsWrapper getVariableSummary(@NotNull StudyDataset dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { + public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull StudyDataset dataset, String variableName) { log.info("Caching variable summary {} {}", dataset.getId(), variableName); - return new SummaryStatisticsWrapper(getVariableValueSource(dataset, variableName).getSummary()); - } - - public Search.QueryResultDto getVariableFacet(@NotNull StudyDataset dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - log.debug("Getting variable facet {} {}", dataset.getId(), variableName); - return getVariableValueSource(dataset, variableName).getFacet(); + StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); + Mica.DatasetVariableAggregationDto summary = tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Disposables.silentlyDispose(tableSource); + return summary; } - public Search.QueryResultDto getFacets(@NotNull StudyDataset dataset, Search.QueryTermsDto query) - throws NoSuchValueTableException, NoSuchVariableException { - return getTable(dataset).getFacets(query); - } + public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, + DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { - public Search.QueryResultDto getContingencyTable(@NotNull StudyDataset dataset, DatasetVariable variable, - DatasetVariable crossVariable) throws NoSuchValueTableException, NoSuchVariableException { - return getFacets(dataset, QueryTermsUtil.getContingencyQuery(variable, crossVariable)); + StudyTableSource tableSource = getStudyTableSource(dataset, dataset.getSafeStudyTable()); + Mica.DatasetVariableContingencyDto results = tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Disposables.silentlyDispose(tableSource); + return results; } public void delete(String id) { @@ -515,18 +507,6 @@ protected EventBus getEventBus() { // Private methods // - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param studyTable - * @param callback - * @param - * @return - */ - private T execute(StudyTable studyTable, DatasourceCallback callback) { - return execute(getDatasource(studyTable), callback); - } - private void saveInternal(StudyDataset dataset, String comment) { if (!Strings.isNullOrEmpty(dataset.getId()) && micaConfigService.getConfig().isCommentsRequiredOnDocumentSave() && Strings.isNullOrEmpty(comment)) { throw new MissingCommentException("Due to the server configuration, comments are required when saving this document."); diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java index 52543c8e40..9eb04c2c22 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/DatasetService.java @@ -10,42 +10,28 @@ package org.obiba.mica.dataset.service; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.Serializable; -import java.util.List; - -import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; - -import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf; - -import org.obiba.magma.MagmaRuntimeException; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; -import org.obiba.magma.Variable; +import com.google.common.base.Strings; +import com.google.common.eventbus.EventBus; +import org.obiba.magma.*; +import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.EntityState; -import org.obiba.mica.core.domain.HarmonizationStudyTable; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.core.domain.OpalTable; -import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.core.service.AbstractGitPersistableService; +import org.obiba.mica.core.service.StudyTableSourceServiceRegistry; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestDatasource; -import org.obiba.opal.rest.client.magma.RestValueTable; -import org.obiba.opal.web.model.Magma; -import org.obiba.opal.web.model.Math; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Strings; -import com.google.common.eventbus.EventBus; -import com.google.protobuf.GeneratedMessage; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.List; /** * {@link org.obiba.mica.dataset.domain.Dataset} management service. @@ -55,6 +41,9 @@ public abstract class DatasetService private static final Logger log = LoggerFactory.getLogger(DatasetService.class); + @Inject + private StudyTableSourceServiceRegistry studyTableSourceServiceRegistry; + /** * Get all {@link org.obiba.mica.dataset.domain.DatasetVariable}s from a {@link org.obiba.mica.dataset.domain.Dataset}. * @@ -73,18 +62,7 @@ public abstract class DatasetService public abstract DatasetVariable getDatasetVariable(T dataset, String name) throws NoSuchValueTableException, NoSuchVariableException; - /** - * Get the {@link org.obiba.opal.web.model.Magma.TableDto} of the {@link org.obiba.mica.dataset.domain.Dataset} identified by its id. - * - * @param dataset - * @return - */ - @NotNull - protected abstract RestValueTable getTable(@NotNull T dataset) throws NoSuchValueTableException; - - protected RestValueTable getTable(@NotNull String project, @NotNull String table) throws NoSuchValueTableException { - return execute(getDatasource(project), datasource -> (RestValueTable) datasource.getValueTable(table)); - } + protected abstract ValueTable getValueTable(@NotNull T dataset) throws NoSuchValueTableException; protected abstract StudyService getStudyService(); @@ -150,150 +128,21 @@ private void ensureAcronym(@NotNull T dataset) { */ protected Iterable getVariables(@NotNull T dataset) throws NoSuchDatasetException, NoSuchValueTableException { - return getTable(dataset).getVariables(); - } - - /** - * Get the {@link org.obiba.magma.VariableValueSource} (proxy to the {@link org.obiba.magma.Variable} of - * the {@link org.obiba.mica.dataset.domain.Dataset} identified by its id. - * - * @param dataset - * @param variableName - * @return - * @throws NoSuchDatasetException - */ - protected RestValueTable.RestVariableValueSource getVariableValueSource(@NotNull T dataset, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(dataset).getVariableValueSource(variableName); - } - - protected RestValueTable.RestVariableValueSource getVariableValueSource(String project, String table, String variableName) - throws NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(project, table).getVariableValueSource(variableName); + return getValueTable(dataset).getVariables(); } - public Magma.TableDto getTableDto(@NotNull T dataset) { - return getTable(dataset).getTableDto(); - } - - public Magma.VariableDto getVariable(@NotNull T dataset, String variableName) { - return getVariableValueSource(dataset, variableName).getVariableDto(); - } - - /** - * Callback that can be used to make any operations on a {@link org.obiba.opal.rest.client.magma.RestDatasource} - * - * @param - */ - public interface DatasourceCallback { - R doWithDatasource(RestDatasource datasource); - } - - /** - * Execute the callback on the given datasource. - * - * @param datasource - * @param callback - * @param - * @return - */ - protected R execute(RestDatasource datasource, DatasourceCallback callback) { - return callback.doWithDatasource(datasource); - } - - protected RestDatasource getDatasource(@NotNull OpalTable opalTable) { - String opalUrl = null; - - if (opalTable instanceof StudyTable) { - opalUrl = getStudyService().findDraft(((StudyTable) opalTable).getStudyId()).getOpal(); - } else if (opalTable instanceof HarmonizationStudyTable) { - opalUrl = getStudyService().findDraft(((HarmonizationStudyTable) opalTable).getStudyId()).getOpal(); - } - - return getOpalService().getDatasource(opalUrl, opalTable.getProject()); - } - - protected RestDatasource getDatasource(@NotNull String project) { - return getOpalService().getDatasource(project); + protected StudyTableSource getStudyTableSource(@NotNull T dataset, @NotNull BaseStudyTable studyTable) { + return studyTableSourceServiceRegistry.makeStudyTableSource(dataset, getStudyService().findDraft(studyTable.getStudyId()), studyTable.getSource()); } protected Iterable wrappedGetDatasetVariables(T dataset) { try { return getDatasetVariables(dataset); } catch (NoSuchValueTableException e) { - throw new InvalidDatasetException(e); + throw e; } catch (MagmaRuntimeException e) { throw new DatasourceNotAvailableException(e); } } - /** - * Helper class to serialize protobuf object extension. - */ - public static class SummaryStatisticsWrapper implements Serializable { - @IgnoreSizeOf - private org.obiba.opal.web.model.Math.SummaryStatisticsDto summary; - - public SummaryStatisticsWrapper(Math.SummaryStatisticsDto summary) { - this.summary = summary; - } - - public Math.SummaryStatisticsDto getWrappedDto() { - return summary; - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - summary = (Math.SummaryStatisticsDto)in.readObject(); - GeneratedMessage ext = (GeneratedMessage)in.readObject(); - - if (ext == null) return; - - Math.SummaryStatisticsDto.Builder builder = summary.toBuilder(); - - if(ext instanceof Math.CategoricalSummaryDto) - builder.setExtension(Math.CategoricalSummaryDto.categorical, (Math.CategoricalSummaryDto) ext); - else if(ext instanceof Math.ContinuousSummaryDto) - builder.setExtension(Math.ContinuousSummaryDto.continuous, (Math.ContinuousSummaryDto) ext); - else if(ext instanceof Math.DefaultSummaryDto) - builder.setExtension(Math.DefaultSummaryDto.defaultSummary, (Math.DefaultSummaryDto) ext); - else if(ext instanceof Math.TextSummaryDto) - builder.setExtension(Math.TextSummaryDto.textSummary, (Math.TextSummaryDto) ext); - else if(ext instanceof Math.GeoSummaryDto) - builder.setExtension(Math.GeoSummaryDto.geoSummary, (Math.GeoSummaryDto) ext); - else if(ext instanceof Math.BinarySummaryDto) - builder.setExtension(Math.BinarySummaryDto.binarySummary, (Math.BinarySummaryDto) ext); - - summary = builder.build(); - } - - private void writeObject(java.io.ObjectOutputStream stream) - throws IOException { - GeneratedMessage ext = null; - - Math.SummaryStatisticsDto.Builder builder = Math.SummaryStatisticsDto.newBuilder(summary); - - if(summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { - ext = summary.getExtension(Math.CategoricalSummaryDto.categorical); - builder.clearExtension(Math.CategoricalSummaryDto.categorical); - } else if(summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { - ext = summary.getExtension(Math.ContinuousSummaryDto.continuous); - builder.clearExtension(Math.ContinuousSummaryDto.continuous); - } else if(summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { - ext = summary.getExtension(Math.DefaultSummaryDto.defaultSummary); - builder.clearExtension(Math.DefaultSummaryDto.defaultSummary); - } else if(summary.hasExtension(Math.TextSummaryDto.textSummary)) { - ext = summary.getExtension(Math.TextSummaryDto.textSummary); - builder.clearExtension(Math.TextSummaryDto.textSummary); - } else if(summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { - ext = summary.getExtension(Math.GeoSummaryDto.geoSummary); - builder.clearExtension(Math.GeoSummaryDto.geoSummary); - } else if(summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { - ext = summary.getExtension(Math.BinarySummaryDto.binarySummary); - builder.clearExtension(Math.BinarySummaryDto.binarySummary); - } - - stream.writeObject(builder.build()); - stream.writeObject(ext); - } - } } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java index 7eeeee0226..80e5dc900b 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/HarmonizedDatasetService.java @@ -10,33 +10,15 @@ package org.obiba.mica.dataset.service; -import static java.util.stream.Collectors.toList; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.validation.Valid; -import javax.validation.constraints.NotNull; - -import org.obiba.magma.MagmaRuntimeException; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; -import org.obiba.magma.ValueTable; -import org.obiba.magma.Variable; +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import org.obiba.magma.*; +import org.obiba.magma.support.Disposables; import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.BaseStudyTable; -import org.obiba.mica.core.domain.OpalTable; import org.obiba.mica.core.domain.PublishCascadingScope; import org.obiba.mica.core.repository.EntityStateRepository; import org.obiba.mica.core.service.MissingCommentException; @@ -50,20 +32,19 @@ import org.obiba.mica.dataset.event.DatasetPublishedEvent; import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; import org.obiba.mica.dataset.event.DatasetUpdatedEvent; -import org.obiba.mica.dataset.service.support.QueryTermsUtil; import org.obiba.mica.file.FileUtils; import org.obiba.mica.file.service.FileSystemService; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.service.NetworkService; +import org.obiba.mica.spi.tables.StudyTableSource; import org.obiba.mica.study.NoSuchStudyException; import org.obiba.mica.study.domain.BaseStudy; import org.obiba.mica.study.domain.HarmonizationStudy; import org.obiba.mica.study.service.HarmonizationStudyService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.mica.study.service.StudyService; -import org.obiba.opal.rest.client.magma.RestValueTable; -import org.obiba.opal.web.model.Search; +import org.obiba.mica.web.model.Mica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -76,11 +57,19 @@ import org.springframework.util.Assert; import org.springframework.validation.annotation.Validated; -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.common.eventbus.EventBus; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static java.util.stream.Collectors.toList; @Service @Validated @@ -313,8 +302,7 @@ public void publish(@NotNull String id, boolean published, PublishCascadingScope private void checkIsPublishable(HarmonizationDataset dataset) { if (dataset == null || dataset.getHarmonizationTable() == null - || dataset.getHarmonizationTable().getProject() == null - || dataset.getHarmonizationTable().getTable() == null + || dataset.getHarmonizationTable().getSource() == null || dataset.getHarmonizationTable().getStudyId() == null) { throw new IllegalArgumentException("dataset.harmonization.missing-attributes"); } @@ -357,10 +345,8 @@ public void delete(String id) { } @Override - @NotNull - protected RestValueTable getTable(@NotNull HarmonizationDataset dataset) throws NoSuchValueTableException { - return execute(dataset.getSafeHarmonizationTable().getProject(), - datasource -> (RestValueTable) datasource.getValueTable(dataset.getSafeHarmonizationTable().getTable())); + protected ValueTable getValueTable(@NotNull HarmonizationDataset dataset) throws NoSuchValueTableException { + return getStudyTableSource(dataset, dataset.getSafeHarmonizationTable()).getValueTable(); } @Override @@ -372,52 +358,47 @@ public Iterable getDatasetVariables(HarmonizationDataset datase @Override public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName) throws NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getVariableValueSource(dataset, variableName).getVariable()); + return new DatasetVariable(dataset, getStudyTableSource(dataset, dataset.getSafeHarmonizationTable()).getValueTable().getVariable(variableName)); } - public Iterable getDatasetVariables(HarmonizationDataset dataset, OpalTable opalTable) + public Iterable getDatasetVariables(HarmonizationDataset dataset, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException { - return StreamSupport.stream(getVariables(opalTable).spliterator(), false) - .map(input -> new DatasetVariable(dataset, input, opalTable)).collect(toList()); + return StreamSupport.stream(getVariables(dataset, studyTable).spliterator(), false) + .map(input -> new DatasetVariable(dataset, input, studyTable)).collect(toList()); } - public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, OpalTable opalTable) + public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, BaseStudyTable studyTable) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - return new DatasetVariable(dataset, getTable(opalTable).getVariableValueSource(variableName).getVariable()); + return new DatasetVariable(dataset, getStudyTableSource(dataset, studyTable).getValueTable().getVariable(variableName)); } public DatasetVariable getDatasetVariable(HarmonizationDataset dataset, String variableName, String studyId, - String project, String table) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { + String source) throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { return new DatasetVariable(dataset, - getTable(dataset, studyId, project, table).getVariableValueSource(variableName).getVariable()); - } - - @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", - key = "#variableName + ':' + #studyId + ':' + #project + ':' + #table") - public SummaryStatisticsWrapper getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, - String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.info("Caching variable summary {} {} {} {} {}", dataset.getId(), variableName, studyId, project, table); - - return new SummaryStatisticsWrapper( - getVariableValueSource(dataset, variableName, studyId, project, table).getSummary()); - } - - public Search.QueryResultDto getVariableFacet(@NotNull HarmonizationDataset dataset, String variableName, - String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - log.debug("Getting variable facet {} {}", dataset.getId(), variableName); - return getVariableValueSource(dataset, variableName, studyId, project, table).getFacet(); - } + getTable(dataset, studyId, source).getVariableValueSource(variableName).getVariable()); + } + + @Cacheable(value = "dataset-variables", cacheResolver = "datasetVariablesCacheResolver", key = "#variableName + ':' + #studyId + ':' + #source") + public Mica.DatasetVariableAggregationDto getVariableSummary(@NotNull HarmonizationDataset dataset, String variableName, String studyId, String source) { + for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if(baseTable.isFor(studyId, source)) { + log.info("Caching variable summary {} {} {} {}", dataset.getId(), variableName, studyId, source); + StudyTableSource tableSource = getStudyTableSource(dataset, baseTable); + Mica.DatasetVariableAggregationDto summary = tableSource.providesVariableSummary() ? tableSource.getVariableSummary(variableName) : null; + Disposables.silentlyDispose(tableSource); + return summary; + } + } - public Search.QueryResultDto getFacets(Search.QueryTermsDto query, OpalTable opalTable) - throws NoSuchStudyException, NoSuchValueTableException { - return getTable(opalTable).getFacets(query); + throw NoSuchStudyException.withId(studyId); } - public Search.QueryResultDto getContingencyTable(@NotNull OpalTable opalTable, DatasetVariable variable, - DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { - return getFacets(QueryTermsUtil.getContingencyQuery(variable, crossVariable), opalTable); + public Mica.DatasetVariableContingencyDto getContingencyTable(@NotNull HarmonizationDataset dataset, @NotNull BaseStudyTable studyTable, DatasetVariable variable, + DatasetVariable crossVariable) throws NoSuchStudyException, NoSuchValueTableException { + StudyTableSource tableSource = getStudyTableSource(dataset, studyTable); + Mica.DatasetVariableContingencyDto results = tableSource.providesContingency() ? tableSource.getContingency(variable, crossVariable) : null; + Disposables.silentlyDispose(tableSource); + return results; } @Override @@ -487,71 +468,23 @@ protected HarmonizationDataset prepareSave(HarmonizationDataset dataset) { } } - private Iterable getVariables(OpalTable opalTable) + private Iterable getVariables(@NotNull HarmonizationDataset dataset, BaseStudyTable studyTable) throws NoSuchDatasetException, NoSuchStudyException, NoSuchValueTableException { - return getTable(opalTable).getVariables(); - } - - private RestValueTable getTable(@NotNull OpalTable opalTable) - throws NoSuchStudyException, NoSuchValueTableException { - return execute(opalTable, ds -> (RestValueTable) ds.getValueTable(opalTable.getTable())); + return getStudyTableSource(dataset, studyTable).getValueTable().getVariables(); } - private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String project, String table) + private ValueTable getTable(@NotNull HarmonizationDataset dataset, String studyId, String source) throws NoSuchStudyException, NoSuchValueTableException { - for(BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if(opalTable.isFor(opalTableId, project, table)) { - return getTable(opalTable); + for(BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if(baseTable.isFor(studyId, source)) { + return getStudyTableSource(dataset, baseTable).getValueTable(); } } throw NoSuchStudyException.withId(studyId); } - private RestValueTable.RestVariableValueSource getVariableValueSource(@NotNull HarmonizationDataset dataset, - String variableName, String studyId, String project, String table) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - for(BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if(opalTable.isFor(opalTableId, project, table)) { - return getVariableValueSource(variableName, opalTable); - } - } - - throw NoSuchStudyException.withId(studyId); - } - - private RestValueTable.RestVariableValueSource getVariableValueSource(String variableName, OpalTable opalTable) - throws NoSuchStudyException, NoSuchValueTableException, NoSuchVariableException { - return (RestValueTable.RestVariableValueSource) getTable(opalTable).getVariableValueSource(variableName); - } - - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param project - * @param callback - * @param - * @return - */ - private T execute(String project, DatasourceCallback callback) { - return execute(getDatasource(project), callback); - } - - /** - * Build or reuse the {@link org.obiba.opal.rest.client.magma.RestDatasource} and execute the callback with it. - * - * @param opalTable - * @param callback - * @param - * @return - */ - private T execute(OpalTable opalTable, DatasourceCallback callback) { - return execute(getDatasource(opalTable), callback); - } - @Override protected EntityStateRepository getEntityStateRepository() { return harmonizationDatasetStateRepository; @@ -609,7 +542,7 @@ public void asyncBuildDatasetVariablesCache(HarmonizationDataset dataset, dataset.getBaseStudyTables().forEach(st -> harmonizationVariables.forEach((k, v) -> v.forEach(var -> { try { String studyId = st.getStudyId(); - service.getVariableSummary(dataset, var.getName(), studyId, st.getProject(), st.getTable()); + service.getVariableSummary(dataset, var.getName(), studyId, st.getSource()); } catch(Exception e) { //ignoring } diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java deleted file mode 100644 index c4844e3a00..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/InvalidDatasetException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.dataset.service; - -public class InvalidDatasetException extends RuntimeException { - - public InvalidDatasetException() { - super(); - } - - public InvalidDatasetException(String message) { - super(message); - } - - public InvalidDatasetException(Throwable e) { - super(e); - } - - public InvalidDatasetException(String message, Throwable e) { - super(message, e); - } -} diff --git a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java index f10ae60e22..a85b777ed0 100644 --- a/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java +++ b/mica-core/src/main/java/org/obiba/mica/dataset/service/VariableSetService.java @@ -10,26 +10,14 @@ package org.obiba.mica.dataset.service; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.googlecode.protobuf.format.JsonFormat; -import java.io.IOException; -import java.io.OutputStream; -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; -import org.obiba.mica.core.domain.Attributes; -import org.obiba.mica.core.domain.DocumentSet; -import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.core.domain.OpalTable; +import org.obiba.mica.core.domain.*; import org.obiba.mica.core.service.DocumentSetService; -import org.obiba.mica.dataset.domain.Dataset; -import org.obiba.mica.dataset.domain.DatasetCategory; -import org.obiba.mica.dataset.domain.DatasetVariable; -import org.obiba.mica.dataset.domain.HarmonizationDataset; -import org.obiba.mica.dataset.domain.StudyDataset; +import org.obiba.mica.core.source.OpalTableSource; +import org.obiba.mica.dataset.domain.*; import org.obiba.mica.micaConfig.domain.MicaConfig; import org.obiba.mica.study.service.PublishedDatasetVariableService; import org.obiba.opal.web.model.Magma; @@ -37,6 +25,13 @@ import org.springframework.validation.annotation.Validated; import javax.inject.Inject; +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Service @Validated @@ -262,11 +257,11 @@ private Magma.VariableDto toVariableDto(DatasetVariable datasetVariable) { builder.setName(datasetVariable.getName()); builder.setIndex(datasetVariable.getIndex()); - builder.setReferencedEntityType(datasetVariable.getReferencedEntityType()); + builder.setReferencedEntityType(toNonNullString(datasetVariable.getReferencedEntityType())); builder.setUnit(datasetVariable.getUnit()); - builder.setMimeType(datasetVariable.getMimeType()); + builder.setMimeType(toNonNullString(datasetVariable.getMimeType())); builder.setIsRepeatable(datasetVariable.isRepeatable()); - builder.setOccurrenceGroup(datasetVariable.getOccurrenceGroup()); + builder.setOccurrenceGroup(toNonNullString(datasetVariable.getOccurrenceGroup())); builder.setValueType(datasetVariable.getValueType()); builder.setEntityType(datasetVariable.getEntityType()); @@ -283,6 +278,10 @@ private Magma.VariableDto toVariableDto(DatasetVariable datasetVariable) { return builder.build(); } + private String toNonNullString(String value) { + return value == null ? "" : value; + } + private List toAttributeDtoList(Attributes attributes) { return attributes.asAttributeList().stream().map(attribute -> { Magma.AttributeDto.Builder builder = Magma.AttributeDto.newBuilder(); @@ -320,16 +319,18 @@ private List toCategoryDtoList(List categori private List toOpalTableFullName(Dataset dataset) { if (dataset instanceof StudyDataset) { - OpalTable opalTable = ((StudyDataset) dataset).getSafeStudyTable(); - return Lists.newArrayList(opalTable.getProject() + "." + opalTable.getTable()); + StudyTable studyTable = ((StudyDataset) dataset).getSafeStudyTable(); + return OpalTableSource.isFor(studyTable.getSource()) ? Lists.newArrayList(OpalTableSource.toTableName(studyTable.getSource())) : Lists.newArrayList(); } else { HarmonizationDataset harmoDataset = (HarmonizationDataset) dataset; // one for each study and harmo tables List tableNames = Lists.newArrayList(); tableNames.addAll(harmoDataset.getStudyTables().stream() - .map(st -> st.getProject() + "." + st.getTable()).collect(Collectors.toList())); + .filter(st -> OpalTableSource.isFor(st.getSource())) + .map(st -> OpalTableSource.toTableName(st.getSource())).collect(Collectors.toList())); tableNames.addAll(harmoDataset.getHarmonizationTables().stream() - .map(ht -> ht.getProject() + "." + ht.getTable()).collect(Collectors.toList())); + .filter(st -> OpalTableSource.isFor(st.getSource())) + .map(ht -> OpalTableSource.toTableName(ht.getSource())).collect(Collectors.toList())); return tableNames; } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java index 9127911355..6f12151580 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/domain/MicaConfig.java @@ -41,10 +41,6 @@ public class MicaConfig extends AbstractAuditableDocument { public static final String DEFAULT_CHARSET = Charsets.UTF_8.toString(); - public static final String DEFAULT_OPAL = "https://localhost:8443"; - - public static final String DEFAULT_PUBLIC_URL = "http://localhost:8082"; - public static final String[] LAYOUT_OPTIONS = {"layout1", "layout2"}; public static final long DEFAULT_MAX_ITEMS_PER_SET = 20000; @@ -64,7 +60,7 @@ public class MicaConfig extends AbstractAuditableDocument { @NotBlank private String defaultCharacterSet = DEFAULT_CHARSET; - private String opal = DEFAULT_OPAL; + private String opal; private List roles = Lists.newArrayList(Membership.CONTACT, Membership.INVESTIGATOR); @@ -221,8 +217,12 @@ public void setDefaultCharacterSet(String defaultCharacterSet) { this.defaultCharacterSet = defaultCharacterSet; } + public boolean hasOpal() { + return !Strings.isNullOrEmpty(opal); + } + public String getOpal() { - return opal == null ? "" : opal; + return opal; } public void setOpal(String opal) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java similarity index 51% rename from mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java rename to mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java index becf167450..8ccd2407a4 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/event/OpalTaxonomiesUpdatedEvent.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/event/VariableTaxonomiesUpdatedEvent.java @@ -15,21 +15,17 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentMap; +import java.util.Map; -public class OpalTaxonomiesUpdatedEvent { +public class VariableTaxonomiesUpdatedEvent { - private ConcurrentMap opalTaxonomies; + private Map taxonomies; - public OpalTaxonomiesUpdatedEvent(ConcurrentMap opalTaxonomies) { - this.opalTaxonomies = opalTaxonomies; + public VariableTaxonomiesUpdatedEvent(Map taxonomies) { + this.taxonomies = taxonomies; } - public List extractOpalTaxonomies() { - return new ArrayList<>(opalTaxonomies.values()); - } - - public boolean hasOpalTaxonomies() { - return opalTaxonomies != null; + public List getTaxonomies() { + return new ArrayList<>(taxonomies.values()); } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java index 22bbacab51..a6c61924b5 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/CacheService.java @@ -10,8 +10,7 @@ package org.obiba.mica.micaConfig.service; -import javax.inject.Inject; - +import com.google.common.eventbus.EventBus; import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.dataset.domain.Dataset; import org.obiba.mica.dataset.service.CollectedDatasetService; @@ -23,7 +22,7 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import com.google.common.eventbus.EventBus; +import javax.inject.Inject; @Component public class CacheService { @@ -43,18 +42,18 @@ public class CacheService { private EventBus eventBus; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; - @CacheEvict(value = "opal-taxonomies", allEntries = true, beforeInvocation = true) - public void clearOpalTaxonomiesCache() { - log.info("Clearing opal taxonomies cache"); - taxonomyService.getOpalTaxonomies(); + @CacheEvict(value = "variable-taxonomies", allEntries = true, beforeInvocation = true) + public void clearTaxonomiesCache() { + log.info("Clearing variable taxonomies cache"); + taxonomiesService.getVariableTaxonomies(); } @CacheEvict(value = "micaConfig", allEntries = true) public void clearMicaConfigCache() { log.info("Clearing mica config cache"); - taxonomyService.refresh(); + taxonomiesService.refresh(); } @CacheEvict(value = "aggregations-metadata", allEntries = true) @@ -77,7 +76,7 @@ public void buildDatasetVariablesCache() { public void clearAllCaches() { log.info("Clearing all caches"); - clearOpalTaxonomiesCache(); + clearTaxonomiesCache(); clearMicaConfigCache(); clearAggregationsMetadataCache(); clearDatasetVariablesCache(); @@ -108,10 +107,10 @@ public void buildDatasetVariablesCache() { String studyId = st.getStudyId(); try { harmonizedDatasetService - .getVariableSummary(dataset, v.getName(), studyId, st.getProject(), st.getTable()); - } catch(NoSuchVariableException ex) { + .getVariableSummary(dataset, v.getName(), studyId, st.getSource()); + } catch (NoSuchVariableException ex) { //ignore - } catch(Exception e) { + } catch (Exception e) { log.warn("Error building dataset variable cache of harmonization dataset {}: {} {}", dataset.getId(), st, v, e); } @@ -121,9 +120,9 @@ public void buildDatasetVariablesCache() { .forEach(dataset -> collectedDatasetService.getDatasetVariables(dataset).forEach(v -> { try { collectedDatasetService.getVariableSummary(dataset, v.getName()); - } catch(NoSuchVariableException ex) { + } catch (NoSuchVariableException ex) { //ignore - } catch(Exception e) { + } catch (Exception e) { log.warn("Error building dataset variable cache of study dataset {}: {}", dataset.getId(), v, e); } })); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java index 3f0af22c13..7e0d09706b 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigService.java @@ -94,35 +94,6 @@ public ObjectMapper getObjectMapper() { return objectMapper; } - public Taxonomy getTaxonomy(TaxonomyTarget target) { - return taxonomyConfigService.findByTarget(target); - } - - @NotNull - Taxonomy getNetworkTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.NETWORK); - } - - @NotNull - Taxonomy getStudyTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.STUDY); - } - - @NotNull - Taxonomy getDatasetTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.DATASET); - } - - @NotNull - Taxonomy getVariableTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.VARIABLE); - } - - @NotNull - Taxonomy getTaxonomyTaxonomy() { - return taxonomyConfigService.findByTarget(TaxonomyTarget.TAXONOMY); - } - @Cacheable(value = "micaConfig", key = "#root.methodName") public MicaConfig getConfig() { return getOrCreateMicaConfig(); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java index 63efbb76c6..2a5807731a 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/MicaConfigurationProvider.java @@ -27,7 +27,7 @@ public class MicaConfigurationProvider implements ConfigurationProvider { @Inject @Lazy - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Override public List getLocales() { @@ -46,26 +46,26 @@ public ObjectMapper getObjectMapper() { @Override public Taxonomy getNetworkTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } @Override public Taxonomy getStudyTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } @Override public Taxonomy getVariableTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } @Override public Taxonomy getDatasetTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } @Override public List getVariableTaxonomies() { - return taxonomyService.getVariableTaxonomies(); + return taxonomiesService.getAllVariableTaxonomies(); } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java index cfb12a6a7a..01c02296b8 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/OpalService.java @@ -11,19 +11,15 @@ package org.obiba.mica.micaConfig.service; import com.google.common.base.Strings; -import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; import org.obiba.magma.support.Initialisables; import org.obiba.mica.dataset.service.KeyStoreService; import org.obiba.mica.micaConfig.AuthType; import org.obiba.mica.micaConfig.domain.OpalCredential; -import org.obiba.mica.micaConfig.service.helper.OpalServiceHelper; -import org.obiba.opal.core.cfg.NoSuchTaxonomyException; -import org.obiba.opal.core.cfg.NoSuchVocabularyException; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; -import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.obiba.opal.rest.client.magma.OpalJavaClient; import org.obiba.opal.rest.client.magma.RestDatasource; import org.obiba.opal.rest.client.magma.RestDatasourceFactory; @@ -67,9 +63,6 @@ public class OpalService implements EnvironmentAware { @Inject private OpalCredentialService opalCredentialService; - @Inject - private OpalServiceHelper opalServiceHelper; - @Override public void setEnvironment(Environment environment) { this.environment = environment; @@ -84,7 +77,7 @@ public void setEnvironment(Environment environment) { */ public synchronized RestDatasource getDatasource(@Nullable String opalUrl, String project) { final String projectUrl = getOpalProjectUrl(opalUrl, project); - opalUrl = Strings.isNullOrEmpty(opalUrl) ? getDefaultOpal() : opalUrl; + opalUrl = Strings.isNullOrEmpty(opalUrl) ? getPrimaryOpal() : opalUrl; OpalCredential opalCredential = getOpalCredential(opalUrl); @@ -110,202 +103,105 @@ public synchronized RestDatasource getDatasource(@Nullable String opalUrl, Strin return datasource; } - private RestDatasource createRestDatasource(OpalCredential opalCredential, String projectUrl, String opalUrl, - String project) { - if (opalCredential.getAuthType() == AuthType.CERTIFICATE) { - KeyStoreManager kms = keyStoreService.getKeyStore(OPAL_KEYSTORE); - - if (!kms.aliasExists(opalCredential.getOpalUrl())) throw new IllegalStateException( - "Trying to use opal certificate credential but could not be found in keystore."); - - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, kms.getKeyStore(), opalUrl, - micaConfigService.getConfig().getSecretKey(), project).create(); - } else if (opalCredential.getAuthType() == AuthType.TOKEN) - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getToken(), project).create(); - - return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getUsername(), - opalCredential.getPassword(), project).create(); - } - /** - * Get a {@link RestDatasource} from the default Opal server. + * Get the url of the default Opal server as defined in the configuration. * - * @param project * @return */ - public RestDatasource getDatasource(String project) { - return getDatasource(getDefaultOpal(), project); - } - - private String getOpalProjectUrl(String opalUrl, String project) { - String baseUrl = opalUrl == null ? getDefaultOpal() : opalUrl; - - return String.format("%s/ws/datasource/%s", StringUtils.stripEnd(baseUrl, "/"), project); + private String getPrimaryOpal() { + String opalConf = micaConfigService.getConfig().getOpal(); + String opalDefault = environment.getProperty("opal.url"); + return Strings.isNullOrEmpty(opalConf) ? opalDefault : opalConf; } - /** - * Get the url of the default Opal server as defined in the configuration. - * - * @return - */ - public String getDefaultOpal() { - return environment.getProperty("opal.url"); + public boolean hasPrimaryOpal() { + String primaryOpal = getPrimaryOpal(); + return !Strings.isNullOrEmpty(primaryOpal) && primaryOpal.toLowerCase().startsWith("http"); } // - // Taxonomies + // Opal Project // - public List getTaxonomies() { - Map taxonomies = getTaxonomiesInternal(); - List taxonomyList = Lists.newArrayList(taxonomies.values()); - Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); - return taxonomyList; - } - - public List getTaxonomyDtos() { - return getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); - } - - /** - * Get a summary of all the {@link Taxonomy}s available from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asSummaryDto) - .collect(Collectors.toList()); - - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); - } - - /** - * Get a summary of the {@link Taxonomy} available from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { - return Dtos.asSummaryDto(getTaxonomy(name)); - } + public List getProjectDtos(String opalUrl) throws URISyntaxException { + if (Strings.isNullOrEmpty(opalUrl)) opalUrl = getPrimaryOpal(); - /** - * Get a summary of all the {@link Taxonomy}s with their - * {@link Vocabulary}s from Opal master. - * - * @return - */ - public Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { - List summaries = getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) - .collect(Collectors.toList()); + OpalJavaClient opalJavaClient = getOpalJavaClient(opalUrl); + URI uri = opalJavaClient.newUri().segment("projects").build(); - return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + return opalJavaClient.getResources(Projects.ProjectDto.class, uri, Projects.ProjectDto.newBuilder()); } - /** - * Get a summary of the {@link Taxonomy} with its - * {@link Vocabulary}s from Opal master. - * - * @param name the taxonomy name - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { - return Dtos.asVocabularySummaryDto(getTaxonomy(name)); - } + // + // Entities count + // - /** - * Get a summary of the {@link Vocabulary} from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, - String vocabularyName) { - for (Vocabulary voc : getTaxonomy(name).getVocabularies()) { - if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); + public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, String entityType) { + try { + return getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); + } catch (URISyntaxException e) { + log.error("Malformed opal URI", e); + throw new NoSuchElementException(); } - throw new NoSuchVocabularyException(name, vocabularyName); } - /** - * Get the {@link Taxonomy} from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Taxonomy getTaxonomy(String name) { - return Dtos.fromDto(getTaxonomyDto(name)); - } - - /** - * Get the {@link Taxonomy} as a Dto from Opal master. - * - * @param name - * @return - * @throws NoSuchTaxonomyException - */ - public Opal.TaxonomyDto getTaxonomyDto(String name) { - Map taxonomies = getTaxonomiesInternal(); + // + // Private/package methods + // - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); + synchronized Map getTaxonomiesInternal() { + if (hasPrimaryOpal()) { + try { + return getTaxonomies(getOpalJavaClient()); + } catch (Exception e) { + log.error("Cannot retrieve Opal taxonomies", e); + throw new NoSuchElementException(); + } + } else { + return Maps.newHashMap(); } - - return Dtos.asDto(taxonomies.get(name)); } - /** - * Get the {@link Vocabulary} as a Dto from Opal master. - * - * @param name - * @param vocabularyName - * @return - */ - public Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { - Map taxonomies = getTaxonomiesInternal(); - - if (!taxonomies.containsKey(name)) { - throw new NoSuchTaxonomyException(name); - } - - return Dtos.asDto(taxonomies.get(name).getVocabulary(vocabularyName)); + private Map getTaxonomies(OpalJavaClient opalJavaClient) { + log.info("Fetching opal taxonomies"); + URI uri = opalJavaClient.newUri().segment("system", "conf", "taxonomies").build(); + List taxonomies = opalJavaClient + .getResources(Opal.TaxonomyDto.class, uri, Opal.TaxonomyDto.newBuilder()); + return taxonomies.stream() + .map(Dtos::fromDto).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); } - public List getProjectDtos(String opalUrl) throws URISyntaxException { - if (Strings.isNullOrEmpty(opalUrl)) opalUrl = getDefaultOpal(); + private Search.EntitiesResultDto getEntitiesCount(OpalJavaClient opalJavaClient, String query, String entityType) { + log.info("Fetching opal entities count"); + log.debug(" Entities query: {}", query); + URI uri = opalJavaClient.newUri().segment("datasources", "entities", "_count") + .query("query", query) + .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); + Search.EntitiesResultDto result = opalJavaClient.getResource(Search.EntitiesResultDto.class, uri, Search.EntitiesResultDto.newBuilder()); + return result; + } - OpalJavaClient opalJavaClient = getOpalJavaClient(opalUrl); - URI uri = opalJavaClient.newUri().segment("projects").build(); + private RestDatasource createRestDatasource(OpalCredential opalCredential, String projectUrl, String opalUrl, + String project) { + if (opalCredential.getAuthType() == AuthType.CERTIFICATE) { + KeyStoreManager kms = keyStoreService.getKeyStore(OPAL_KEYSTORE); - return opalJavaClient.getResources(Projects.ProjectDto.class, uri, Projects.ProjectDto.newBuilder()); - } + if (!kms.aliasExists(opalCredential.getOpalUrl())) throw new IllegalStateException( + "Trying to use opal certificate credential but could not be found in keystore."); - // - // Private methods - // + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, kms.getKeyStore(), opalUrl, + micaConfigService.getConfig().getSecretKey(), project).create(); + } else if (opalCredential.getAuthType() == AuthType.TOKEN) + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getToken(), project).create(); - private synchronized Map getTaxonomiesInternal() { - try { - return opalServiceHelper.getTaxonomies(getOpalJavaClient()); - } catch (URISyntaxException e) { - log.error("Malformed opal URI", e); - throw new NoSuchElementException(); - } + return (RestDatasource) new RestDatasourceFactory(projectUrl, opalUrl, opalCredential.getUsername(), + opalCredential.getPassword(), project).create(); } - public Search.EntitiesResultDto getEntitiesCount(String query, String entityType) { - return getEntitiesCount(getDefaultOpal(), query, entityType); - } + private String getOpalProjectUrl(String opalUrl, String project) { + String baseUrl = opalUrl == null ? getPrimaryOpal() : opalUrl; - public Search.EntitiesResultDto getEntitiesCount(String opalUrl, String query, String entityType) { - try { - return opalServiceHelper.getEntitiesCount(getOpalJavaClient(opalUrl), query, entityType); - } catch (URISyntaxException e) { - log.error("Malformed opal URI", e); - throw new NoSuchElementException(); - } + return String.format("%s/ws/datasource/%s", StringUtils.stripEnd(baseUrl, "/"), project); } private String getOpalUsername() { @@ -324,9 +220,9 @@ private OpalJavaClient getOpalJavaClient() throws URISyntaxException { if (opalJavaClient != null) return opalJavaClient; if (Strings.isNullOrEmpty(getOpalToken())) - opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getDefaultOpal()), getOpalUsername(), getOpalPassword()); + opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getPrimaryOpal()), getOpalUsername(), getOpalPassword()); else - opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getDefaultOpal()), getOpalToken()); + opalJavaClient = new OpalJavaClient(cleanupOpalUrl(getPrimaryOpal()), getOpalToken()); return opalJavaClient; } @@ -354,9 +250,9 @@ private OpalCredential getOpalCredential(String opalUrl) { if (opalCredential.isPresent()) return opalCredential.get(); if (Strings.isNullOrEmpty(getOpalToken())) - return new OpalCredential(getDefaultOpal(), AuthType.USERNAME, getOpalUsername(), getOpalPassword()); + return new OpalCredential(getPrimaryOpal(), AuthType.USERNAME, getOpalUsername(), getOpalPassword()); else - return new OpalCredential(getDefaultOpal(), AuthType.TOKEN, getOpalToken()); + return new OpalCredential(getPrimaryOpal(), AuthType.TOKEN, getOpalToken()); } private String cleanupOpalUrl(String opalUrl) { diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java index f06ba2d76a..15fa85c264 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/PluginsService.java @@ -19,10 +19,11 @@ import org.obiba.mica.spi.search.ConfigurationProvider; import org.obiba.mica.spi.search.SearchEngineService; import org.obiba.mica.spi.search.SearchEngineServiceLoader; -import org.obiba.plugins.PluginRepositoryCache; -import org.obiba.plugins.PluginRepositoryException; -import org.obiba.plugins.PluginResources; -import org.obiba.plugins.PluginsManagerHelper; +import org.obiba.mica.spi.tables.StudyTableSourceService; +import org.obiba.mica.spi.tables.StudyTableSourceServiceLoader; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderServiceLoader; +import org.obiba.plugins.*; import org.obiba.plugins.spi.ServicePlugin; import org.obiba.runtime.Version; import org.slf4j.Logger; @@ -35,11 +36,7 @@ import javax.inject.Inject; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.stream.Collectors; @Component @@ -49,6 +46,14 @@ public class PluginsService implements EnvironmentAware { private static final String PLUGINS_PATH = "${MICA_HOME}/plugins"; + private static final String MICA_SEARCH_PLUGIN_TYPE = "mica-search"; + + private static final String MICA_TABLES_PLUGIN_TYPE = "mica-tables"; + + private static final String MICA_TAXONOMIES_PLUGIN_TYPE = "mica-taxonomies"; + + + private static final String MICA_SEARCH_PLUGIN_NAME = "plugins.micaSearchPlugin"; private static final String DEFAULT_MICA_SEARCH_PLUGIN_NAME = "mica-search-es"; @@ -86,12 +91,205 @@ public SearchEngineService getSearchEngineService() { return (SearchEngineService) getServicePlugins(SearchEngineService.class).iterator().next(); } + public Collection getStudyTableSourceServices() { + return getServicePlugins(StudyTableSourceService.class).stream() + .map(service -> (StudyTableSourceService)service) + .collect(Collectors.toList()); + } + + public Collection getTaxonomiesProviderServices() { + return getServicePlugins(TaxonomiesProviderService.class).stream() + .map(service -> (TaxonomiesProviderService)service) + .collect(Collectors.toList()); + } + + /** + * Get the location of the plugin packages repository. + * + * @return + */ + public String getUpdateSite() { + return environment.getProperty("plugins.updateSite", DEFAULT_PLUGINS_UPDATE_SITE); + } + + /** + * Get the last time at which the update site was successfully. + * + * @return + */ + public Date getLastUpdate() { + return getPluginRepositoryCache().getLastUpdate(); + } + + /** + * Reports if system restart is required to finalize plugin installation. + * + * @return + */ + public boolean restartRequired() { + File[] children = pluginsDir.listFiles(pathname -> !pathname.getName().startsWith(".")); + if (children == null || children.length == 0) return false; + for (File child : children) { + if (child.isFile() && child.getName().endsWith(PluginResources.PLUGIN_DIST_SUFFIX)) return true; + if (child.isDirectory() && new File(child, PluginResources.UNINSTALL_FILE).exists()) return true; + } + return false; + } + + /** + * Get the plugins registered in the system. + * + * @return + */ + public List getInstalledPlugins() { + return registeredPlugins.stream() + .map(PluginPackage::new) + .collect(Collectors.toList()); + } + + /** + * Get the list of plugins that are marked to uninstallation. + * + * @return + */ + public Collection getUninstalledPluginNames() { + List names = Lists.newArrayList(); + File[] children = pluginsDir.listFiles(pathname -> pathname.isDirectory() && !pathname.getName().startsWith(".")); + if (children == null || children.length == 0) return names; + for (File child : children) { + PluginResources plugin = new MicaPlugin(child); + if (plugin.isToUninstall()) names.add(plugin.getName()); + } + return names; + } + + /** + * Get the plugins registered in the system that can be updated according to the update site registry. + * + * @return + */ + public List getUpdatablePlugins() { + // exclude already installed plugin packages whatever the version is + return getPluginRepositoryCache().getOrUpdatePluginRepository().getPlugins().stream() + .filter(PluginPackage::hasMicaVersion) + .filter(pp -> registeredPlugins.stream().anyMatch(rp -> pp.isNewerThan(rp.getName(), rp.getVersion()))) + .filter(pp -> runtimeVersionProvider.getVersion().compareTo(pp.getMicaVersion()) >= 0) + .collect(Collectors.toList()); + } + + /** + * Get the plugins that are not installed and that are available from the update site registry. + * + * @return + */ + public List getAvailablePlugins() { + // exclude already installed plugin packages whatever the version is + return getPluginRepositoryCache().getOrUpdatePluginRepository().getPlugins().stream() + .filter(PluginPackage::hasMicaVersion) + .filter(pp -> registeredPlugins.stream().noneMatch(rp -> pp.isSameAs(rp.getName()))) + .filter(pp -> runtimeVersionProvider.getVersion().compareTo(pp.getMicaVersion()) >= 0) + .collect(Collectors.toList()); + } + + /** + * Perform the plugin installation by retrieving the plugin package from the update site. + * + * @param name + * @param version + */ + public void installPlugin(String name, String version) { + String pVersion = version; + if (Strings.isNullOrEmpty(version)) { + // no version specified: get the latest + pVersion = getPluginRepositoryCache().getPluginLatestVersion(name); + } + try { + File tmpDir = Files.createTempDir(); + installPlugin(getPluginRepositoryCache().downloadPlugin(name, pVersion, tmpDir), true); + FileUtil.delete(tmpDir); + } catch (IOException e) { + throw new PluginRepositoryException("Failed to install plugin " + name + ":" + version + " : " + e.getMessage(), e); + } + } + + /** + * Get the installed plugin with the given name. + * + * @param name + * @return + */ + public PluginResources getInstalledPlugin(String name) { + Optional plugin = registeredPlugins.stream().filter(p -> p.getName().equals(name)).findFirst(); + if (!plugin.isPresent()) throw new NoSuchElementException("No such plugin with name: " + name); + return plugin.get(); + } + + /** + * Uninstall a plugin. + * + * @param name + */ + public void prepareUninstallPlugin(String name) { + PluginResources plugin = getInstalledPlugin(name); + plugin.prepareForUninstall(); + } + + /** + * Cancel plugin uninstallation before it is effective. + * + * @param name + */ + public void cancelUninstallPlugin(String name) { + PluginResources plugin = getInstalledPlugin(name); + plugin.cancelUninstall(); + } + + /** + * Set the site properties of the installed plugin with the given name. + * + * @param name + * @param properties + */ + public void setInstalledPluginSiteProperties(String name, String properties) { + try { + PluginResources thePlugin = getInstalledPlugin(name); + thePlugin.writeSiteProperties(properties); + updateServiceProperties(name, thePlugin.getProperties()); + } catch (IOException e) { + throw new PluginRepositoryException("Failed to save plugin " + name + " site properties: " + e.getMessage(), e); + } + } + + public boolean isInstalledPluginRunning(String name) { + return getServicePlugin(name).isRunning(); + } + + public void startInstalledPlugin(String name) { + getServicePlugin(name).start(); + } + + public void stopInstalledPlugin(String name) { + getServicePlugin(name).stop(); + } + // // Private methods // + private void updateServiceProperties(String name, Properties properties) { + ServicePlugin servicePlugin = getServicePlugin(name); + if (servicePlugin != null) { + servicePlugin.configure(properties); + } + } + + ServicePlugin getServicePlugin(String name) { + Optional service = servicePlugins.stream().filter(s -> name.equals(s.getName())).findFirst(); + if (!service.isPresent()) throw new NoSuchElementException(name); + return service.get(); + } + private Collection getServicePlugins(Class clazz) { - //init(); return servicePlugins.stream().filter(s -> clazz.isAssignableFrom(s.getClass())).collect(Collectors.toList()); } @@ -106,21 +304,25 @@ public void init() { initPlugins(); } + public String getSearchPluginName() { + return environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); + } + /** * Initialize plugin resources. */ private void initPlugins() { Collection plugins = getPlugins(true); - String pluginName = environment.getProperty(MICA_SEARCH_PLUGIN_NAME, DEFAULT_MICA_SEARCH_PLUGIN_NAME); + String searchPluginName = getSearchPluginName(); try { - String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(pluginName); + String pluginLatestVersion = getPluginRepositoryCache().getPluginLatestVersion(searchPluginName); // ensure there is a mica-search plugin installed - if (plugins.stream().noneMatch(p -> "mica-search".equals(p.getType())) + if (plugins.stream().noneMatch(p -> MICA_SEARCH_PLUGIN_TYPE.equals(p.getType())) || plugins.stream() - .filter(plugin -> pluginName.equals(plugin.getName())) + .filter(plugin -> searchPluginName.equals(plugin.getName())) .filter(plugin -> plugin.getVersion().compareTo(new Version(pluginLatestVersion)) >= 0).count() == 0) { - installPlugin(pluginName, null); + installPlugin(searchPluginName, null); // rescan plugins plugins = getPlugins(true); } @@ -130,21 +332,27 @@ private void initPlugins() { boolean micaSearchFound = false; // mica-search plugin is a singleton List filteredPlugins = - plugins.stream().filter(plugin -> pluginName.equals(plugin.getName())) + plugins.stream().filter(plugin -> searchPluginName.equals(plugin.getName()) + || MICA_TABLES_PLUGIN_TYPE.equals(plugin.getType()) + || MICA_TAXONOMIES_PLUGIN_TYPE.equals(plugin.getType())) .sorted(Comparator.comparing(PluginResources::getVersion)) .collect(Collectors.toList()); for (PluginResources plugin : filteredPlugins) { - if ("mica-search".equals(plugin.getType()) && !micaSearchFound) { + if (MICA_SEARCH_PLUGIN_TYPE.equals(plugin.getType()) && !micaSearchFound) { initSearchEngineServicePlugin(plugin); micaSearchFound = true; + } else if (MICA_TABLES_PLUGIN_TYPE.equals(plugin.getType())) { + initStudyTableSourceServicePlugin(plugin); + } else if (MICA_TAXONOMIES_PLUGIN_TYPE.equals(plugin.getType())) { + initTaxonomiesProviderServicePlugin(plugin); } } } private void initSearchEngineServicePlugin(PluginResources plugin) { SearchEngineService service = SearchEngineServiceLoader.get(plugin.getURLClassLoader(false)).iterator().next(); - Properties properties = plugin.getProperties(); + Properties properties = cleanProperties(plugin.getProperties()); for (String key : ES_CONFIGURATION) { if (environment.containsProperty(key)) properties.setProperty(key, environment.getProperty(key)); @@ -155,6 +363,30 @@ private void initSearchEngineServicePlugin(PluginResources plugin) { servicePlugins.add(service); } + private void initStudyTableSourceServicePlugin(PluginResources plugin) { + StudyTableSourceServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { + Properties properties = cleanProperties(plugin.getProperties()); + service.configure(properties); + service.start(); + servicePlugins.add(service); + }); + } + + private void initTaxonomiesProviderServicePlugin(PluginResources plugin) { + TaxonomiesProviderServiceLoader.get(plugin.getURLClassLoader(false)).forEach(service -> { + Properties properties = cleanProperties(plugin.getProperties()); + service.configure(properties); + service.start(); + servicePlugins.add(service); + }); + } + + private Properties cleanProperties(Properties properties) { + properties.setProperty("MICA_HOME", properties.getProperty("OPAL_HOME")); + properties.remove("OPAL_HOME"); + return properties; + } + private synchronized Collection getPlugins(boolean extract) { Map pluginsMap = Maps.newLinkedHashMap(); // make sure plugins directory exists @@ -181,21 +413,6 @@ private void processPlugins(Map pluginsMap, File plugin } } - private void installPlugin(String name, String version) { - String pVersion = version; - if (Strings.isNullOrEmpty(version)) { - // no version specified: get the latest - pVersion = getPluginRepositoryCache().getPluginLatestVersion(name); - } - try { - File tmpDir = Files.createTempDir(); - installPlugin(getPluginRepositoryCache().downloadPlugin(name, pVersion, tmpDir), true); - FileUtil.delete(tmpDir); - } catch (IOException e) { - throw new PluginRepositoryException("Failed to install plugin " + name + ":" + version + " : " + e.getMessage(), e); - } - } - private void installPlugin(File pluginFile, boolean rmAfterInstall) { try { if (!pluginsDir.exists()) pluginsDir.mkdirs(); @@ -208,7 +425,7 @@ private void installPlugin(File pluginFile, boolean rmAfterInstall) { private PluginRepositoryCache getPluginRepositoryCache() { if (pluginRepositoryCache == null) - pluginRepositoryCache = new PluginRepositoryCache(runtimeVersionProvider, environment.getProperty("plugins.updateSite", DEFAULT_PLUGINS_UPDATE_SITE)); + pluginRepositoryCache = new PluginRepositoryCache(runtimeVersionProvider, getUpdateSite()); return pluginRepositoryCache; } } diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java new file mode 100644 index 0000000000..7f1af9cb1e --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomiesService.java @@ -0,0 +1,539 @@ +/* + * Copyright (c) 2018 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.service; + +import com.google.common.base.Strings; +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import com.google.common.eventbus.Subscribe; +import org.apache.commons.compress.utils.Lists; +import org.obiba.mica.core.event.DocumentSetUpdatedEvent; +import org.obiba.mica.dataset.event.DatasetPublishedEvent; +import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; +import org.obiba.mica.micaConfig.domain.MicaConfig; +import org.obiba.mica.micaConfig.service.helper.*; +import org.obiba.mica.network.event.NetworkPublishedEvent; +import org.obiba.mica.network.event.NetworkUnpublishedEvent; +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.mica.study.event.StudyPublishedEvent; +import org.obiba.mica.study.event.StudyUnpublishedEvent; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; +import org.obiba.opal.core.domain.taxonomy.Term; +import org.obiba.opal.core.domain.taxonomy.Vocabulary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service +public class TaxonomiesService { + + private static final Logger log = LoggerFactory.getLogger(TaxonomiesService.class); + + private final VariableTaxonomiesService variableTaxonomiesService; + + private final MicaConfigService micaConfigService; + + private final StudyIdAggregationMetaDataHelper studyHelper; + + private final DatasetIdAggregationMetaDataHelper datasetHelper; + + private final NetworkIdAggregationMetaDataHelper networkHelper; + + private final PopulationIdAggregationMetaDataHelper populationHelper; + + private final DceIdAggregationMetaDataHelper dceHelper; + + private final NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper; + + private final StudiesSetsAggregationMetaDataHelper studiesSetsHelper; + + private final VariablesSetsAggregationMetaDataHelper variablesSetsHelper; + + private final TaxonomyConfigService taxonomyConfigService; + + private Taxonomy taxonomyTaxonomy; + + private Taxonomy variableTaxonomy; + + private Taxonomy datasetTaxonomy; + + private Taxonomy studyTaxonomy; + + private Taxonomy networkTaxonomy; + + @Inject + public TaxonomiesService( + VariableTaxonomiesService variableTaxonomiesService, + MicaConfigService micaConfigService, + StudyIdAggregationMetaDataHelper studyHelper, + DatasetIdAggregationMetaDataHelper datasetHelper, + NetworkIdAggregationMetaDataHelper networkHelper, + PopulationIdAggregationMetaDataHelper populationHelper, + DceIdAggregationMetaDataHelper dceHelper, + NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper, + StudiesSetsAggregationMetaDataHelper studiesSetsHelper, + VariablesSetsAggregationMetaDataHelper variablesSetsHelper, + TaxonomyConfigService taxonomyConfigService) { + this.variableTaxonomiesService = variableTaxonomiesService; + this.micaConfigService = micaConfigService; + this.studyHelper = studyHelper; + this.datasetHelper = datasetHelper; + this.networkHelper = networkHelper; + this.populationHelper = populationHelper; + this.dceHelper = dceHelper; + this.networksSetsAggregationMetaDataHelper = networksSetsAggregationMetaDataHelper; + this.studiesSetsHelper = studiesSetsHelper; + this.variablesSetsHelper = variablesSetsHelper; + this.taxonomyConfigService = taxonomyConfigService; + } + + @NotNull + public Taxonomy getTaxonomyTaxonomy() { + initialize(); + boolean modified = false; + List variableTaxonomies = getAllVariableTaxonomies(); + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals("variable")) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + // check variable taxonomies to be added to meta + List variableTaxonomiesNames = variableChars.getTerms().stream() + .map(TaxonomyEntity::getName).collect(Collectors.toList()); + for (Taxonomy variableTaxonomy : variableTaxonomies) { + if (!variableTaxonomiesNames.contains(variableTaxonomy.getName())) { + Term newTerm = new Term(variableTaxonomy.getName()); + newTerm.addAttribute("hidden", "true"); + newTerm.setTitle(variableTaxonomy.getTitle()); + newTerm.setDescription(variableTaxonomy.getDescription()); + variableChars.getTerms().add(newTerm); + modified = true; + } else { + // check any title/description modifications + Term term = variableChars.getTerm(variableTaxonomy.getName()); + MapDifference diff = Maps.difference(term.getTitle(), variableTaxonomy.getTitle()); + if (!diff.areEqual()) { + term.setTitle(variableTaxonomy.getTitle()); + modified = true; + } + diff = Maps.difference(term.getDescription(), variableTaxonomy.getDescription()); + if (!diff.areEqual()) { + term.setDescription(variableTaxonomy.getDescription()); + modified = true; + } + } + } + // check variable taxonomies to be removed from meta + List reverseVariableTaxonomiesNames = variableTaxonomies.stream() + .map(TaxonomyEntity::getName).collect(Collectors.toList()); + List newTerms = variableChars.getTerms().stream() + .filter(term -> "Mica_variable".equals(term.getName()) || reverseVariableTaxonomiesNames.contains(term.getName())) + .collect(Collectors.toList()); + if (newTerms.size() < variableChars.getTerms().size()) { + variableChars.setTerms(newTerms); + modified = true; + } + } + } + if (modified) { + log.debug("Taxonomy of taxonomies was modified"); + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, taxonomyTaxonomy); + } + return taxonomyTaxonomy; + } + + public boolean metaTaxonomyContains(String taxonomy) { + for (Vocabulary targetVocabulary : getTaxonomyTaxonomy().getVocabularies()) { + Optional termOpt = getTerm(targetVocabulary, taxonomy); + if (termOpt.isPresent()) { + Term term = termOpt.get(); + String hidden = term.getAttributeValue("hidden"); + // visible by default + if (Strings.isNullOrEmpty(hidden)) return true; + // check visible attribute value + try { + return !Boolean.parseBoolean(hidden.toLowerCase()); + } catch (Exception e) { + return false; + } + } + } + return false; + } + + /** + * Change described taxonomy position when there are several for the target considered. + * + * @param target + * @param name + * @param up + */ + public void moveTaxonomy(TaxonomyTarget target, String name, boolean up) { + Taxonomy metaTaxonomy = getTaxonomyTaxonomy(); + boolean modified = false; + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals(target.asId())) { + if (TaxonomyTarget.VARIABLE.equals(target)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + modified = moveTerm(variableChars.getTerms(), name, up); + } else if (vocabulary.hasTerm(name) && vocabulary.getTerms().size() > 1) { + modified = moveTerm(vocabulary.getTerms(), name, up); + } + } + } + if (modified) { + this.taxonomyTaxonomy = metaTaxonomy; + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, metaTaxonomy); + } + } + + /** + * Modify term list by moving up/down the term with provided name. + * + * @param terms + * @param taxonomyName + * @param up + * @return Whether the list was modified + */ + private boolean moveTerm(List terms, String taxonomyName, boolean up) { + int idx = -1; + for (Term term : terms) { + idx++; + if (term.getName().equals(taxonomyName)) { + break; + } + } + if (idx > -1) { + int newIdx = up ? idx - 1 : idx + 1; + if (newIdx > -1 && newIdx < terms.size()) { + Term term = terms.remove(idx); + terms.add(newIdx, term); + return true; + } + } + return false; + } + + /** + * Hide/show the described taxonomy for the target considered. + * + * @param target + * @param taxonomyName + * @param name + * @param value + */ + public void setTaxonomyAttribute(TaxonomyTarget target, String taxonomyName, String name, String value) { + if (Strings.isNullOrEmpty(name)) return; + Taxonomy metaTaxonomy = getTaxonomyTaxonomy(); + boolean modified = false; + for (Vocabulary vocabulary : taxonomyTaxonomy.getVocabularies()) { + if (vocabulary.getName().equals(target.asId())) { + if (TaxonomyTarget.VARIABLE.equals(target)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + Optional found = variableChars.getTerms().stream().filter(term -> term.getName().equals(taxonomyName)).findFirst(); + if (found.isPresent()) { + Term term = found.get(); + term.getAttributes().put(name, value); + modified = true; + } + } else if (vocabulary.hasTerm(taxonomyName)) { + Term term = vocabulary.getTerm(taxonomyName); + term.getAttributes().put(name, value); + modified = true; + } + } + } + if (modified) { + this.taxonomyTaxonomy = metaTaxonomy; + taxonomyConfigService.update(TaxonomyTarget.TAXONOMY, metaTaxonomy); + } + } + + /** + * Get the taxonomy that describes the {@link org.obiba.mica.network.domain.Network} properties. + * + * @return + */ + @NotNull + public Taxonomy getNetworkTaxonomy() { + initialize(); + return networkTaxonomy; + } + + /** + * Get the taxonomy that describes the {@link org.obiba.mica.study.domain.BaseStudy} properties. + * + * @return + */ + @NotNull + public Taxonomy getStudyTaxonomy() { + initialize(); + return studyTaxonomy; + } + + /** + * Get the taxonomy that describes the {@link org.obiba.mica.dataset.domain.Dataset} properties. + * + * @return + */ + @NotNull + public Taxonomy getDatasetTaxonomy() { + initialize(); + return datasetTaxonomy; + } + + /** + * Get the taxonomy that describes the {@link org.obiba.mica.dataset.domain.DatasetVariable} properties. + * + * @return + */ + @NotNull + public Taxonomy getVariableTaxonomy() { + initialize(); + return variableTaxonomy; + } + + /** + * Get all taxonomies that apply to the variables, including the one about the built-in properties of the {@link org.obiba.mica.dataset.domain.DatasetVariable}. + * + * @return + */ + @NotNull + public List getAllVariableTaxonomies() { + return Stream.concat(getVariableTaxonomies().stream(), Stream.of(getVariableTaxonomy())).collect(Collectors.toList()); + } + + /** + * Get the taxonomies that apply to the variables' annotations. + * + * @return + */ + @NotNull + public synchronized List getVariableTaxonomies() { + List taxonomies = null; + try { + taxonomies = variableTaxonomiesService.getTaxonomies(); + } catch (Exception e) { + // ignore + } + return taxonomies == null ? Collections.emptyList() : taxonomies; + } + + /** + * Prepare taxonomies for being re-initialized. + */ + public synchronized void refresh() { + taxonomyTaxonomy = null; + networkTaxonomy = null; + studyTaxonomy = null; + datasetTaxonomy = null; + variableTaxonomy = null; + } + + // + // Private methods + // + + private synchronized void initialize() { + initializeTaxonomyTaxonomy(); + initializeNetworkTaxonomy(); + initializeStudyTaxonomy(); + initializeDatasetTaxonomy(); + initializeVariableTaxonomy(); + } + + private void initializeTaxonomyTaxonomy() { + if (taxonomyTaxonomy == null) + taxonomyTaxonomy = copy(findTaxonomy(TaxonomyTarget.TAXONOMY)); + MicaConfig config = micaConfigService.getConfig(); + if (!config.isNetworkEnabled() || config.isSingleNetworkEnabled()) { + hideMetaVocabularyTerms("network"); + } + if (!config.isStudyDatasetEnabled() && !config.isHarmonizationDatasetEnabled()) { + hideMetaVocabularyTerms("dataset"); + hideMetaVocabularyTerms("variable"); + } + if (config.isSingleStudyEnabled() && !config.isHarmonizationDatasetEnabled()) { + hideMetaVocabularyTerms("study"); + } + } + + private void hideMetaVocabularyTerms(String vocabularyName) { + if (taxonomyTaxonomy.hasVocabulary(vocabularyName)) { + Vocabulary vocabulary = taxonomyTaxonomy.getVocabulary(vocabularyName); + if (TaxonomyTarget.VARIABLE.asId().equals(vocabularyName)) { + Term variableChars = vocabulary.getTerm("Variable_chars"); + if (variableChars.hasTerms()) + variableChars.getTerms().forEach(term -> term.addAttribute("hidden", "true")); + } + else if (vocabulary.hasTerms()) { + vocabulary.getTerms().forEach(term -> term.addAttribute("hidden", "true")); + } + } + } + + private void initializeNetworkTaxonomy() { + if (networkTaxonomy != null) return; + networkTaxonomy = copy(findTaxonomy(TaxonomyTarget.NETWORK)); + networkHelper.applyIdTerms(networkTaxonomy, "id"); + networksSetsAggregationMetaDataHelper.applyIdTerms(networkTaxonomy, "sets"); + studyHelper.applyIdTerms(networkTaxonomy, "studyIds"); + } + + private void initializeStudyTaxonomy() { + if (studyTaxonomy != null) return; + studyTaxonomy = copy(findTaxonomy(TaxonomyTarget.STUDY)); + studyHelper.applyIdTerms(studyTaxonomy, "id"); + studiesSetsHelper.applyIdTerms(studyTaxonomy, "sets"); + } + + private void initializeDatasetTaxonomy() { + if (datasetTaxonomy != null) return; + datasetTaxonomy = copy(findTaxonomy(TaxonomyTarget.DATASET)); + datasetHelper.applyIdTerms(datasetTaxonomy, "id"); + } + + private void initializeVariableTaxonomy() { + if (variableTaxonomy != null) return; + variableTaxonomy = copy(findTaxonomy(TaxonomyTarget.VARIABLE)); + studyHelper.applyIdTerms(variableTaxonomy, "studyId"); + datasetHelper.applyIdTerms(variableTaxonomy, "datasetId"); + populationHelper.applyIdTerms(variableTaxonomy, "populationId"); + dceHelper.applyIdTerms(variableTaxonomy, "dceId"); + variablesSetsHelper.applyIdTerms(variableTaxonomy, "sets"); + } + + private Taxonomy copy(Taxonomy source) { + Taxonomy target = new Taxonomy(); + BeanUtils.copyProperties(source, target, "vocabularies"); + if (source.hasVocabularies()) { + source.getVocabularies().forEach(sourceVoc -> { + Vocabulary targetVoc = new Vocabulary(); + BeanUtils.copyProperties(sourceVoc, targetVoc, "terms"); + if (sourceVoc.hasTerms()) { + sourceVoc.getTerms().forEach(sourceTerm -> { + Term targetTerm = new Term(); + BeanUtils.copyProperties(sourceTerm, targetTerm); + targetVoc.addTerm(targetTerm); + }); + } + target.addVocabulary(targetVoc); + }); + } + + return target; + } + + /** + * Check if vocabulary has a term with the given name. + * + * @param vocabulary + * @param name + * @return + */ + private Optional getTerm(Vocabulary vocabulary, String name) { + if (!vocabulary.hasTerms()) return Optional.empty(); + if (vocabulary.hasTerm(name)) return Optional.of(vocabulary.getTerm(name)); + for (Term t : vocabulary.getTerms()) { + Optional res = getTerm(t, name); + if (res.isPresent()) return res; + } + return Optional.empty(); + } + + /** + * Check if term has a term with the given name. + * + * @param term + * @param name + * @return + */ + private Optional getTerm(Term term, String name) { + if (!term.hasTerms()) return Optional.empty(); + if (term.hasTerm(name)) return Optional.of(term.getTerm(name)); + for (Term t : term.getTerms()) { + Optional res = getTerm(t, name); + if (res.isPresent()) return res; + } + return Optional.empty(); + } + + private Taxonomy findTaxonomy(TaxonomyTarget target) { + return taxonomyConfigService.findByTarget(target); + } + + // + // Event handling + // + + @Async + @Subscribe + public void networkPublished(NetworkPublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void networkUnpublished(NetworkUnpublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void studyPublished(StudyPublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void studyUnpublished(StudyUnpublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void datasetPublished(DatasetPublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void datasetUnpublished(DatasetUnpublishedEvent event) { + refresh(); + } + + @Async + @Subscribe + public void documentSetUpdated(DocumentSetUpdatedEvent event) { + refresh(); + } + + public void refreshTaxonomyTaxonomyIfNeeded(MicaConfig currentConfig, MicaConfig newConfig) { + if (currentConfig.isSingleStudyEnabled() != newConfig.isSingleStudyEnabled() + || currentConfig.isNetworkEnabled() != newConfig.isNetworkEnabled() + || currentConfig.isSingleNetworkEnabled() != newConfig.isSingleNetworkEnabled() + || currentConfig.isStudyDatasetEnabled() != newConfig.isStudyDatasetEnabled() + || currentConfig.isHarmonizationDatasetEnabled() != newConfig.isHarmonizationDatasetEnabled()) { + + taxonomyTaxonomy = null; + } + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java index 9476c2b5a2..1f7bb1c259 100644 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyConfigService.java @@ -17,7 +17,7 @@ import org.obiba.mica.NoSuchEntityException; import org.obiba.mica.core.domain.TaxonomyEntityWrapper; -import org.obiba.mica.core.support.YamlClassPathResourceReader; +import org.obiba.mica.core.support.YamlResourceReader; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; import org.obiba.mica.micaConfig.repository.TaxonomyConfigRepository; import org.obiba.mica.spi.search.TaxonomyTarget; @@ -75,12 +75,10 @@ public void mergeWithDefault(TaxonomyTarget target) { } private Taxonomy readTaxonomyFromYaml(String yamlResourcePath) { - return YamlClassPathResourceReader.read(yamlResourcePath, Taxonomy.class); + return YamlResourceReader.readClassPath(yamlResourcePath, Taxonomy.class); } private Taxonomy findByTargetInternal(TaxonomyTarget target) { - // taxonomy of taxonomies is not editable so fall back to the one that comes from the classpath - if(TaxonomyTarget.TAXONOMY.equals(target)) return defaultTaxonomyTaxonomy; String id = target.asId(); Optional taxonomyEntityWrapper = taxonomyConfigRepository.findById(id); diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java deleted file mode 100644 index c2d105378a..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/TaxonomyService.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.micaConfig.service; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.inject.Inject; -import javax.validation.constraints.NotNull; - -import org.obiba.mica.core.event.DocumentSetUpdatedEvent; -import org.obiba.mica.dataset.event.DatasetPublishedEvent; -import org.obiba.mica.dataset.event.DatasetUnpublishedEvent; -import org.obiba.mica.micaConfig.domain.MicaConfig; -import org.obiba.mica.micaConfig.service.helper.*; -import org.obiba.mica.network.event.NetworkPublishedEvent; -import org.obiba.mica.network.event.NetworkUnpublishedEvent; -import org.obiba.mica.study.event.StudyPublishedEvent; -import org.obiba.mica.study.event.StudyUnpublishedEvent; -import org.obiba.opal.core.domain.taxonomy.Taxonomy; -import org.obiba.opal.core.domain.taxonomy.Term; -import org.obiba.opal.core.domain.taxonomy.Vocabulary; -import org.springframework.beans.BeanUtils; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; - -import com.google.common.eventbus.Subscribe; - -@Service -public class TaxonomyService { - - private final OpalService opalService; - - private final MicaConfigService micaConfigService; - - private final StudyIdAggregationMetaDataHelper studyHelper; - - private final DatasetIdAggregationMetaDataHelper datasetHelper; - - private final NetworkIdAggregationMetaDataHelper networkHelper; - - private final PopulationIdAggregationMetaDataHelper populationHelper; - - private final DceIdAggregationMetaDataHelper dceHelper; - - private final NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper; - - private final StudiesSetsAggregationMetaDataHelper studiesSetsHelper; - - private final VariablesSetsAggregationMetaDataHelper variablesSetsHelper; - - private Taxonomy taxonomyTaxonomy; - - private Taxonomy variableTaxonomy; - - private Taxonomy datasetTaxonomy; - - private Taxonomy studyTaxonomy; - - private Taxonomy networkTaxonomy; - - @Inject - public TaxonomyService( - OpalService opalService, - MicaConfigService micaConfigService, - StudyIdAggregationMetaDataHelper studyHelper, - DatasetIdAggregationMetaDataHelper datasetHelper, - NetworkIdAggregationMetaDataHelper networkHelper, - PopulationIdAggregationMetaDataHelper populationHelper, - DceIdAggregationMetaDataHelper dceHelper, - NetworksSetsAggregationMetaDataHelper networksSetsAggregationMetaDataHelper, - StudiesSetsAggregationMetaDataHelper studiesSetsHelper, - VariablesSetsAggregationMetaDataHelper variablesSetsHelper) { - this.opalService = opalService; - this.micaConfigService = micaConfigService; - this.studyHelper = studyHelper; - this.datasetHelper = datasetHelper; - this.networkHelper = networkHelper; - this.populationHelper = populationHelper; - this.dceHelper = dceHelper; - this.networksSetsAggregationMetaDataHelper = networksSetsAggregationMetaDataHelper; - this.studiesSetsHelper = studiesSetsHelper; - this.variablesSetsHelper = variablesSetsHelper; - } - - @NotNull - public Taxonomy getTaxonomyTaxonomy() { - initialize(); - return taxonomyTaxonomy; - } - - public boolean metaTaxonomyContains(String taxonomy) { - for(Vocabulary target : getTaxonomyTaxonomy().getVocabularies()) { - if(hasTerm(target, taxonomy)) return true; - } - return false; - } - - @NotNull - public Taxonomy getNetworkTaxonomy() { - initialize(); - return networkTaxonomy; - } - - @NotNull - public Taxonomy getStudyTaxonomy() { - initialize(); - return studyTaxonomy; - } - - @NotNull - public Taxonomy getDatasetTaxonomy() { - initialize(); - return datasetTaxonomy; - } - - @NotNull - public Taxonomy getVariableTaxonomy() { - initialize(); - return variableTaxonomy; - } - - @NotNull - public List getVariableTaxonomies() { - return Stream.concat(getOpalTaxonomies().stream(), Stream.of(getVariableTaxonomy())).collect(Collectors.toList()); - } - - @NotNull - public synchronized List getOpalTaxonomies() { - List taxonomies = null; - - try { - taxonomies = opalService.getTaxonomies(); - } catch(Exception e) { - // ignore - } - - return taxonomies == null ? Collections.emptyList() : taxonomies; - } - - public synchronized void refresh() { - taxonomyTaxonomy = null; - networkTaxonomy = null; - studyTaxonomy = null; - datasetTaxonomy = null; - variableTaxonomy = null; - } - - // - // Private methods - // - - private synchronized void initialize() { - initializeTaxonomyTaxonomy(); - initializeNetworkTaxonomy(); - initializeStudyTaxonomy(); - initializeDatasetTaxonomy(); - initializeVariableTaxonomy(); - } - - private void initializeTaxonomyTaxonomy() { - if(taxonomyTaxonomy != null) return; - taxonomyTaxonomy = copy(micaConfigService.getTaxonomyTaxonomy()); - MicaConfig config = micaConfigService.getConfig(); - if(!config.isNetworkEnabled() || config.isSingleNetworkEnabled()) { - taxonomyTaxonomy.removeVocabulary("network"); - } - if(!config.isStudyDatasetEnabled() && !config.isHarmonizationDatasetEnabled()) { - taxonomyTaxonomy.removeVocabulary("dataset"); - taxonomyTaxonomy.removeVocabulary("variable"); - } - if(config.isSingleStudyEnabled() && !config.isHarmonizationDatasetEnabled()) { - taxonomyTaxonomy.removeVocabulary("study"); - } - } - - private void initializeNetworkTaxonomy() { - if(networkTaxonomy != null) return; - networkTaxonomy = copy(micaConfigService.getNetworkTaxonomy()); - networkHelper.applyIdTerms(networkTaxonomy, "id"); - networksSetsAggregationMetaDataHelper.applyIdTerms(networkTaxonomy, "sets"); - studyHelper.applyIdTerms(networkTaxonomy, "studyIds"); - } - - private void initializeStudyTaxonomy() { - if(studyTaxonomy != null) return; - studyTaxonomy = copy(micaConfigService.getStudyTaxonomy()); - studyHelper.applyIdTerms(studyTaxonomy, "id"); - studiesSetsHelper.applyIdTerms(studyTaxonomy, "sets"); - } - - private void initializeDatasetTaxonomy() { - if(datasetTaxonomy != null) return; - datasetTaxonomy = copy(micaConfigService.getDatasetTaxonomy()); - datasetHelper.applyIdTerms(datasetTaxonomy, "id"); - } - - private void initializeVariableTaxonomy() { - if(variableTaxonomy != null) return; - variableTaxonomy = copy(micaConfigService.getVariableTaxonomy()); - studyHelper.applyIdTerms(variableTaxonomy, "studyId"); - datasetHelper.applyIdTerms(variableTaxonomy, "datasetId"); - populationHelper.applyIdTerms(variableTaxonomy, "populationId"); - dceHelper.applyIdTerms(variableTaxonomy, "dceId"); - variablesSetsHelper.applyIdTerms(variableTaxonomy, "sets"); - } - - private Taxonomy copy(Taxonomy source) { - Taxonomy target = new Taxonomy(); - BeanUtils.copyProperties(source, target, "vocabularies"); - if(source.hasVocabularies()) { - source.getVocabularies().forEach(sourceVoc -> { - Vocabulary targetVoc = new Vocabulary(); - BeanUtils.copyProperties(sourceVoc, targetVoc, "terms"); - if(sourceVoc.hasTerms()) { - sourceVoc.getTerms().forEach(sourceTerm -> { - Term targetTerm = new Term(); - BeanUtils.copyProperties(sourceTerm, targetTerm); - targetVoc.addTerm(targetTerm); - }); - } - target.addVocabulary(targetVoc); - }); - } - - return target; - } - - /** - * Check if vocabulary has a term with the given name. - * - * @param vocabulary - * @param name - * @return - */ - private boolean hasTerm(Vocabulary vocabulary, String name) { - if(!vocabulary.hasTerms()) return false; - if(vocabulary.hasTerm(name)) return true; - for(Term t : vocabulary.getTerms()) { - if(hasTerm(t, name)) return true; - } - return false; - } - - /** - * Check if term has a term with the givane name. - * - * @param term - * @param name - * @return - */ - private boolean hasTerm(Term term, String name) { - if(!term.hasTerms()) return false; - if(term.hasTerm(name)) return true; - for(Term t : term.getTerms()) { - if(hasTerm(t, name)) return true; - } - return false; - } - - // - // Event handling - // - - @Async - @Subscribe - public void networkPublished(NetworkPublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void networkUnpublished(NetworkUnpublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void studyPublished(StudyPublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void studyUnpublished(StudyUnpublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void datasetPublished(DatasetPublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void datasetUnpublished(DatasetUnpublishedEvent event) { - refresh(); - } - - @Async - @Subscribe - public void documentSetUpdated(DocumentSetUpdatedEvent event) { - refresh(); - } - - public void refreshTaxonomyTaxonomyIfNeeded(MicaConfig currentConfig, MicaConfig newConfig) { - if (currentConfig.isSingleStudyEnabled() != newConfig.isSingleStudyEnabled() - || currentConfig.isNetworkEnabled() != newConfig.isNetworkEnabled() - || currentConfig.isSingleNetworkEnabled() != newConfig.isSingleNetworkEnabled() - || currentConfig.isStudyDatasetEnabled() != newConfig.isStudyDatasetEnabled() - || currentConfig.isHarmonizationDatasetEnabled() != newConfig.isHarmonizationDatasetEnabled()) { - - taxonomyTaxonomy = null; - } - } -} diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java new file mode 100644 index 0000000000..ca40443d7d --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/VariableTaxonomiesService.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.service; + + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import org.obiba.mica.core.support.YamlResourceReader; +import org.obiba.mica.micaConfig.event.VariableTaxonomiesUpdatedEvent; +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.mica.spi.search.support.AttributeKey; +import org.obiba.mica.spi.taxonomies.TaxonomiesProviderService; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.context.EnvironmentAware; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.io.File; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Access to the variable taxonomies, from the different providers, including opal, local files and plugins. + */ +@Component +public class VariableTaxonomiesService implements EnvironmentAware { + + private static final Logger log = LoggerFactory.getLogger(VariableTaxonomiesService.class); + + private static final String VARIABLE_TAXONOMIES_PATH = "${MICA_HOME}/conf/taxonomies/variable"; + + private Environment environment; + + @Inject + private EventBus eventBus; + + @Inject + private OpalService opalService; + + @Inject + private PluginsService pluginsService; + + private File variableTaxonomiesDir; + + @PostConstruct + public void init() { + if (variableTaxonomiesDir == null) { + variableTaxonomiesDir = new File(VARIABLE_TAXONOMIES_PATH.replace("${MICA_HOME}", System.getProperty("MICA_HOME"))); + } + } + + @Cacheable(value = "variable-taxonomies", key = "'variable'") + public List getTaxonomies() { + List taxonomyList = Lists.newArrayList(getTaxonomiesMap().values()); + Collections.sort(taxonomyList, Comparator.comparing(TaxonomyEntity::getName)); + return taxonomyList; + } + + @Override + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + // + // Private methods + // + + private Map getTaxonomiesMap() { + Map taxonomies = Maps.newConcurrentMap(); + // init with the ones from opal + try { + taxonomies.putAll(opalService.getTaxonomiesInternal()); + if (log.isDebugEnabled()) + taxonomies.keySet().forEach(name -> log.debug("Taxonomy from opal: {}", name)); + } catch (Exception e) { + // ignore + } + // read local files + if (variableTaxonomiesDir.exists()) { + File[] yamlFiles = variableTaxonomiesDir.listFiles(file -> !file.isDirectory() && file.getName().endsWith(".yml")); + if (yamlFiles != null) { + log.info("Fetching local taxonomies: {}", VARIABLE_TAXONOMIES_PATH); + for (File yamlFile : yamlFiles) { + try { + Taxonomy taxonomy = YamlResourceReader.readFile(yamlFile.getAbsolutePath(), Taxonomy.class); + log.debug("Taxonomy from folder {}: {}", variableTaxonomiesDir.getAbsolutePath(), taxonomy.getName()); + // override any duplicated taxonomy + if (taxonomies.containsKey(taxonomy.getName())) + log.warn("Taxonomy is duplicated and will be overridden: {}", taxonomy.getName()); + taxonomies.put(taxonomy.getName(), taxonomy); + } catch (Exception e) { + log.error("Taxonomy file could not be read: {}", yamlFile.getAbsolutePath(), e); + } + } + } + } + // get the ones from plugins + for (TaxonomiesProviderService provider : pluginsService.getTaxonomiesProviderServices().stream() + .filter(provider -> provider.isFor(TaxonomyTarget.VARIABLE)) + .collect(Collectors.toList())) { + log.info("Fetching taxonomies from plugin: {}", provider.getName()); + try { + for (Taxonomy taxonomy : provider.getTaxonomies()) { + log.debug("Taxonomy from plugin {}: {}", provider.getName(), taxonomy.getName()); + // override any duplicated taxonomy + if (taxonomies.containsKey(taxonomy.getName())) + log.warn("Taxonomy is duplicated and will be overridden: {}", taxonomy.getName()); + taxonomies.put(taxonomy.getName(), taxonomy); + } + } catch (Exception e) { + log.warn("Taxonomies retrieval from plugin {} failed", provider.getName(), e); + } + } + // apply mica attributes + taxonomies.replaceAll((k, v) -> applyAttributes(taxonomies.get(k))); + eventBus.post(new VariableTaxonomiesUpdatedEvent(taxonomies)); + return taxonomies; + } + + /** + * Decorate the variable taxonomies with some Mica specific attributes. + * + * @param taxonomy + * @return + */ + private Taxonomy applyAttributes(Taxonomy taxonomy) { + String defaultTermsSortOrder = environment.getProperty("opalTaxonomies.defaultTermsSortOrder"); + + taxonomy.getVocabularies().forEach(vocabulary -> { + String field = vocabulary.getAttributeValue("field"); + if (Strings.isNullOrEmpty(field)) { + vocabulary.addAttribute("field", + "attributes." + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + ".und"); + } + String alias = vocabulary.getAttributeValue("alias"); + if (Strings.isNullOrEmpty(alias)) { + vocabulary.addAttribute("alias", + "attributes-" + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + "-und"); + } + if (!Strings.isNullOrEmpty(defaultTermsSortOrder)) { + vocabulary.addAttribute("termsSortKey", defaultTermsSortOrder); + } + }); + return taxonomy; + } + +} diff --git a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java b/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java deleted file mode 100644 index 934a0c9683..0000000000 --- a/mica-core/src/main/java/org/obiba/mica/micaConfig/service/helper/OpalServiceHelper.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2018 OBiBa. All rights reserved. - * - * This program and the accompanying materials - * are made available under the terms of the GNU Public License v3.0. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.obiba.mica.micaConfig.service.helper; - -import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.stream.Collectors; - -import javax.inject.Inject; - -import org.obiba.mica.spi.search.support.AttributeKey; -import org.obiba.mica.micaConfig.event.OpalTaxonomiesUpdatedEvent; -import org.obiba.opal.core.domain.taxonomy.Taxonomy; -import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; -import org.obiba.opal.rest.client.magma.OpalJavaClient; -import org.obiba.opal.web.model.Opal; -import org.obiba.opal.web.model.Search; -import org.obiba.opal.web.taxonomy.Dtos; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.context.EnvironmentAware; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import com.google.common.base.Strings; -import com.google.common.eventbus.EventBus; - -@Component -public class OpalServiceHelper implements EnvironmentAware { - - private static final Logger log = LoggerFactory.getLogger(OpalServiceHelper.class); - - private Environment environment; - - @Inject - private EventBus eventBus; - - @Cacheable(value = "opal-taxonomies", key = "#opalJavaClient.newUri().build()") //opal root url as key - public Map getTaxonomies(OpalJavaClient opalJavaClient) { - log.info("Fetching opal taxonomies"); - URI uri = opalJavaClient.newUri().segment("system", "conf", "taxonomies").build(); - List taxonomies = opalJavaClient - .getResources(Opal.TaxonomyDto.class, uri, Opal.TaxonomyDto.newBuilder()); - - ConcurrentMap taxonomiesList = taxonomies.stream().map(taxonomyDto -> { - Taxonomy taxonomy = fromDto(taxonomyDto); - String defaultTermsSortOrder = environment.getProperty("opalTaxonomies.defaultTermsSortOrder"); - - if (!Strings.isNullOrEmpty(defaultTermsSortOrder)) { - taxonomy.getVocabularies().forEach(vocabulary -> vocabulary.addAttribute("termsSortKey", defaultTermsSortOrder)); - } - - return taxonomy; - }).collect(Collectors.toConcurrentMap(TaxonomyEntity::getName, taxonomy -> taxonomy)); - eventBus.post(new OpalTaxonomiesUpdatedEvent(taxonomiesList)); - return taxonomiesList; - } - - public Search.EntitiesResultDto getEntitiesCount(OpalJavaClient opalJavaClient, String query, String entityType) { - log.info("Fetching opal entities count"); - log.debug(" Entities query: {}", query); - URI uri = opalJavaClient.newUri().segment("datasources", "entities", "_count") - .query("query", query) - .query("type", Strings.isNullOrEmpty(entityType) ? "Participant" : entityType).build(); - Search.EntitiesResultDto result = opalJavaClient.getResource(Search.EntitiesResultDto.class, uri, Search.EntitiesResultDto.newBuilder()); - return result; - } - - /** - * Decorate the variable taxonomies with some Mica specific attributes. - * - * @param dto - * @return - */ - private Taxonomy fromDto(Opal.TaxonomyDto dto) { - Taxonomy taxonomy = Dtos.fromDto(dto); - taxonomy.getVocabularies().forEach(vocabulary -> { - String field = vocabulary.getAttributeValue("field"); - if(Strings.isNullOrEmpty(field)) { - vocabulary.addAttribute("field", - "attributes." + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + ".und"); - } - String alias = vocabulary.getAttributeValue("alias"); - if(Strings.isNullOrEmpty(alias)) { - vocabulary.addAttribute("alias", - "attributes-" + AttributeKey.getMapKey(vocabulary.getName(), taxonomy.getName()) + "-und"); - } - }); - return taxonomy; - } - - @Override - public void setEnvironment(Environment environment) { - this.environment = environment; - } -} diff --git a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java index b136120a2c..5b8946975f 100644 --- a/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java +++ b/mica-core/src/main/java/org/obiba/mica/study/domain/BaseStudy.java @@ -16,6 +16,7 @@ import org.obiba.mica.core.domain.*; import org.obiba.mica.file.Attachment; import org.obiba.mica.spi.search.Indexable; +import org.obiba.mica.spi.tables.IStudy; import javax.validation.constraints.NotNull; import java.beans.Transient; @@ -26,7 +27,7 @@ /** * Base class for representing all type of studies. */ -public abstract class BaseStudy extends AbstractModelAware implements PersonAware, Indexable { +public abstract class BaseStudy extends AbstractModelAware implements PersonAware, Indexable, IStudy { private Attachment logo; @@ -97,6 +98,7 @@ public void setObjectives(LocalizedString objectives) { this.objectives = objectives; } + @Override public String getOpal() { return opal; } diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java index 45565580fa..1ce4740c64 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/DatasetDtos.java @@ -10,44 +10,32 @@ package org.obiba.mica.web.model; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.validation.constraints.NotNull; - -import org.obiba.magma.type.BooleanType; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.obiba.mica.JSONUtils; import org.obiba.mica.core.domain.*; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.StudyDatasetStateRepository; -import org.obiba.mica.dataset.domain.Dataset; -import org.obiba.mica.dataset.domain.DatasetCategory; -import org.obiba.mica.dataset.domain.DatasetVariable; -import org.obiba.mica.dataset.domain.HarmonizationDataset; -import org.obiba.mica.dataset.domain.HarmonizationDatasetState; -import org.obiba.mica.dataset.domain.StudyDataset; -import org.obiba.mica.dataset.domain.StudyDatasetState; -import org.obiba.mica.micaConfig.domain.MicaConfig; +import org.obiba.mica.dataset.domain.*; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.study.service.PublishedStudyService; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Term; import org.obiba.opal.core.domain.taxonomy.Vocabulary; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.stereotype.Component; import org.springframework.util.Assert; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; @Component class DatasetDtos { @@ -194,11 +182,13 @@ Mica.DatasetVariableResolverDto.Builder asDto(@NotNull DatasetVariable.IdResolve if(resolver.hasStudyId()) { builder.setStudyId(resolver.getStudyId()); } - if(resolver.hasProject()) { - builder.setProject(resolver.getProject()); - } - if(resolver.hasTable()) { - builder.setTable(resolver.getTable()); + if(resolver.hasSource()) { + builder.setSource(resolver.getSource()); + if (OpalTableSource.isFor(resolver.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(resolver.getSource()); + builder.setProject(source.getProject()); + builder.setTable(source.getTable()); + } } return builder; @@ -314,8 +304,8 @@ Mica.DatasetVariableDto asDto(@NotNull DatasetVariable variable, @NotNull List builder.addAttributes(attributeDtos.asDto(attribute))); } - if(opalTable instanceof StudyTable) builder.setStudyTable(asDto((StudyTable) opalTable, includeSummaries)); - else if(opalTable instanceof HarmonizationStudyTable) { - builder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable, includeSummaries)); + if(studyTable instanceof StudyTable) builder.setStudyTable(asDto((StudyTable) studyTable, includeSummaries)); + else if(studyTable instanceof HarmonizationStudyTable) { + builder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable, includeSummaries)); } return builder.build(); @@ -411,13 +401,18 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable) { } public Mica.DatasetDto.StudyTableDto.Builder asDto(StudyTable studyTable, boolean includeSummary) { - Mica.DatasetDto.StudyTableDto.Builder sbuilder = Mica.DatasetDto.StudyTableDto.newBuilder() // - .setProject(studyTable.getProject())// - .setTable(studyTable.getTable()) // - .setWeight(studyTable.getWeight()) // - .setStudyId(studyTable.getStudyId()) // + Mica.DatasetDto.StudyTableDto.Builder sbuilder = Mica.DatasetDto.StudyTableDto.newBuilder() + .setSource(studyTable.getSource()) + .setWeight(studyTable.getWeight()) + .setStudyId(studyTable.getStudyId()) .setDceId(studyTable.getDataCollectionEventUId()); + if (OpalTableSource.isFor(studyTable.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(studyTable.getSource()); + sbuilder.setProject(source.getProject()); + sbuilder.setTable(source.getTable()); + } + String populationId = studyTable.getPopulationId(); if (!Strings.isNullOrEmpty(populationId)) { sbuilder.setPopulationId(populationId); @@ -444,11 +439,16 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTable harmonizationTable, boolean includeSummary) { Mica.DatasetDto.HarmonizationTableDto.Builder hBuilder = Mica.DatasetDto.HarmonizationTableDto.newBuilder() - .setProject(harmonizationTable.getProject()) - .setTable(harmonizationTable.getTable()) + .setSource(harmonizationTable.getSource()) .setWeight(harmonizationTable.getWeight()) .setStudyId(harmonizationTable.getStudyId()); + if (OpalTableSource.isFor(harmonizationTable.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationTable.getSource()); + hBuilder.setProject(source.getProject()); + hBuilder.setTable(source.getTable()); + } + if(includeSummary) hBuilder.setStudySummary(studySummaryDtos.asDto(harmonizationTable.getStudyId())); hBuilder.addAllName(localizedStringDtos.asDto(harmonizationTable.getName())); @@ -458,269 +458,39 @@ public Mica.DatasetDto.HarmonizationTableDto.Builder asDto(HarmonizationStudyTab return hBuilder; } - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull OpalTable opalTable, - @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - - aggDto = simpleAggregationDto(aggDto, summary); - - if(opalTable instanceof StudyTable) - aggDto.setStudyTable(asDto((StudyTable) opalTable, withStudySummary)); - else if (opalTable instanceof HarmonizationStudyTable) - aggDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable, withStudySummary)); - - return aggDto; - } - - public Mica.DatasetVariableAggregationDto.Builder simpleAggregationDto(@NotNull Mica.DatasetVariableAggregationDto.Builder aggDto, @Nullable Math.SummaryStatisticsDto summary) { - - if(summary == null) return aggDto.setTotal(0).setN(0); + public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, + @Nullable Mica.DatasetVariableAggregationDto summary, boolean withStudySummary) { + Mica.DatasetVariableAggregationDto.Builder aggDto = summary == null ? Mica.DatasetVariableAggregationDto.newBuilder() : summary.toBuilder(); - if(summary.hasExtension(Math.CategoricalSummaryDto.categorical)) { - aggDto = asDto(summary.getExtension(Math.CategoricalSummaryDto.categorical)); - } else if(summary.hasExtension(Math.ContinuousSummaryDto.continuous)) { - aggDto = asDto(summary.getExtension(Math.ContinuousSummaryDto.continuous)); - } else if(summary.hasExtension(Math.DefaultSummaryDto.defaultSummary)) { - aggDto = asDto(summary.getExtension(Math.DefaultSummaryDto.defaultSummary)); - } else if(summary.hasExtension(Math.TextSummaryDto.textSummary)) { - aggDto = asDto(summary.getExtension(Math.TextSummaryDto.textSummary)); - } else if(summary.hasExtension(Math.GeoSummaryDto.geoSummary)) { - aggDto = asDto(summary.getExtension(Math.GeoSummaryDto.geoSummary)); - } else if(summary.hasExtension(Math.BinarySummaryDto.binarySummary)) { - aggDto = asDto(summary.getExtension(Math.BinarySummaryDto.binarySummary)); + if (summary == null) { + aggDto.setTotal(0).setN(0); } + if(studyTable instanceof StudyTable) + aggDto.setStudyTable(asDto((StudyTable) studyTable, withStudySummary)); + else if (studyTable instanceof HarmonizationStudyTable) + aggDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable, withStudySummary)); + return aggDto; } - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull OpalTable opalTable, - DatasetVariable variable, DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { - Mica.DatasetVariableContingencyDto.Builder crossDto = Mica.DatasetVariableContingencyDto.newBuilder(); - - if(opalTable instanceof StudyTable) - crossDto.setStudyTable(asDto((StudyTable) opalTable, true)); - else if (opalTable instanceof HarmonizationStudyTable) - crossDto.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) opalTable)); + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(@NotNull BaseStudyTable studyTable, @Nullable Mica.DatasetVariableContingencyDto crossDto) { + Mica.DatasetVariableContingencyDto.Builder crossDtoBuilder = crossDto == null ? Mica.DatasetVariableContingencyDto.newBuilder() : crossDto.toBuilder(); - Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); + if(studyTable instanceof StudyTable) + crossDtoBuilder.setStudyTable(asDto((StudyTable) studyTable, true)); + else if (studyTable instanceof HarmonizationStudyTable) + crossDtoBuilder.setHarmonizationStudyTable(asDto((HarmonizationStudyTable) studyTable)); - if(results == null) { + if(crossDto == null) { + Mica.DatasetVariableAggregationDto.Builder allAggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); allAggBuilder.setN(0); allAggBuilder.setTotal(0); - crossDto.setAll(allAggBuilder); - return crossDto; - } - - allAggBuilder.setTotal(results.getTotalHits()); - MicaConfig micaConfig = micaConfigService.getConfig(); - int privacyThreshold = micaConfig.getPrivacyThreshold(); - crossDto.setPrivacyThreshold(privacyThreshold); - boolean privacyChecks = !crossVariable.hasCategories() || validatePrivacyThreshold(results, privacyThreshold); - boolean totalPrivacyChecks = validateTotalPrivacyThreshold(results, privacyThreshold); - - // add facet results in the same order as the variable categories - List catNames = variable.getValueType().equals(BooleanType.get().getName()) ? - Lists.newArrayList("true", "false") : variable.getCategories().stream().map(DatasetCategory::getName).collect(Collectors.toList()); - catNames.forEach(catName -> results.getFacetsList().stream() - .filter(facet -> facet.hasFacet() && catName.equals(facet.getFacet())).forEach(facet -> { - boolean privacyCheck = privacyChecks && checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold); - Mica.DatasetVariableAggregationDto.Builder aggBuilder = Mica.DatasetVariableAggregationDto.newBuilder(); - aggBuilder.setTotal(totalPrivacyChecks ? results.getTotalHits() : 0); - aggBuilder.setTerm(facet.getFacet()); - DatasetCategory category = variable.getCategory(facet.getFacet()); - aggBuilder.setMissing(category != null && category.isMissing()); - addSummaryStatistics(crossVariable, aggBuilder, facet, privacyCheck, totalPrivacyChecks); - crossDto.addAggregations(aggBuilder); - })); - - // add total facet for all variable categories - results.getFacetsList().stream().filter(facet -> facet.hasFacet() && "_total".equals(facet.getFacet())) - .forEach(facet -> { - boolean privacyCheck = privacyChecks && facet.getFilters(0).getCount() >= micaConfig.getPrivacyThreshold(); - addSummaryStatistics(crossVariable, allAggBuilder, facet, privacyCheck, totalPrivacyChecks); - }); - - crossDto.setAll(allAggBuilder); - - return crossDto; - } - - private boolean checkPrivacyThreshold(int count, int threshold) { - return count == 0 || count >= threshold; - } - - private boolean validateTotalPrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { - return results.getFacetsList().stream() - .allMatch(facet -> checkPrivacyThreshold(facet.getFilters(0).getCount(), privacyThreshold)); - } - - private boolean validatePrivacyThreshold(Search.QueryResultDtoOrBuilder results, int privacyThreshold) { - return results.getFacetsList().stream().map(Search.FacetResultDto::getFrequenciesList).flatMap(Collection::stream) - .allMatch(freq -> checkPrivacyThreshold(freq.getCount(), privacyThreshold)); - } - - private void addSummaryStatistics(DatasetVariable crossVariable, - Mica.DatasetVariableAggregationDto.Builder aggBuilder, Search.FacetResultDto facet, boolean privacyCheck, - boolean totalPrivacyCheck) { - aggBuilder.setN(totalPrivacyCheck ? facet.getFilters(0).getCount() : -1); - if(!privacyCheck) return; - - List catNames = crossVariable.getValueType().equals(BooleanType.get().getName()) ? - Lists.newArrayList("1", "0") : - (crossVariable.hasCategories() ? crossVariable.getCategories().stream().map(DatasetCategory::getName).collect(Collectors.toList()) : Lists.newArrayList()); - // order results as the order of cross variable categories - catNames.forEach(catName -> facet.getFrequenciesList().stream().filter(freq -> catName.equals(freq.getTerm())) - .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq)))); - // observed terms, not described by categories - facet.getFrequenciesList().stream().filter(freq -> !catNames.contains(freq.getTerm())) - .forEach(freq -> aggBuilder.addFrequencies(asDto(crossVariable, freq))); - - if(facet.hasStatistics()) { - aggBuilder.setStatistics(asDto(facet.getStatistics())); - } - } - - private Mica.FrequencyDto.Builder asDto(DatasetVariable crossVariable, - Search.FacetResultDto.TermFrequencyResultDto result) { - if (crossVariable.getValueType().equals(BooleanType.get().getName())) { - // for some reason 0/1 is returned instead of false/true - return Mica.FrequencyDto.newBuilder() - .setValue("1".equals(result.getTerm()) ? "true" : "false") - .setCount(result.getCount()) - .setMissing(false); - } else if (crossVariable.getCategory(result.getTerm()) != null) { - DatasetCategory category = crossVariable.getCategory(result.getTerm()); - return Mica.FrequencyDto.newBuilder() - .setValue(result.getTerm()) - .setCount(result.getCount()) - .setMissing(category != null && category.isMissing()); - } else { - // observed value, not described by a category - return Mica.FrequencyDto.newBuilder() - .setValue(result.getTerm()) - .setCount(result.getCount()) - .setMissing(false); - } - } - - private Mica.StatisticsDto.Builder asDto(Search.FacetResultDto.StatisticalResultDto result) { - return Mica.StatisticsDto.newBuilder() // - .setMin(result.getMin()) // - .setMax(result.getMax()) // - .setMean(result.getMean()) // - .setSum(result.getTotal()) // - .setSumOfSquares(result.getSumOfSquares()) // - .setVariance(result.getVariance()) // - .setStdDeviation(result.getStdDeviation()); - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.CategoricalSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList(), - summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.DefaultSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.TextSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList(), - summary.hasOtherFrequency() ? Long.valueOf(summary.getOtherFrequency()).intValue() : 0); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.GeoSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.BinarySummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - aggDto.setTotal(Long.valueOf(summary.getN()).intValue()); - addFrequenciesDto(aggDto, summary.getFrequenciesList()); - return aggDto; - } - - private Mica.FrequencyDto.Builder asDto(Math.FrequencyDto freq) { - return Mica.FrequencyDto.newBuilder().setValue(freq.getValue()).setCount(Long.valueOf(freq.getFreq()).intValue()) - .setMissing(freq.getMissing()); - } - - private Mica.IntervalFrequencyDto.Builder asDto(Math.IntervalFrequencyDto inter) { - return Mica.IntervalFrequencyDto.newBuilder().setCount((int)inter.getFreq()) - .setLower(inter.getLower()).setUpper(inter.getUpper()); - } - - private void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, - List frequencies) { - addFrequenciesDto(aggDto, frequencies, 0); - } - - private void addFrequenciesDto(Mica.DatasetVariableAggregationDto.Builder aggDto, List frequencies, - int otherFrequency) { - int n = otherFrequency; - if(frequencies != null) { - for(Math.FrequencyDto freq : frequencies) { - aggDto.addFrequencies(asDto(freq)); - if(!freq.getMissing()) n += freq.getFreq(); - } + crossDtoBuilder.setAll(allAggBuilder); + return crossDtoBuilder; } - if (otherFrequency>0) - aggDto.addFrequencies(Mica.FrequencyDto.newBuilder().setValue("???").setCount(otherFrequency) - .setMissing(false)); - aggDto.setN(n); - } - - private Mica.DatasetVariableAggregationDto.Builder asDto(Math.ContinuousSummaryDto summary) { - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); - Math.DescriptiveStatsDto stats = summary.getSummary(); - - aggDto.setN(Long.valueOf(stats.getN()).intValue()); - - Mica.StatisticsDto.Builder builder = Mica.StatisticsDto.newBuilder(); - - if(stats.hasSum()) builder.setSum(Double.valueOf(stats.getSum()).floatValue()); - if(stats.hasMin() && stats.getMin() != Double.POSITIVE_INFINITY) - builder.setMin(Double.valueOf(stats.getMin()).floatValue()); - if(stats.hasMax() && stats.getMax() != Double.NEGATIVE_INFINITY) - builder.setMax(Double.valueOf(stats.getMax()).floatValue()); - if(stats.hasMean() && !Double.isNaN(stats.getMean())) builder.setMean(Double.valueOf(stats.getMean()).floatValue()); - if(stats.hasSumsq() && !Double.isNaN(stats.getSumsq())) - builder.setSumOfSquares(Double.valueOf(stats.getSumsq()).floatValue()); - if(stats.hasVariance() && !Double.isNaN(stats.getVariance())) - builder.setVariance(Double.valueOf(stats.getVariance()).floatValue()); - if(stats.hasStdDev() && !Double.isNaN(stats.getStdDev())) - builder.setStdDeviation(Double.valueOf(stats.getStdDev()).floatValue()); - - aggDto.setStatistics(builder); - if(summary.getFrequenciesCount() > 0) { - summary.getFrequenciesList().forEach(freq -> aggDto.addFrequencies(asDto(freq))); - } - - if (summary.getIntervalFrequencyCount() > 0) { - summary.getIntervalFrequencyList().forEach(inter -> aggDto.addIntervalFrequencies(asDto(inter))); - } - - int total = 0; - if(summary.getFrequenciesCount() > 0) { - for(Math.FrequencyDto freq : summary.getFrequenciesList()) { - total += freq.getFreq(); - } - } - aggDto.setTotal(total); - - return aggDto; + return crossDtoBuilder; } private Mica.DatasetDto.Builder asBuilder(Dataset dataset) { @@ -783,8 +553,12 @@ private Dataset fromDto(@NotNull Mica.HarmonizedDatasetDto dto) { if(dto.hasHarmonizationTable()) { HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); - harmonizationLink.setProject(dto.getHarmonizationTable().getProject()); - harmonizationLink.setTable(dto.getHarmonizationTable().getTable()); + // legacy + if (dto.getHarmonizationTable().hasProject() && dto.getHarmonizationTable().hasTable()) { + harmonizationLink.setSource(makesource(dto.getHarmonizationTable().getProject(), dto.getHarmonizationTable().getTable())); + } else { + harmonizationLink.setSource(dto.getHarmonizationTable().getSource()); + } harmonizationLink.setStudyId(dto.getHarmonizationTable().getStudyId()); harmonizationDataset.setHarmonizationTable(harmonizationLink); } @@ -807,41 +581,56 @@ private StudyTable fromDto(Mica.DatasetDto.StudyTableDto dto) { if (dto.hasDataCollectionEventId()) { table.setDataCollectionEventId(dto.getDataCollectionEventId()); } - table.setProject(dto.getProject()); - table.setTable(dto.getTable()); table.setWeight(dto.getWeight()); - table.setName(localizedStringDtos.fromDto(dto.getNameList())); table.setDescription(localizedStringDtos.fromDto(dto.getDescriptionList())); table.setAdditionalInformation(localizedStringDtos.fromDto(dto.getAdditionalInformationList())); + if (dto.hasProject() && dto.hasTable()) { + table.setSource(makesource(dto.getProject(), dto.getTable())); + } else { + table.setSource(dto.getSource()); + } + return table; } private HarmonizationStudyTable fromDto(Mica.DatasetDto.HarmonizationTableDto dto) { HarmonizationStudyTable table = new HarmonizationStudyTable(); table.setStudyId(dto.getStudyId()); - table.setProject(dto.getProject()); - table.setTable(dto.getTable()); table.setWeight(dto.getWeight()); - table.setName(localizedStringDtos.fromDto(dto.getNameList())); table.setDescription(localizedStringDtos.fromDto(dto.getDescriptionList())); table.setAdditionalInformation(localizedStringDtos.fromDto(dto.getAdditionalInformationList())); + // legacy + if (dto.hasProject() && dto.hasTable()) { + table.setSource(makesource(dto.getProject(), dto.getTable())); + } else { + table.setSource(dto.getSource()); + } + return table; } + // legacy + private String makesource(String project, String table) { + return OpalTableSource.newSource(project, table).getURN(); + } + private Mica.DatasetDto.HarmonizationTableDto.Builder createHarmonizationLinkDtoFromHarmonizationTable( HarmonizationStudyTable harmonizationLink, boolean asDraft) { Mica.DatasetDto.HarmonizationTableDto.Builder harmonizationLinkBuilder = Mica.DatasetDto.HarmonizationTableDto .newBuilder(); - if(!Strings.isNullOrEmpty(harmonizationLink.getProject())) - harmonizationLinkBuilder.setProject(harmonizationLink.getProject()); - - if(!Strings.isNullOrEmpty(harmonizationLink.getTable())) - harmonizationLinkBuilder.setTable(harmonizationLink.getTable()); + if(!Strings.isNullOrEmpty(harmonizationLink.getSource())) { + harmonizationLinkBuilder.setSource(harmonizationLink.getSource()); + if (OpalTableSource.isFor(harmonizationLink.getSource())) { + OpalTableSource source = OpalTableSource.fromURN(harmonizationLink.getSource()); + harmonizationLinkBuilder.setProject(source.getProject()); + harmonizationLinkBuilder.setTable(source.getTable()); + } + } String studyId = harmonizationLink.getStudyId(); diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java index efecd0583b..0d277e8440 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/Dtos.java @@ -397,8 +397,8 @@ public Mica.DatasetHarmonizedVariableSummaryDto asHarmonizedSummaryDto(@NotNull } @NotNull - public Mica.DatasetVariableSummaryDto asSummaryDto(@NotNull DatasetVariable variable, OpalTable opalTable, boolean includeSummaries) { - return datasetDtos.asSummaryDto(variable, opalTable, includeSummaries); + public Mica.DatasetVariableSummaryDto asSummaryDto(@NotNull DatasetVariable variable, BaseStudyTable studyTable, boolean includeSummaries) { + return datasetDtos.asSummaryDto(variable, studyTable, includeSummaries); } @NotNull @@ -427,20 +427,14 @@ public Mica.DatasetDto.StudyTableDto.Builder asDto(@NotNull StudyTable studyTabl } @NotNull - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull OpalTable opalTable, - @Nullable Math.SummaryStatisticsDto summary, boolean withStudySummary) { - return datasetDtos.asDto(opalTable, summary, withStudySummary); + public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull BaseStudyTable studyTable, + @Nullable Mica.DatasetVariableAggregationDto summary, boolean withStudySummary) { + return datasetDtos.asDto(studyTable, summary, withStudySummary); } @NotNull - public Mica.DatasetVariableAggregationDto.Builder asDto(@NotNull Math.SummaryStatisticsDto summary) { - return datasetDtos.simpleAggregationDto(Mica.DatasetVariableAggregationDto.newBuilder(), summary); - } - - @NotNull - public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(OpalTable opalTable, DatasetVariable variable, - DatasetVariable crossVariable, @Nullable Search.QueryResultDto results) { - return datasetDtos.asContingencyDto(opalTable, variable, crossVariable, results); + public Mica.DatasetVariableContingencyDto.Builder asContingencyDto(BaseStudyTable studyTable, @Nullable Mica.DatasetVariableContingencyDto results) { + return datasetDtos.asContingencyDto(studyTable, results); } @NotNull diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java index 43611063c6..af8939af94 100644 --- a/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java +++ b/mica-core/src/main/java/org/obiba/mica/web/model/MicaConfigDtos.java @@ -103,7 +103,8 @@ Mica.MicaConfigDto asDto(@NotNull MicaConfig config, String language) { } builder.setIsUsePublicUrlForSharedLink(config.isUsePublicUrlForSharedLink()); - builder.setOpal(config.getOpal()); + if (config.hasOpal()) + builder.setOpal(config.getOpal()); builder.setPrivacyThreshold(config.getPrivacyThreshold()); if(config.getMicaVersion() != null) { diff --git a/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java new file mode 100644 index 0000000000..3b50fe6430 --- /dev/null +++ b/mica-core/src/main/java/org/obiba/mica/web/model/PluginDtos.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.web.model; + +import com.google.common.base.Strings; +import org.obiba.magma.type.DateTimeType; +import org.obiba.plugins.PluginPackage; +import org.obiba.plugins.PluginResources; + +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +public class PluginDtos { + + public static MicaPlugins.PluginPackagesDto asDto(String site, Date updated, boolean restart, List packages) { + return asDto(site, updated, restart, packages, null); + } + + public static MicaPlugins.PluginPackagesDto asDto(String site, Date updated, boolean restart, List packages, Collection uninstalledNames) { + MicaPlugins.PluginPackagesDto.Builder builder = asDto(site, updated, restart); + builder.addAllPackages(packages.stream().map(p -> asDto(p, uninstalledNames == null ? null : uninstalledNames.contains(p.getName()))) + .collect(Collectors.toList())); + return builder.build(); + } + + public static MicaPlugins.PluginPackagesDto.Builder asDto(String site, Date updated, boolean restart) { + MicaPlugins.PluginPackagesDto.Builder builder = MicaPlugins.PluginPackagesDto.newBuilder() + .setSite(site) + .setRestart(restart); + if (updated != null) builder.setUpdated(DateTimeType.get().valueOf(updated).toString()); + return builder; + } + + public static MicaPlugins.PluginPackageDto asDto(PluginPackage pluginPackage, Boolean uninstalled) { + MicaPlugins.PluginPackageDto.Builder buider = MicaPlugins.PluginPackageDto.newBuilder() + .setName(pluginPackage.getName()) + .setType(pluginPackage.getType()) + .setTitle(pluginPackage.getTitle()) + .setDescription(pluginPackage.getDescription()) + .setAuthor(Strings.isNullOrEmpty(pluginPackage.getAuthor()) ? "-" : pluginPackage.getAuthor()) + .setMaintainer(Strings.isNullOrEmpty(pluginPackage.getMaintainer()) ? "-" : pluginPackage.getMaintainer()) + .setLicense(Strings.isNullOrEmpty(pluginPackage.getLicense()) ? "-" : pluginPackage.getLicense()) + .setVersion(pluginPackage.getVersion().toString()) + .setMicaVersion(pluginPackage.getMicaVersion().toString()); + if (!Strings.isNullOrEmpty(pluginPackage.getWebsite())) + buider.setWebsite(pluginPackage.getWebsite()); + if (!Strings.isNullOrEmpty(pluginPackage.getFileName())) + buider.setFile(pluginPackage.getFileName()); + if (uninstalled != null) buider.setUninstalled(uninstalled); + return buider.build(); + } + + public static MicaPlugins.PluginDto asDto(PluginResources plugin) { + MicaPlugins.PluginDto.Builder builder = MicaPlugins.PluginDto.newBuilder() + .setName(plugin.getName()) + .setTitle(plugin.getTitle()) + .setDescription(plugin.getDescription()) + .setAuthor(plugin.getAuthor()) + .setMaintainer(plugin.getMaintainer()) + .setLicense(plugin.getLicense()) + .setVersion(plugin.getVersion().toString()) + .setMicaVersion(plugin.getHostVersion().toString()) + .setType(plugin.getType()) + .setSiteProperties(plugin.getSitePropertiesString()); + return builder.build(); + } + +} diff --git a/mica-core/src/main/resources/ehcache.xml b/mica-core/src/main/resources/ehcache.xml index 119df23ac4..bcaf3d4549 100644 --- a/mica-core/src/main/resources/ehcache.xml +++ b/mica-core/src/main/resources/ehcache.xml @@ -64,11 +64,11 @@ timeToLiveSeconds="0" timeToIdleSeconds="0"/> - + diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java new file mode 100644 index 0000000000..f34d50ac7c --- /dev/null +++ b/mica-core/src/test/java/org/obiba/mica/core/source/ExcelTableSourceTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExcelTableSourceTest { + + @Test + public void test_is_for() { + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xlsx:MyTable")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xlsx")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:/path/to/MyProject.xlsx:MyTable")).isTrue(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.xls")).isFalse(); + assertThat(ExcelTableSource.isFor("urn:file:MyProject.spss")).isFalse(); + } + + @Test + public void test_urn_parse() { + ExcelTableSource source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx:MyTable"); + assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); + + source = ExcelTableSource.fromURN("urn:file:MyProject.xlsx"); + assertThat(source.getPath()).isEqualTo("MyProject.xlsx"); + + source = ExcelTableSource.fromURN("urn:file:/path/to/MyProject.xlsx:MyTable"); + assertThat(source.getPath()).isEqualTo("/path/to/MyProject.xlsx"); + } +} diff --git a/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java b/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java new file mode 100644 index 0000000000..b64cf1999b --- /dev/null +++ b/mica-core/src/test/java/org/obiba/mica/core/source/OpalTableSourceTest.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.core.source; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class OpalTableSourceTest { + + @Test + public void test_urn_parse() { + OpalTableSource source = OpalTableSource.fromURN("urn:opal:MyProject.MyTable"); + assertThat(source.getProject()).isEqualTo("MyProject"); + assertThat(source.getTable()).isEqualTo("MyTable"); + } +} diff --git a/mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java b/mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java similarity index 80% rename from mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java rename to mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java index 7befaf8137..d76c96076b 100644 --- a/mica-core/src/test/java/org/obiba/mica/core/support/YamlClassPathResourceReaderTest.java +++ b/mica-core/src/test/java/org/obiba/mica/core/support/YamlResourceReaderTest.java @@ -11,11 +11,11 @@ import static org.junit.Assert.assertTrue; @RunWith(JUnit4.class) -public class YamlClassPathResourceReaderTest { +public class YamlResourceReaderTest { @Test public void readEmptyExclusionsListYamlTest() { - Map read = YamlClassPathResourceReader.read("/empty-exclusions-list.yml", Map.class); + Map read = YamlResourceReader.readClassPath("/empty-exclusions-list.yml", Map.class); assertTrue(read.containsKey("exclusions")); assertTrue(((List) read.get("exclusions")).isEmpty()); @@ -23,7 +23,7 @@ public void readEmptyExclusionsListYamlTest() { @Test public void readExclusionsListYamlTest() { - Map read = YamlClassPathResourceReader.read("/exclusions-list.yml", Map.class); + Map read = YamlResourceReader.readClassPath("/exclusions-list.yml", Map.class); assertTrue(read.containsKey("exclusions")); assertNotNull(read.get("exclusions")); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java index f3247ef05a..c359a8c396 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/CollectedDatasetServiceTest.java @@ -19,6 +19,7 @@ import org.mockito.MockitoAnnotations; import org.obiba.magma.MagmaRuntimeException; import org.obiba.mica.core.domain.LocalizedString; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.core.service.GitService; import org.obiba.mica.dataset.StudyDatasetRepository; @@ -98,8 +99,7 @@ public void testDatasourceConnectionErrorIsIgnoredForDraft() { private StudyDataset buildStudyDataset() { StudyDataset ds = new StudyDataset(); StudyTable st = new StudyTable(); - st.setProject("proj"); - st.setTable("tab"); + st.setSource(OpalTableSource.newSource("proj", "tab").getURN()); st.setPopulationId("1"); st.setDataCollectionEventId("1"); ds.setStudyTable(st); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java index a7ef5e54cf..4e70e0fa99 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/service/HarmonizedDatasetServiceTest.java @@ -13,7 +13,6 @@ import java.util.Locale; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.Supplier; import org.junit.Before; @@ -24,6 +23,7 @@ import org.mockito.Spy; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.magma.Variable; import org.obiba.magma.type.BooleanType; @@ -98,8 +98,7 @@ private HarmonizationDataset buildHarmonizationDataset(String id, StudyTable... private StudyTable buildStudyTable(String project, String table, String studyId) { StudyTable st = new StudyTable(); - st.setProject(project); - st.setTable(table); + st.setSource(OpalTableSource.newSource(project, table).getURN()); st.setStudyId(studyId); st.setPopulationId("pop"); st.setDataCollectionEventId("ev"); diff --git a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java index 619a01fbe9..ccb26edad6 100644 --- a/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java +++ b/mica-core/src/test/java/org/obiba/mica/dataset/support/HarmonizedDatasetHelperTest.java @@ -19,6 +19,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.HarmonizationStudyTable; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import static org.obiba.mica.assertj.Assertions.assertThat; @@ -107,8 +108,9 @@ private StudyTable createStudyTable(String studyId) { private void initTable(String studyId, BaseStudyTable table) { table.setStudyId(studyId); - table.setProject(RandomStringUtils.random(10, true, false)); - table.setTable(RandomStringUtils.random(10, true, false)); + String project = RandomStringUtils.random(10, true, false); + String tbl = RandomStringUtils.random(10, true, false); + table.setSource(OpalTableSource.newSource(project, tbl).getURN()); table.setPopulationId(RandomStringUtils.random(10, true, true)); } } diff --git a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java index 3e43c7e942..7a4010b262 100644 --- a/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java +++ b/mica-core/src/test/java/org/obiba/mica/web/model/DatasetDtosTest.java @@ -18,6 +18,7 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import org.obiba.mica.core.domain.HarmonizationStudyTable; +import org.obiba.mica.core.source.OpalTableSource; import org.obiba.mica.dataset.HarmonizationDatasetStateRepository; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.domain.HarmonizationDatasetState; @@ -108,8 +109,7 @@ private HarmonizationDataset createHarmonizedDataset() { HarmonizationDataset harmonizationDataset = new HarmonizationDataset(); harmonizationDataset.setId("123"); HarmonizationStudyTable harmonizationLink = new HarmonizationStudyTable(); - harmonizationLink.setProject("project123"); - harmonizationLink.setTable("table123"); + harmonizationLink.setSource(OpalTableSource.newSource("project123", "table123").getURN()); harmonizationLink.setStudyId("study123"); harmonizationLink.setPopulationId("population123"); harmonizationDataset.setHarmonizationTable(harmonizationLink); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java index fea8745cd4..7e7eec0e01 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/DatasourceNotAvailableExceptionMapper.java @@ -21,6 +21,6 @@ public class DatasourceNotAvailableExceptionMapper implements ExceptionMapper { @Override public Response toResponse(DatasourceNotAvailableException e) { - return Response.status(Status.SERVICE_UNAVAILABLE).entity("Verify the datasource is available.").build(); + return Response.status(Status.SERVICE_UNAVAILABLE).entity("Verify the datasource is available. " + e.getMessage()).build(); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java similarity index 61% rename from mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java rename to mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java index 2f512f3513..203e43a583 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/InvalidDatasetExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchStudyTableSourceExceptionMapper.java @@ -10,17 +10,19 @@ package org.obiba.mica.dataset.rest; +import org.obiba.mica.spi.tables.NoSuchStudyTableSourceException; + import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; -import org.obiba.mica.dataset.service.InvalidDatasetException; - @Provider -public class InvalidDatasetExceptionMapper implements ExceptionMapper { +public class NoSuchStudyTableSourceExceptionMapper implements ExceptionMapper { + @Override - public Response toResponse(InvalidDatasetException e) { - return Response.status(Status.CONFLICT).entity(e.getMessage()).build(); + public Response toResponse(NoSuchStudyTableSourceException exception) { + return Response.status(Status.NOT_FOUND).entity(exception.getMessage()).build(); } + } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java index 00a02cd95d..9d4ad180a0 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/NoSuchValueTableExceptionMapper.java @@ -22,7 +22,7 @@ public class NoSuchValueTableExceptionMapper implements ExceptionMapper getVariables() { @@ -157,13 +147,6 @@ public DraftCollectedDatasetVariableResource getVariable(@PathParam("variable") return resource; } - @POST - @Path("/facets") - public Search.QueryResultDto getFacets(Search.QueryTermsDto query) { - checkPermission("/draft/collected-dataset", "VIEW"); - return datasetService.getFacets(getDataset(), query); - } - @PUT @Path("/_status") public Response updateStatus(@QueryParam("value") String status) { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java index 48085456d0..9e9056edca 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/collection/DraftCollectedDatasetVariableResource.java @@ -10,20 +10,17 @@ package org.obiba.mica.dataset.rest.collection; -import javax.inject.Inject; -import javax.ws.rs.GET; -import javax.ws.rs.Path; - import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import javax.inject.Inject; +import javax.ws.rs.GET; + @Component @Scope("request") public class DraftCollectedDatasetVariableResource implements DatasetVariableResource { @@ -43,18 +40,6 @@ public Mica.DatasetVariableDto getVariable() { return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName)); } - @GET - @Path("/summary") - public Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName).getWrappedDto(); - } - - @GET - @Path("/facet") - public Search.QueryResultDto getVariableFacet() { - return datasetService.getVariableFacet(getDataset(), variableName); - } - private StudyDataset getDataset() { return datasetService.findById(datasetId); } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java index 13520f0078..e51c3c4b9a 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLCriteriaOpalConverter.java @@ -30,7 +30,6 @@ import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.study.domain.BaseStudy; -import org.obiba.mica.study.domain.HarmonizationStudy; import org.obiba.mica.study.service.StudyService; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -182,8 +181,7 @@ private RQLFieldReferences parseField(String path) { } else if (DatasetVariable.Type.Harmonized.equals(resolver.getType())) { HarmonizationDataset ds = harmonizedDatasetService.findById(resolver.getDatasetId()); Optional studyTable = ds.getBaseStudyTables().stream().filter(st -> st.getStudyId().equals(resolver.getStudyId()) - && st.getProject().equals(resolver.getProject()) - && st.getTable().equals(resolver.getTable())).findFirst(); + && st.getSource().equals(resolver.getSource())).findFirst(); if (!studyTable.isPresent()) throw new IllegalArgumentException("Not a valid variable: " + path); BaseStudy study = studyService.findStudy(studyTable.get().getStudyId()); return new RQLFieldReferences(path, ds, studyTable.get(), study, getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, path)); diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java index b3a7cc7a11..bda2f47579 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/entity/rql/RQLFieldReferences.java @@ -89,6 +89,6 @@ public LocalizedString getStudyTableName() { * @return */ private String getOpalTablePath(BaseStudyTable studyTable) { - return studyTable.getProject() + "." + studyTable.getTable(); + return studyTable.getSource().replace("urn:opal:", ""); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java index 59f08ddd36..c8165edacf 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftDataschemaDatasetVariableResource.java @@ -24,7 +24,6 @@ import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -49,40 +48,6 @@ public Mica.DatasetVariableDto getVariable() { return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName)); } - @GET - @Path("/summary") - public List getVariableSummaries() { - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) - .getWrappedDto()); - } catch(NoSuchVariableException | NoSuchValueTableException e) { - // ignore (case the study has not implemented this dataschema variable) - } - }); - return builder.build(); - } - - @GET - @Path("/facet") - public List getVariableFacets() { - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService.getVariableFacet(dataset, variableName, studyId, table.getProject(), table.getTable())); - } catch(NoSuchVariableException | NoSuchValueTableException e) { - // ignore (case the study has not implemented this dataschema variable) - } - }); - return builder.build(); - } - @GET @Path("/harmonizations") public List getHarmonizedVariables() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java index 9cf772160c..55fb77cbf0 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetResource.java @@ -13,10 +13,8 @@ import com.codahale.metrics.annotation.Timed; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; - import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; -import org.obiba.mica.core.domain.BaseStudyTable; import org.obiba.mica.core.domain.PublishCascadingScope; import org.obiba.mica.core.domain.RevisionStatus; import org.obiba.mica.core.service.AbstractGitPersistableService; @@ -27,8 +25,6 @@ import org.obiba.mica.security.rest.SubjectAclResource; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Magma; -import org.obiba.opal.web.model.Search; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -137,13 +133,6 @@ public Response unPublish() { return Response.noContent().build(); } - @GET - @Path("/table") - public Magma.TableDto getTable() { - checkPermission("/draft/harmonized-dataset", "VIEW"); - return datasetService.getTableDto(getDataset()); - } - @GET @Path("/variables") public List getVariables() { @@ -172,18 +161,6 @@ public DraftHarmonizedDatasetVariableResource getVariable(@PathParam("study") St return resource; } - @POST - @Path("/facets") - public List getFacets(Search.QueryTermsDto query) { - checkPermission("/draft/harmonized-dataset", "VIEW"); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(); - for (BaseStudyTable table : dataset.getBaseStudyTables()) { - builder.add(datasetService.getFacets(query, table)); - } - return builder.build(); - } - @PUT @Path("/_status") @Timed diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java index 6e50046272..e192f32460 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/harmonization/DraftHarmonizedDatasetVariableResource.java @@ -10,19 +10,17 @@ package org.obiba.mica.dataset.rest.harmonization; -import javax.inject.Inject; -import javax.ws.rs.GET; -import javax.ws.rs.Path; - import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import javax.inject.Inject; +import javax.ws.rs.GET; + @Component @Scope("request") public class DraftHarmonizedDatasetVariableResource implements DatasetVariableResource { @@ -39,37 +37,19 @@ public class DraftHarmonizedDatasetVariableResource implements DatasetVariableRe private String studyId; - private String project; - - private String table; + private String source; @GET public Mica.DatasetVariableDto getVariable() { - return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, project, table)); - } - - @GET - @Path("/summary") - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - return datasetService.getVariableSummary(getDataset(), variableName, studyId, project, table).getWrappedDto(); - } - - @GET - @Path("/facet") - public Search.QueryResultDto getVariableFacet() { - return datasetService.getVariableFacet(getDataset(), variableName, studyId, project, table); + return dtos.asDto(datasetService.getDatasetVariable(getDataset(), variableName, studyId, source)); } public void setStudyId(String studyId) { this.studyId = studyId; } - public void setProject(String project) { - this.project = project; - } - - public void setTable(String table) { - this.table = table; + public void setSource(String source) { + this.source = source; } private HarmonizationDataset getDataset() { diff --git a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java index fdd222a13b..40fc090bac 100644 --- a/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/dataset/rest/variable/DraftDatasetVariableResource.java @@ -52,8 +52,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id) { subjectAclService.isPermitted("/draft/harmonized-dataset", "VIEW", resolver.getDatasetId()); resource = applicationContext.getBean(DraftHarmonizedDatasetVariableResource.class); ((DraftHarmonizedDatasetVariableResource) resource).setStudyId(resolver.getStudyId()); - ((DraftHarmonizedDatasetVariableResource) resource).setProject(resolver.getProject()); - ((DraftHarmonizedDatasetVariableResource) resource).setTable(resolver.getTable()); + ((DraftHarmonizedDatasetVariableResource) resource).setSource(resolver.getSource()); break; } diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java index 16bacf229e..89998266d6 100644 --- a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/MicaConfigResource.java @@ -72,7 +72,7 @@ public class MicaConfigResource { private MicaConfigService micaConfigService; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private OpalService opalService; @@ -86,9 +86,6 @@ public class MicaConfigResource { @Inject private Dtos dtos; - @Inject - private CacheService cacheService; - @Inject private EventBus eventBus; @@ -119,7 +116,7 @@ public Mica.PublicMicaConfigDto getPublic() { @RequiresRoles(Roles.MICA_ADMIN) public Response create(@SuppressWarnings("TypeMayBeWeakened") Mica.MicaConfigDto dto) { MicaConfig micaConfig = dtos.fromDto(dto); - taxonomyService.refreshTaxonomyTaxonomyIfNeeded(micaConfigService.getConfig(), micaConfig); + taxonomiesService.refreshTaxonomyTaxonomyIfNeeded(micaConfigService.getConfig(), micaConfig); micaConfigService.save(micaConfig); return Response.noContent().build(); } @@ -669,42 +666,6 @@ public Map getAvailableLanguages(@QueryParam("locale") @DefaultV .collect(Collectors.toMap(lang -> lang, lang -> new Locale(lang).getDisplayLanguage(locale))); } - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies") - @RequiresAuthentication - @Deprecated - public List getTaxonomies() { - return opalService.getTaxonomyDtos(); - } - - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies/summaries") - @RequiresAuthentication - @Deprecated - public Opal.TaxonomiesDto getTaxonomySummaries() { - return opalService.getTaxonomySummaryDtos(); - } - - /** - * @deprecated kept for backward compatibility. - * @return - */ - @GET - @Path("/taxonomies/vocabularies/summaries") - @RequiresAuthentication - @Deprecated - public Opal.TaxonomiesDto getTaxonomyVocabularySummaries() { - return opalService.getTaxonomyVocabularySummaryDtos(); - } - /** * @deprecated kept for backward compatibility. * @return @@ -714,6 +675,6 @@ public Opal.TaxonomiesDto getTaxonomyVocabularySummaries() { @RequiresAuthentication @Deprecated public Opal.TaxonomyDto getStudyTaxonomy() { - return org.obiba.opal.web.taxonomy.Dtos.asDto(taxonomyService.getStudyTaxonomy()); + return org.obiba.opal.web.taxonomy.Dtos.asDto(taxonomiesService.getStudyTaxonomy()); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java new file mode 100644 index 0000000000..eef19a8fac --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginResource.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.rest; + + +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresRoles; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.security.Roles; +import org.obiba.mica.web.model.MicaPlugins; +import org.obiba.mica.web.model.PluginDtos; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +@Component +@Scope("request") +@Path("/config/plugin/{name}") +@RequiresAuthentication +@RequiresRoles(Roles.MICA_ADMIN) +public class PluginResource { + + @Inject + private PluginsService pluginsService; + + @GET + public MicaPlugins.PluginDto get(@PathParam("name") String name) { + return PluginDtos.asDto(pluginsService.getInstalledPlugin(name)); + } + + @DELETE + public Response uninstall(@PathParam("name") String name) { + pluginsService.prepareUninstallPlugin(name); + return Response.noContent().build(); + } + + @PUT + public Response cancelUninstallation(@PathParam("name") String name) { + pluginsService.cancelUninstallPlugin(name); + return Response.noContent().build(); + } + + @PUT + @Path("/cfg") + @Consumes("text/plain") + public Response saveConfig(@PathParam("name") String name, String properties) { + pluginsService.setInstalledPluginSiteProperties(name, properties); + return Response.ok().build(); + } + + @GET + @Path("/service") + public MicaPlugins.ServicePluginDto getServiceStatus(@PathParam("name") String name) { + return MicaPlugins.ServicePluginDto.newBuilder() + .setName(name) + .setStatus(pluginsService.isInstalledPluginRunning(name) ? MicaPlugins.ServicePluginStatus.RUNNING : MicaPlugins.ServicePluginStatus.STOPPED) + .build(); + } + + @PUT + @Path("/service") + public Response startService(@PathParam("name") String name) { + pluginsService.startInstalledPlugin(name); + return Response.ok().build(); + } + + @DELETE + @Path("/service") + public Response stopService(@PathParam("name") String name) { + pluginsService.stopInstalledPlugin(name); + return Response.ok().build(); + } +} diff --git a/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java new file mode 100644 index 0000000000..6b425e41ab --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/micaConfig/rest/PluginsResource.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.micaConfig.rest; + + +import com.google.common.base.Strings; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresRoles; +import org.obiba.mica.micaConfig.service.PluginsService; +import org.obiba.mica.security.Roles; +import org.obiba.mica.web.model.MicaPlugins; +import org.obiba.mica.web.model.PluginDtos; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Response; +import java.util.stream.Collectors; + +@Component +@Path("/config/plugins") +@RequiresAuthentication +@RequiresRoles(Roles.MICA_ADMIN) +public class PluginsResource { + + @Inject + private PluginsService pluginsService; + + @GET + public MicaPlugins.PluginPackagesDto getInstalledPlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getInstalledPlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList()), pluginsService.getUninstalledPluginNames()); + } + + @GET + @Path("/_updates") + public MicaPlugins.PluginPackagesDto getUpdatablePlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getUpdatablePlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList())); + } + + @GET + @Path("/_available") + public MicaPlugins.PluginPackagesDto getAvailablePlugins(@QueryParam("type") String type) { + return PluginDtos.asDto(pluginsService.getUpdateSite(), pluginsService.getLastUpdate(), pluginsService.restartRequired(), + pluginsService.getAvailablePlugins().stream() + .filter(p -> Strings.isNullOrEmpty(type) || (type.equals(p.getType()))) + .collect(Collectors.toList())); + } + + @POST + public Response installPlugin(@QueryParam("name") String name, @QueryParam("version") String version) { + if (!Strings.isNullOrEmpty(name)) { + pluginsService.installPlugin(name, version); + return Response.ok().build(); + } + return Response.status(Response.Status.BAD_REQUEST).build(); + } +} diff --git a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java index c384acdd6d..929dff76b4 100644 --- a/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/network/rest/DraftNetworkResource.java @@ -11,7 +11,6 @@ package org.obiba.mica.network.rest; import com.google.common.base.Strings; - import org.obiba.mica.AbstractGitPersistableResource; import org.obiba.mica.JSONUtils; import org.obiba.mica.NoSuchEntityException; @@ -20,7 +19,6 @@ import org.obiba.mica.core.service.AbstractGitPersistableService; import org.obiba.mica.file.Attachment; import org.obiba.mica.file.rest.FileResource; -import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.network.NoSuchNetworkException; import org.obiba.mica.network.domain.Network; import org.obiba.mica.network.domain.NetworkState; @@ -28,7 +26,8 @@ import org.obiba.mica.security.rest.SubjectAclResource; import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Projects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -40,18 +39,18 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; -import java.net.URISyntaxException; import java.util.HashMap; -import java.util.List; import java.util.Map; /** - * REST controller for managing draft Study. + * REST controller for managing draft Network. */ @Component @Scope("request") public class DraftNetworkResource extends AbstractGitPersistableResource { + private static final Logger log = LoggerFactory.getLogger(DraftNetworkResource.class); + @Inject private NetworkService networkService; @@ -61,9 +60,6 @@ public class DraftNetworkResource extends AbstractGitPersistableResource projects() throws URISyntaxException { - checkPermission("/draft/network", "VIEW"); - return opalService.getProjectDtos(networkService.findById(id).getOpal()); - } - @Override protected String getId() { return id; diff --git a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java index daef38d827..30440d7aca 100644 --- a/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/study/rest/StudyStateResource.java @@ -18,6 +18,7 @@ import javax.ws.rs.GET; import javax.ws.rs.Path; +import org.apache.commons.compress.utils.Lists; import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.study.domain.BaseStudy; @@ -25,6 +26,8 @@ import org.obiba.mica.web.model.Dtos; import org.obiba.mica.web.model.Mica; import org.obiba.opal.web.model.Projects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -37,6 +40,8 @@ @Scope("request") public class StudyStateResource { + private static final Logger log = LoggerFactory.getLogger(StudyStateResource.class); + @Inject private StudyService studyService; @@ -65,11 +70,15 @@ public Mica.StudySummaryDto get() { } @GET - @Path("/projects") + @Path("/opal-projects") public List projects() throws URISyntaxException { subjectAclService.checkPermission(resource, "VIEW", id); String opalUrl = Optional.ofNullable(studyService.findStudy(id)).map(BaseStudy::getOpal).orElse(null); - - return opalService.getProjectDtos(opalUrl); + try { + return opalService.getProjectDtos(opalUrl); + } catch (Exception e) { + log.warn("Failed at retrieving opal projects: {}", e.getMessage()); + return Lists.newArrayList(); + } } } diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java new file mode 100644 index 0000000000..978b71e231 --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomyResource.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.taxonomy.rest; + +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; +import org.obiba.opal.core.cfg.NoSuchTaxonomyException; +import org.obiba.opal.core.cfg.NoSuchVocabularyException; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.Vocabulary; +import org.obiba.opal.web.model.Opal; +import org.obiba.opal.web.taxonomy.Dtos; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class AbstractTaxonomyResource { + + @Inject + protected VariableTaxonomiesService variableTaxonomiesService; + + protected List getTaxonomyDtos() { + return variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asDto).collect(Collectors.toList()); + } + + /** + * Get a summary of all the {@link Taxonomy}s available from Opal master. + * + * @return + */ + protected Opal.TaxonomiesDto getTaxonomySummaryDtos() { + List summaries = variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asSummaryDto) + .collect(Collectors.toList()); + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} available from Opal master. + * + * @param name the taxonomy name + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummaryDto(String name) { + return Dtos.asSummaryDto(getTaxonomyInternal(name)); + } + + /** + * Get a summary of all the {@link Taxonomy}s with their + * {@link Vocabulary}s from Opal master. + * + * @return + */ + protected Opal.TaxonomiesDto getTaxonomyVocabularySummaryDtos() { + List summaries = variableTaxonomiesService.getTaxonomies().stream().map(Dtos::asVocabularySummaryDto) + .collect(Collectors.toList()); + + return Opal.TaxonomiesDto.newBuilder().addAllSummaries(summaries).build(); + } + + /** + * Get a summary of the {@link Taxonomy} with its + * {@link Vocabulary}s from Opal master. + * + * @param name the taxonomy name + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomyVocabularySummaryDto(String name) { + return Dtos.asVocabularySummaryDto(getTaxonomyInternal(name)); + } + + /** + * Get a summary of the {@link Vocabulary} from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + protected Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getTaxonomyVocabularySummaryDto(String name, + String vocabularyName) { + for (Vocabulary voc : getTaxonomyInternal(name).getVocabularies()) { + if (voc.getName().equals(vocabularyName)) return Dtos.asSummaryDto(voc); + } + throw new NoSuchVocabularyException(name, vocabularyName); + } + + + /** + * Get the {@link Taxonomy} as a Dto from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + protected Opal.TaxonomyDto getTaxonomyDto(String name) { + return Dtos.asDto(getTaxonomyInternal(name)); + } + + /** + * Get the {@link Vocabulary} as a Dto from Opal master. + * + * @param name + * @param vocabularyName + * @return + */ + protected Opal.VocabularyDto getTaxonomyVocabularyDto(String name, String vocabularyName) { + return Dtos.asDto(getTaxonomyInternal(name).getVocabulary(vocabularyName)); + } + + // + // Private methods + // + + /** + * Get the {@link Taxonomy} from Opal master. + * + * @param name + * @return + * @throws NoSuchTaxonomyException + */ + private Taxonomy getTaxonomyInternal(String name) { + Optional taxonomy = variableTaxonomiesService.getTaxonomies().stream() + .filter(taxo -> taxo.getName().equals(name)) + .findFirst(); + if (!taxonomy.isPresent()) { + throw new NoSuchTaxonomyException(name); + } + return taxonomy.get(); + } + +} diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java new file mode 100644 index 0000000000..42f26b84f9 --- /dev/null +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/MetaTaxonomyResource.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.taxonomy.rest; + + +import com.codahale.metrics.annotation.Timed; +import org.apache.shiro.authz.annotation.RequiresRoles; +import org.obiba.mica.micaConfig.service.TaxonomiesService; +import org.obiba.mica.security.Roles; +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.opal.web.model.Opal; +import org.obiba.opal.web.taxonomy.Dtos; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +@Component +@Path("/meta-taxonomy") +public class MetaTaxonomyResource { + + @Inject + private TaxonomiesService taxonomiesService; + + @GET + @Timed + public Opal.TaxonomyDto getTaxonomy() { + return Dtos.asDto(taxonomiesService.getTaxonomyTaxonomy()); + } + + @PUT + @Path("/{target}/{taxonomy}/_move") + @RequiresRoles(Roles.MICA_ADMIN) + public Response up(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("dir") String dir) { + taxonomiesService.moveTaxonomy(TaxonomyTarget.fromId(target), taxonomy, "up".equals(dir)); + return Response.ok().build(); + } + + @PUT + @Path("/{target}/{taxonomy}/_attribute") + @RequiresRoles(Roles.MICA_ADMIN) + public Response hide(@PathParam("target") @DefaultValue("variable") String target, @PathParam("taxonomy") String taxonomy, @QueryParam("name") String name, @QueryParam("value") String value) { + taxonomiesService.setTaxonomyAttribute(TaxonomyTarget.fromId(target), taxonomy, name, value); + return Response.ok().build(); + } + +} diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java index f5d32b12c9..0736ee2dc6 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesResource.java @@ -10,45 +10,34 @@ package org.obiba.mica.taxonomy.rest; -import java.util.List; - -import javax.inject.Inject; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Response; - +import com.codahale.metrics.annotation.Timed; import com.google.common.eventbus.EventBus; -import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresRoles; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; -import org.obiba.mica.micaConfig.service.OpalService; import org.obiba.mica.security.Roles; import org.obiba.opal.web.model.Opal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.ws.rs.*; +import javax.ws.rs.core.Response; +import java.util.List; @Component @Path("/taxonomies") -public class TaxonomiesResource { +public class TaxonomiesResource extends AbstractTaxonomyResource { private static final Logger logger = LoggerFactory.getLogger(TaxonomiesResource.class); - @Inject - private OpalService opalService; - @Inject private EventBus eventBus; @GET @Timed public List getTaxonomies() { - return opalService.getTaxonomyDtos(); + return getTaxonomyDtos(); } @GET @@ -56,8 +45,8 @@ public List getTaxonomies() { @Timed public Opal.TaxonomiesDto getTaxonomySummaries( @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return opalService.getTaxonomyVocabularySummaryDtos(); - return opalService.getTaxonomySummaryDtos(); + if(withVocabularies) return getTaxonomyVocabularySummaryDtos(); + return getTaxonomySummaryDtos(); } @PUT diff --git a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java index 924438856b..6b09619f96 100644 --- a/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomyResource.java @@ -10,31 +10,20 @@ package org.obiba.mica.taxonomy.rest; -import javax.inject.Inject; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; - -import org.apache.shiro.authz.annotation.RequiresAuthentication; -import org.obiba.mica.micaConfig.service.OpalService; +import com.codahale.metrics.annotation.Timed; import org.obiba.opal.web.model.Opal; import org.springframework.stereotype.Component; -import com.codahale.metrics.annotation.Timed; +import javax.ws.rs.*; @Component @Path("/taxonomy/{name}") -public class TaxonomyResource { - - @Inject - private OpalService opalService; +public class TaxonomyResource extends AbstractTaxonomyResource { @GET @Timed public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { - return opalService.getTaxonomyDto(name); + return getTaxonomyDto(name); } @GET @@ -42,8 +31,8 @@ public Opal.TaxonomyDto getTaxonomy(@PathParam("name") String name) { @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name") String name, @QueryParam("vocabularies") @DefaultValue("false") boolean withVocabularies) { - if(withVocabularies) return opalService.getTaxonomyVocabularySummaryDto(name); - return opalService.getTaxonomySummaryDto(name); + if(withVocabularies) return getTaxonomyVocabularySummaryDto(name); + return getTaxonomySummaryDto(name); } @GET @@ -51,7 +40,7 @@ public Opal.TaxonomiesDto.TaxonomySummaryDto getTaxonomySummary(@PathParam("name @Timed public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return opalService.getTaxonomyVocabularyDto(name, vocabularyName); + return getTaxonomyVocabularyDto(name, vocabularyName); } @GET @@ -59,7 +48,7 @@ public Opal.VocabularyDto getVocabulary(@PathParam("name") String name, @Timed public Opal.TaxonomiesDto.TaxonomySummaryDto.VocabularySummaryDto getVocabularySummary(@PathParam("name") String name, @PathParam("vocabulary") String vocabularyName) { - return opalService.getTaxonomyVocabularySummaryDto(name, vocabularyName); + return getTaxonomyVocabularySummaryDto(name, vocabularyName); } } diff --git a/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java b/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java index da97d065bc..00b98da417 100644 --- a/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java +++ b/mica-rest/src/main/java/org/obiba/mica/web/rest/CacheResource.java @@ -27,7 +27,7 @@ @RequiresRoles(Roles.MICA_ADMIN) public class CacheResource { - private static final Logger logger = LoggerFactory.getLogger(IllegalArgumentExceptionMapper.class); + private static final Logger logger = LoggerFactory.getLogger(CacheResource.class); @Inject private CacheService cacheService; @@ -43,14 +43,15 @@ public Response deleteCaches() { @DELETE public Response deleteCache(@PathParam("id") String id) { - logger.info("clear cache [{}]", id); + logger.info("Clear cache [{}]", id); switch(id) { case "micaConfig" : cacheService.clearMicaConfigCache(); break; case "opalTaxonomies": - cacheService.clearOpalTaxonomiesCache(); + case "variableTaxonomies": + cacheService.clearTaxonomiesCache(); break; case "aggregationsMetadata": cacheService.clearAggregationsMetadataCache(); diff --git a/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java b/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java index 2fbdb9ed25..1990efe7ef 100644 --- a/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java +++ b/mica-rest/src/main/java/org/obiba/mica/web/rest/IllegalArgumentExceptionMapper.java @@ -24,7 +24,7 @@ public class IllegalArgumentExceptionMapper implements ExceptionMapper { protected ObjectMapper objectMapper; @Inject - protected OpalService opalService; + protected TaxonomiesService taxonomiesService; @Inject protected MicaConfigService micaConfigService; @@ -211,22 +209,21 @@ protected Mica.DatasetVariableHarmonizationSummaryDto getVariableHarmonizationSu } protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, OpalTable opalTable) { + DatasetVariable.Type variableType, BaseStudyTable studyTable) { - if (opalTable != null) { + if (studyTable != null) { return getDatasetVariable(datasetId, variableName, variableType, - opalTable instanceof BaseStudyTable ? ((BaseStudyTable) opalTable).getStudyId() : null, - opalTable.getProject(), - opalTable.getTable(), - opalTable instanceof StudyTable + studyTable.getStudyId(), + studyTable.getSource(), + studyTable instanceof StudyTable ? DatasetVariable.OPAL_STUDY_TABLE_PREFIX : DatasetVariable.OPAL_HARMONIZATION_TABLE_PREFIX); } - return getDatasetVariable(datasetId, variableName, variableType, null, null, null, null); + return getDatasetVariable(datasetId, variableName, variableType, null, null, null); } /** @@ -236,18 +233,18 @@ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull * @param datasetId * @param variableName * @param studyId - * @param project - * @param table + * @param source + * @param tableType * @return * @throws NoSuchVariableException */ protected DatasetVariable getDatasetVariable(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String project, @Nullable String table, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String source, @Nullable String tableType) throws NoSuchVariableException { String variableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, variableType, studyId, project, table, tableType); + .encode(datasetId, variableName, variableType, studyId, source, tableType); if (variableType.equals(DatasetVariable.Type.Harmonized)) { return getHarmonizedDatasetVariable(datasetId, variableId, variableName); @@ -262,23 +259,23 @@ protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetI } protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { + DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { return dtos - .asDto(getDatasetVariable(datasetId, variableName, variableType, opalTable), getTaxonomies(), + .asDto(getDatasetVariable(datasetId, variableName, variableType, studyTable), getTaxonomies(), getLocale()); } protected Mica.DatasetVariableDto getDatasetVariableDto(@NotNull String datasetId, @NotNull String variableName, - DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String project, @Nullable String table, + DatasetVariable.Type variableType, @Nullable String studyId, @Nullable String source, @Nullable String tableType) { - return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, project, table, tableType), + return dtos.asDto(getDatasetVariable(datasetId, variableName, variableType, studyId, source, tableType), getTaxonomies(), getLocale()); } protected Mica.DatasetHarmonizedVariableSummaryDto getDatasetHarmonizedVariableSummaryDto(@NotNull String datasetId, - @NotNull String variableName, DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { + @NotNull String variableName, DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { try { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); return dtos.asHarmonizedSummaryDto(variable); } catch (NoSuchVariableException e) { return Mica.DatasetHarmonizedVariableSummaryDto.newBuilder().setStatus("").build(); @@ -287,29 +284,23 @@ protected Mica.DatasetHarmonizedVariableSummaryDto getDatasetHarmonizedVariableS protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull String datasetId, - @NotNull String variableName, DatasetVariable.Type variableType, @Nullable OpalTable opalTable) { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); - return dtos.asSummaryDto(variable, opalTable, true); + @NotNull String variableName, DatasetVariable.Type variableType, @Nullable BaseStudyTable studyTable) { + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); + return dtos.asSummaryDto(variable, studyTable, true); } protected Mica.DatasetVariableSummaryDto getDatasetVariableSummaryDto(@NotNull String datasetId, @NotNull String variableName, DatasetVariable.Type variableType, - @Nullable OpalTable opalTable, + @Nullable BaseStudyTable studyTable, boolean includeSummaries) { - DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, opalTable); - return dtos.asSummaryDto(variable, opalTable, includeSummaries); + DatasetVariable variable = getDatasetVariable(datasetId, variableName, variableType, studyTable); + return dtos.asSummaryDto(variable, studyTable, includeSummaries); } @NotNull protected List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + return taxonomiesService.getVariableTaxonomies(); } protected void checkContingencyAccess() { @@ -324,7 +315,7 @@ protected void checkVariableSummaryAccess() { private DatasetVariable getHarmonizedDatasetVariable(String datasetId, String variableId, String variableName) { String dataSchemaVariableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, DatasetVariable.Type.Dataschema, null, null, null, null); + .encode(datasetId, variableName, DatasetVariable.Type.Dataschema); DatasetVariable harmonizedDatasetVariable = getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, variableId, variableName); DatasetVariable dataSchemaVariable = getDatasetVariableInternal(Indexer.PUBLISHED_VARIABLE_INDEX, Indexer.VARIABLE_TYPE, diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java index f6af8217b8..68b5b66d88 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/collection/PublishedCollectedDatasetVariableResource.java @@ -22,7 +22,6 @@ import org.obiba.mica.dataset.search.rest.harmonization.ExcelContingencyWriter; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -54,24 +53,6 @@ public Mica.DatasetVariableDto getVariable() { return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Collected); } - @GET - @Path("/summary") - @Timed - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService.getVariableSummary(getDataset(StudyDataset.class, datasetId), variableName).getWrappedDto(); - } - - @GET - @Path("/facet") - @Timed - public Search.QueryResultDto getVariableFacet() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService.getVariableFacet(getDataset(StudyDataset.class, datasetId), variableName); - } - @GET @Path("/aggregation") @Timed @@ -80,11 +61,13 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s checkVariableSummaryAccess(); StudyDataset dataset = getDataset(StudyDataset.class, datasetId); StudyTable studyTable = dataset.getSafeStudyTable(); - Mica.DatasetVariableAggregationDto.Builder aggDto = Mica.DatasetVariableAggregationDto.newBuilder(); try { - return dtos.asDto(studyTable, datasetService.getVariableSummary(dataset, variableName).getWrappedDto(), withStudySummary).build(); + return dtos.asDto(studyTable, datasetService.getVariableSummary(dataset, variableName), withStudySummary).build(); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); return dtos.asDto(studyTable, null, withStudySummary).build(); } } @@ -106,11 +89,11 @@ private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var try { return dtos - .asContingencyDto(studyTable, var, crossVar, datasetService.getContingencyTable(dataset, var, crossVar)) + .asContingencyDto(studyTable, datasetService.getContingencyTable(dataset, var, crossVar)) .build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - return dtos.asContingencyDto(studyTable, var, crossVar, null).build(); + return dtos.asContingencyDto(studyTable, null).build(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java index 5b922ada9b..aa2651989f 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvContingencyWriter.java @@ -84,10 +84,9 @@ private void writeBody(CSVWriter writer, Mica.DatasetVariableContingencyDto dto) private void writeHeaders(CSVWriter writer, Mica.DatasetVariableContingencyDto c, List terms) { if(c.hasStudyTable()) writer.writeNext(new String[] { String - .format("%s - %s - %s", c.getStudyTable().getProject(), c.getStudyTable().getTable(), - c.getStudyTable().getDceId()) }); + .format("%s - %s", c.getStudyTable().getSource(), c.getStudyTable().getDceId()) }); else if(c.hasHarmonizationStudyTable()) writer.writeNext(new String[] { String - .format("%s - %s", c.getHarmonizationStudyTable().getProject(), c.getHarmonizationStudyTable().getTable())}); + .format("%s", c.getHarmonizationStudyTable().getSource())}); writer.writeNext(concat(concat(Stream.of(""), terms.stream()), Stream.of("Total")).toArray(String[]::new)); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java index b4197e64bb..c0adeb06ab 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/CsvHarmonizationVariablesWriter.java @@ -83,11 +83,10 @@ private void writeBody(CSVWriter writer, HarmonizationDataset dataset, final boolean[] found = { false }; variableHarmonization.getDatasetVariableSummariesList().forEach( summary -> { - String id = table instanceof StudyTable ? ((StudyTable) table).getStudyId() : ((HarmonizationStudyTable) table).getStudyId(); + String id = table.getStudyId(); Mica.DatasetVariableResolverDto resolver = summary.getResolver(); if ((resolver.getStudyId().equals(id) - && resolver.getProject().equals(table.getProject()) - && resolver.getTable().equals(table.getTable()))) { + && resolver.getSource().equals(table.getSource()))) { String statusDetail = getAttributeByName(summary, "status_detail", locale); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java index 45bbf17570..2b1e23951a 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedDataschemaDatasetVariableResource.java @@ -17,20 +17,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.commons.math3.util.Pair; -import org.obiba.magma.NoSuchValueTableException; -import org.obiba.magma.NoSuchVariableException; import org.obiba.mica.core.domain.BaseStudyTable; -import org.obiba.mica.core.domain.HarmonizationStudyTable; -import org.obiba.mica.core.domain.OpalTable; -import org.obiba.mica.core.domain.StudyTable; import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.search.rest.AbstractPublishedDatasetResource; import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Math; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -73,49 +66,6 @@ public Mica.DatasetVariableDto getVariable() { return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Dataschema); } - @GET - @Path("/summary") - @Timed - public List getVariableSummaries() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) - .getWrappedDto()); - } catch (NoSuchVariableException | NoSuchValueTableException e) { - // case the study has not implemented this dataschema variable - builder.add(Math.SummaryStatisticsDto.newBuilder().setResource(variableName).build()); - } - }); - return builder.build(); - } - - @GET - @Path("/facet") - @Timed - public List getVariableFacets() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - ImmutableList.Builder builder = ImmutableList.builder(); - HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - dataset.getBaseStudyTables().forEach(table -> { - try { - String studyId = table.getStudyId(); - builder.add(datasetService - .getVariableFacet(dataset, variableName, studyId, table.getProject(), table.getTable())); - } catch (NoSuchVariableException | NoSuchValueTableException e) { - // case the study has not implemented this dataschema variable - builder.add(Search.QueryResultDto.newBuilder().setTotalHits(0).build()); - } - }); - return builder.build(); - } - @GET @Path("/aggregation") @Timed @@ -126,16 +76,19 @@ public Mica.DatasetVariableAggregationsDto getVariableAggregations(@QueryParam(" HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); Mica.DatasetVariableAggregationsDto.Builder aggDto = Mica.DatasetVariableAggregationsDto.newBuilder(); - List> results = Lists.newArrayList(); + List> results = Lists.newArrayList(); dataset.getBaseStudyTables().forEach(table -> results.add(helper.getVariableFacet(dataset, variableName, table))); for (int i = 0; i < dataset.getBaseStudyTables().size(); i++) { BaseStudyTable opalTable = dataset.getBaseStudyTables().get(i); - Future futureResult = results.get(i); + Future futureResult = results.get(i); try { builder.add(dtos.asDto(opalTable, futureResult.get(), withStudySummary).build()); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); builder.add(dtos.asDto(opalTable, null, withStudySummary).build()); } } @@ -228,25 +181,25 @@ private Mica.DatasetVariableContingenciesDto getDatasetVariableContingenciesDto( HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); Mica.DatasetVariableContingenciesDto.Builder crossDto = Mica.DatasetVariableContingenciesDto.newBuilder(); - List> results = Lists.newArrayList(); + List> results = Lists.newArrayList(); dataset.getBaseStudyTables().forEach(table -> results.add(helper.getContingencyTable(dataset, var, crossVar, table))); Multimap termAggregations = LinkedListMultimap.create(); for (int i = 0; i < dataset.getBaseStudyTables().size(); i++) { - BaseStudyTable opalTable = dataset.getBaseStudyTables().get(i); - Future futureResult = results.get(i); + BaseStudyTable studyTable = dataset.getBaseStudyTables().get(i); + Future futureResult = results.get(i); try { Mica.DatasetVariableContingencyDto studyTableCrossDto = dtos - .asContingencyDto(opalTable, var, crossVar, futureResult.get()).build(); + .asContingencyDto(studyTable, futureResult.get()).build(); termAggregations.put(null, studyTableCrossDto.getAll()); studyTableCrossDto.getAggregationsList() .forEach(termAggDto -> termAggregations.put(termAggDto.getTerm(), termAggDto)); crossDto.addContingencies(studyTableCrossDto); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - crossDto.addContingencies(dtos.asContingencyDto(opalTable, var, crossVar, null)); + crossDto.addContingencies(dtos.asContingencyDto(studyTable, null)); } } @@ -287,31 +240,25 @@ public static class Helper { protected HarmonizedDatasetService datasetService; @Async - protected Future getVariableFacet(HarmonizationDataset dataset, String variableName, - OpalTable table) { + protected Future getVariableFacet(HarmonizationDataset dataset, String variableName, + BaseStudyTable table) { try { - String studyId = null; - - if (table instanceof StudyTable) { - studyId = ((StudyTable) table).getStudyId(); - } else if (table instanceof HarmonizationStudyTable) { - studyId = ((HarmonizationStudyTable) table).getStudyId(); - } - - return new AsyncResult<>(datasetService - .getVariableSummary(dataset, variableName, studyId, table.getProject(), table.getTable()) - .getWrappedDto()); + String studyId = table.getStudyId(); + return new AsyncResult<>(datasetService.getVariableSummary(dataset, variableName, studyId, table.getSource())); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); return new AsyncResult<>(null); } } @Async - protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, - DatasetVariable crossVar, OpalTable table) { + protected Future getContingencyTable(HarmonizationDataset dataset, DatasetVariable var, + DatasetVariable crossVar, BaseStudyTable studyTable) { try { - return new AsyncResult<>(datasetService.getContingencyTable(table, var, crossVar)); + return new AsyncResult<>(datasetService.getContingencyTable(dataset, studyTable, var, crossVar)); } catch (Exception e) { log.warn("Unable to retrieve contingency statistics: " + e.getMessage(), e); return new AsyncResult<>(null); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java index d315837740..1f89d56c96 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetResource.java @@ -215,16 +215,17 @@ private Mica.DatasetVariablesHarmonizationSummaryDto getVariableHarmonizationsSu // harmonized variables, extract one column per study table List> harmonizedVariables = Lists.newArrayList(); builder.setTotal(variablesDto.getTotal()).setLimit(variablesDto.getLimit()).setFrom(variablesDto.getFrom()); - String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(project,%s),eq(table,%s))"; + + String query = "and(%s,eq(datasetId,%s),eq(studyId,%s),eq(populationId,%s),eq(dceId,%s),eq(opalTableType,%s),eq(source,%s))"; dataset.getStudyTables().forEach(st -> { builder.addStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getProject(), st.getTable()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Study, st.getSource()); List datasetVariablesInternal = getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true); harmonizedVariables.add(datasetVariablesInternal.stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); dataset.getHarmonizationTables().forEach(st -> { builder.addHarmonizationStudyTable(dtos.asDto(st, includeSummaries)); - String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getProject(), st.getTable()); + String rql = String.format(query, namesQuery, id, st.getStudyId(), st.getPopulationUId(), st.getDataCollectionEventUId(), DatasetVariable.OpalTableType.Harmonization, st.getSource()); harmonizedVariables.add(getDatasetVariablesInternal(rql, 0, hvariablesLimit, sort, order, true).stream().collect(Collectors.toMap(DatasetVariable::getName, v -> v))); }); diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java index 3e246b9539..88ddd8eba3 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/harmonization/PublishedHarmonizedDatasetVariableResource.java @@ -21,7 +21,6 @@ import org.obiba.mica.dataset.search.rest.AbstractPublishedDatasetResource; import org.obiba.mica.dataset.service.HarmonizedDatasetService; import org.obiba.mica.web.model.Mica; -import org.obiba.opal.web.model.Search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -49,9 +48,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe private String studyId; - private String project; - - private String table; + private String source; private String tableType; @@ -62,29 +59,7 @@ public class PublishedHarmonizedDatasetVariableResource extends AbstractPublishe @Timed public Mica.DatasetVariableDto getVariable() { checkDatasetAccess(); - return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, project, table, - tableType); - } - - @GET - @Path("/summary") - @Timed - public org.obiba.opal.web.model.Math.SummaryStatisticsDto getVariableSummary() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService - .getVariableSummary(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, project, table) - .getWrappedDto(); - } - - @GET - @Path("/facet") - @Timed - public Search.QueryResultDto getVariableFacet() { - checkDatasetAccess(); - checkVariableSummaryAccess(); - return datasetService - .getVariableFacet(getDataset(HarmonizationDataset.class, datasetId), variableName, studyId, project, table); + return getDatasetVariableDto(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, source, tableType); } @GET @@ -94,20 +69,22 @@ public Mica.DatasetVariableAggregationDto getVariableAggregations(@QueryParam("s checkDatasetAccess(); checkVariableSummaryAccess(); HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - for (BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if (opalTable.isFor(opalTableId, project, table)) { + for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if (baseTable.isFor(studyId, source)) { try { - return dtos.asDto(opalTable, - datasetService.getVariableSummary(dataset, variableName, studyId, project, table).getWrappedDto(), withStudySummary).build(); + return dtos.asDto(baseTable, + datasetService.getVariableSummary(dataset, variableName, studyId, source), withStudySummary).build(); } catch (Exception e) { - log.warn("Unable to retrieve statistics: " + e.getMessage(), e); - return dtos.asDto(opalTable, null, withStudySummary).build(); + if (log.isDebugEnabled()) + log.warn("Unable to retrieve statistics: {}", e.getMessage(), e); + else + log.warn("Unable to retrieve statistics: {}", e.getMessage()); + return dtos.asDto(baseTable, null, withStudySummary).build(); } } } - throw new NoSuchValueTableException(project, table); + throw new NoSuchValueTableException(source); } @GET @@ -123,20 +100,18 @@ public Mica.DatasetVariableContingencyDto getContingency(@QueryParam("by") Strin private Mica.DatasetVariableContingencyDto getContingencyDto(DatasetVariable var, DatasetVariable crossVar) { HarmonizationDataset dataset = getDataset(HarmonizationDataset.class, datasetId); - for (BaseStudyTable opalTable : dataset.getBaseStudyTables()) { - String opalTableId = studyId; - if (opalTable.isFor(opalTableId, project, table)) { + for (BaseStudyTable baseTable : dataset.getBaseStudyTables()) { + if (baseTable.isFor(studyId, source)) { try { - return dtos.asContingencyDto(opalTable, var, crossVar, - datasetService.getContingencyTable(opalTable, var, crossVar)).build(); + return dtos.asContingencyDto(baseTable, datasetService.getContingencyTable(dataset, baseTable, var, crossVar)).build(); } catch (Exception e) { log.warn("Unable to retrieve contingency table: " + e.getMessage(), e); - return dtos.asContingencyDto(opalTable, var, crossVar, null).build(); + return dtos.asContingencyDto(baseTable, null).build(); } } } - throw new NoSuchValueTableException(project, table); + throw new NoSuchValueTableException(source); } @GET @@ -183,12 +158,8 @@ public void setStudyId(String studyId) { this.studyId = studyId; } - public void setProject(String project) { - this.project = project; - } - - public void setTable(String table) { - this.table = table; + public void setSource(String source) { + this.source = source; } public void setTableType(String tableType) { @@ -199,10 +170,8 @@ private Pair getContingencyVariables(String cr if (Strings.isNullOrEmpty(crossVariable)) throw new BadRequestException("Cross variable name is required for the contingency table"); - DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, project, - table, tableType); - DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, - project, table, tableType); + DatasetVariable var = getDatasetVariable(datasetId, variableName, DatasetVariable.Type.Harmonized, studyId, source, tableType); + DatasetVariable crossVar = getDatasetVariable(datasetId, crossVariable, DatasetVariable.Type.Harmonized, studyId, source, tableType); return Pair.create(var, crossVar); } diff --git a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java index 0a590df80f..e3523ff049 100644 --- a/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java +++ b/mica-search/src/main/java/org/obiba/mica/dataset/search/rest/variable/PublishedDatasetVariableResource.java @@ -16,7 +16,6 @@ import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; -import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.obiba.mica.dataset.DatasetVariableResource; import org.obiba.mica.dataset.NoSuchDatasetException; import org.obiba.mica.dataset.domain.DatasetVariable; @@ -64,8 +63,7 @@ public DatasetVariableResource getVariable(@PathParam("id") String id, if (!harmonizedDatasetService.isPublished(resolver.getDatasetId())) throw NoSuchDatasetException.withId(resolver.getDatasetId()); resource = applicationContext.getBean(PublishedHarmonizedDatasetVariableResource.class); ((PublishedHarmonizedDatasetVariableResource)resource).setStudyId(resolver.getStudyId()); - ((PublishedHarmonizedDatasetVariableResource)resource).setProject(resolver.getProject()); - ((PublishedHarmonizedDatasetVariableResource)resource).setTable(resolver.getTable()); + ((PublishedHarmonizedDatasetVariableResource)resource).setSource(resolver.getSource()); ((PublishedHarmonizedDatasetVariableResource)resource).setTableType(resolver.getTableType()); ((PublishedHarmonizedDatasetVariableResource)resource).setLocale(locale); break; diff --git a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java index a4ea340c0c..9b3f0c8318 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java +++ b/mica-search/src/main/java/org/obiba/mica/search/CoverageQueryExecutor.java @@ -12,7 +12,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.mica.spi.search.support.JoinQuery; @@ -24,7 +24,6 @@ import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import sun.util.locale.LanguageTag; import javax.annotation.Nullable; import javax.inject.Inject; @@ -43,7 +42,7 @@ public class CoverageQueryExecutor { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; @Inject - private OpalService opalService; + private TaxonomiesService taxonomiesService; @Inject private JoinQueryExecutor joinQueryExecutor; @@ -330,13 +329,7 @@ private MicaSearch.BucketCoverageDto.Builder getBucketCoverageDtoBuilder(String @NotNull private List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + return taxonomiesService.getVariableTaxonomies(); } @Nullable diff --git a/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java b/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java index 2330389680..defe3889fd 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java +++ b/mica-search/src/main/java/org/obiba/mica/search/JoinQueryExecutor.java @@ -17,7 +17,7 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.UncheckedTimeoutException; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.search.queries.DatasetQuery; import org.obiba.mica.search.queries.DocumentQueryInterface; import org.obiba.mica.search.queries.NetworkQuery; @@ -74,7 +74,7 @@ public class JoinQueryExecutor { private long concurrentJoinQueriesWaitTimeout; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private VariableQuery variableQuery; @@ -182,22 +182,22 @@ private JoinQueryResultDto buildQueryResult(JoinQuery joinQueryDto) { builder.setVariableResultDto(joinQueryDto.isWithFacets() ? addAggregationTitles(variableQuery.getResultQuery(), - ImmutableList.builder().add(taxonomyService.getVariableTaxonomy()) - .addAll(taxonomyService.getOpalTaxonomies()).build(), aggregationPostProcessor()) + ImmutableList.builder().add(taxonomiesService.getVariableTaxonomy()) + .addAll(taxonomiesService.getVariableTaxonomies()).build(), aggregationPostProcessor()) : removeAggregations(variableQuery.getResultQuery())); builder.setDatasetResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(datasetQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getDatasetTaxonomy()), + ? addAggregationTitles(datasetQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getDatasetTaxonomy()), null) : removeAggregations(datasetQuery.getResultQuery())); builder.setStudyResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(studyQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getStudyTaxonomy()), + ? addAggregationTitles(studyQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getStudyTaxonomy()), null) : removeAggregations(studyQuery.getResultQuery())); builder.setNetworkResultDto(joinQueryDto.isWithFacets() - ? addAggregationTitles(networkQuery.getResultQuery(), Lists.newArrayList(taxonomyService.getNetworkTaxonomy()), + ? addAggregationTitles(networkQuery.getResultQuery(), Lists.newArrayList(taxonomiesService.getNetworkTaxonomy()), null) : removeAggregations(networkQuery.getResultQuery())); @@ -259,7 +259,7 @@ private String getVocabularyAggregationName(Vocabulary vocabulary) { protected Function, List> aggregationPostProcessor() { return (aggregationResultDtos) -> { - Map buildres = taxonomyService.getOpalTaxonomies().stream() + Map buildres = taxonomiesService.getVariableTaxonomies().stream() .collect(Collectors.toMap(Taxonomy::getName, t -> { MicaSearch.AggregationResultDto.Builder builder = MicaSearch.AggregationResultDto.newBuilder() .setAggregation(t.getName()); diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java index d476908e16..1be9371a78 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/ConfigurationTaxonomyMetaDataProvider.java @@ -18,7 +18,7 @@ import com.google.common.base.Strings; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Vocabulary; @@ -32,7 +32,7 @@ public abstract class ConfigurationTaxonomyMetaDataProvider implements Aggregati private static final Logger log = LoggerFactory.getLogger(ConfigurationTaxonomyMetaDataProvider.class); @Inject - protected TaxonomyService taxonomyService; + protected TaxonomiesService taxonomiesService; Map> cache; diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java index 3bba075774..503b0f15d8 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/DatasetTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class DatasetTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaDa @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java index 9dfefbe1e2..e160d53c6d 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/NetworkTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class NetworkTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaDa @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java index 65714bac1e..c885a14183 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/StudyTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class StudyTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaData @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java index 0fbedae7e2..2e6fcc7c08 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/TaxonomyAggregationMetaDataProvider.java @@ -10,31 +10,28 @@ package org.obiba.mica.search.aggregations; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import javax.inject.Inject; - -import org.obiba.mica.spi.search.support.AttributeKey; +import com.google.common.base.Strings; +import com.google.common.collect.Maps; import org.obiba.mica.core.domain.LocalizedString; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; +import org.obiba.mica.spi.search.support.AttributeKey; import org.obiba.opal.core.domain.taxonomy.Taxonomy; import org.obiba.opal.core.domain.taxonomy.Term; import org.obiba.opal.core.domain.taxonomy.Vocabulary; import org.springframework.stereotype.Component; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; +import javax.inject.Inject; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; @Component public class TaxonomyAggregationMetaDataProvider implements AggregationMetaDataProvider { @Inject - OpalService opalService; + TaxonomiesService taxonomiesService; Map> cache; @@ -105,7 +102,7 @@ private Optional getVocabulary(String aggregation) { String targetTaxonomy = attrKey.hasNamespace(null) ? "Default" : attrKey.getNamespace(); String targetVocabulary = attrKey.getName(); - return getTaxonomies().stream() // + return getVariableTaxonomies().stream() // .filter(taxonomy -> !Strings.isNullOrEmpty(targetTaxonomy) && taxonomy.getName().equals(targetTaxonomy)) // .map(Taxonomy::getVocabularies) // .flatMap(Collection::stream) // @@ -113,12 +110,7 @@ private Optional getVocabulary(String aggregation) { .findFirst(); } - protected List getTaxonomies() { - try { - return opalService.getTaxonomies(); - } catch(Exception e) { - // ignore - } - return Collections.emptyList(); + protected List getVariableTaxonomies() { + return taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java b/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java index 1a8ba47814..7a102a116b 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java +++ b/mica-search/src/main/java/org/obiba/mica/search/aggregations/VariableTaxonomyMetaDataProvider.java @@ -18,6 +18,6 @@ public class VariableTaxonomyMetaDataProvider extends ConfigurationTaxonomyMetaD @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java index 7e0bfe45ad..350171a1bf 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/AbstractDocumentQuery.java @@ -16,7 +16,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.obiba.mica.micaConfig.service.MicaConfigService; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.mica.search.DocumentQueryHelper; import org.obiba.mica.search.aggregations.AggregationMetaDataResolver; @@ -55,7 +55,7 @@ public abstract class AbstractDocumentQuery implements DocumentQueryInterface { protected MicaConfigService micaConfigService; @Inject - protected TaxonomyService taxonomyService; + protected TaxonomiesService taxonomiesService; @Inject protected SubjectAclService subjectAclService; diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java index 05df358b7d..ad2af62a63 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/DatasetQuery.java @@ -99,7 +99,7 @@ public Collection getValues() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); } public void setDatasetIdProvider(DocumentQueryIdProvider provider) { @@ -135,7 +135,7 @@ public void query(List studyIds, CountStatsData counts, QueryScope scope @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getDatasetTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getDatasetTaxonomy()); if (!properties.containsKey(STUDY_JOIN_FIELD)) properties.put(STUDY_JOIN_FIELD, ""); if (!properties.containsKey(HARMONIZATION_STUDY_JOIN_FIELD)) properties.put(HARMONIZATION_STUDY_JOIN_FIELD, ""); if (!properties.containsKey(ID)) properties.put(ID, ""); diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java index 4633d4e16c..2931149467 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/NetworkQuery.java @@ -103,13 +103,13 @@ public Collection getValues() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); } @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getNetworkTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getNetworkTaxonomy()); if (!properties.containsKey(JOIN_FIELD)) properties.put(JOIN_FIELD, ""); return properties; } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java index 8a7d83b887..b5aa2dc989 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/StudyQuery.java @@ -91,7 +91,7 @@ protected Searcher.IdFilter getAccessibleIdFilter() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); } @Override @@ -139,7 +139,7 @@ private Consumer getStudyConsumer(QueryScope scope, StudyResultDto.Bu @Nullable @Override protected Properties getAggregationsProperties(List filter) { - Properties properties = getAggregationsProperties(filter, taxonomyService.getStudyTaxonomy()); + Properties properties = getAggregationsProperties(filter, taxonomiesService.getStudyTaxonomy()); if (!properties.containsKey(JOIN_FIELD)) properties.put(JOIN_FIELD, ""); return properties; } diff --git a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java index 6e9a448bf7..150d9ee3f4 100644 --- a/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java +++ b/mica-search/src/main/java/org/obiba/mica/search/queries/VariableQuery.java @@ -14,11 +14,10 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import javax.inject.Inject; import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.dataset.service.HarmonizedDatasetService; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.VariableTaxonomiesService; import org.obiba.mica.micaConfig.service.helper.AggregationMetaDataProvider; import org.obiba.mica.network.domain.Network; import org.obiba.mica.search.DocumentQueryHelper; @@ -40,6 +39,7 @@ import org.springframework.stereotype.Component; import javax.annotation.Nullable; +import javax.inject.Inject; import javax.validation.constraints.NotNull; import java.io.IOException; import java.util.*; @@ -63,8 +63,6 @@ public class VariableQuery extends AbstractDocumentQuery { private static final String LANGUAGE_TAG_UNDETERMINED = "und"; - private final OpalService opalService; - private final PublishedStudyService publishedStudyService; private final Dtos dtos; @@ -91,7 +89,6 @@ public class VariableQuery extends AbstractDocumentQuery { @Inject public VariableQuery( - OpalService opalService, PublishedStudyService publishedStudyService, CollectedDatasetService collectedDatasetService, HarmonizedDatasetService harmonizedDatasetService, @@ -103,7 +100,6 @@ public VariableQuery( PopulationAggregationMetaDataProvider populationAggregationMetaDataProvider, StudyAggregationMetaDataProvider studyAggregationMetaDataProvider, VariablesSetsAggregationMetaDataProvider variablesSetsAggregationMetaDataProvider) { - this.opalService = opalService; this.publishedStudyService = publishedStudyService; this.dtos = dtos; this.datasetAggregationMetaDataProvider = datasetAggregationMetaDataProvider; @@ -320,7 +316,7 @@ protected Properties getAggregationsProperties(List filter) { if (mode != QueryMode.LIST && filter != null && !filter.isEmpty()) { List patterns = filter.stream().map(Pattern::compile).collect(Collectors.toList()); - getOpalTaxonomies().stream().filter(Taxonomy::hasVocabularies) + getVariableTaxonomies().stream().filter(Taxonomy::hasVocabularies) .forEach(taxonomy -> taxonomy.getVocabularies().stream().filter(Vocabulary::hasTerms).forEach(vocabulary -> { String field = vocabulary.getAttributes().containsKey("field") ? vocabulary.getAttributeValue("field") @@ -330,7 +326,7 @@ protected Properties getAggregationsProperties(List filter) { properties.put(field, ""); })); - taxonomyService.getVariableTaxonomy().getVocabularies().forEach(vocabulary -> { + taxonomiesService.getVariableTaxonomy().getVocabularies().forEach(vocabulary -> { String field = vocabulary.getAttributes().containsKey("field") ? vocabulary.getAttributeValue("field") : vocabulary.getName().replace('-', '.'); @@ -357,19 +353,11 @@ protected List getAdditionalAggregationBuckets() { @Override protected Taxonomy getTaxonomy() { - return taxonomyService.getVariableTaxonomy(); + return taxonomiesService.getVariableTaxonomy(); } @NotNull - private List getOpalTaxonomies() { - List taxonomies = null; - - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - - return taxonomies == null ? Collections.emptyList() : taxonomies; + private List getVariableTaxonomies() { + return taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java index 6106a22d22..7d6058f40c 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/TaxonomyIndexer.java @@ -15,9 +15,9 @@ import com.google.common.eventbus.Subscribe; import org.apache.commons.lang3.tuple.ImmutablePair; import org.obiba.mica.micaConfig.event.DeleteTaxonomiesEvent; -import org.obiba.mica.micaConfig.event.OpalTaxonomiesUpdatedEvent; import org.obiba.mica.micaConfig.event.TaxonomiesUpdatedEvent; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.event.VariableTaxonomiesUpdatedEvent; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.opal.core.domain.taxonomy.Taxonomy; @@ -38,20 +38,20 @@ public class TaxonomyIndexer { private static final Logger log = LoggerFactory.getLogger(TaxonomyIndexer.class); @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private Indexer indexer; @Async @Subscribe - public void opalTaxonomiesUpdatedEvent(OpalTaxonomiesUpdatedEvent event) { - log.info("Reindex all opal taxonomies"); + public void variableTaxonomiesUpdatedEvent(VariableTaxonomiesUpdatedEvent event) { + log.info("Reindex all variable taxonomies"); index( TaxonomyTarget.VARIABLE, - event.extractOpalTaxonomies() + event.getTaxonomies() .stream() - .filter(t -> taxonomyService.metaTaxonomyContains(t.getName())) + .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())) .collect(Collectors.toList())); } @@ -73,13 +73,13 @@ public void taxonomiesUpdated(TaxonomiesUpdatedEvent event) { if(indexer.hasIndex(Indexer.TERM_INDEX)) indexer.dropIndex(Indexer.TERM_INDEX); index(TaxonomyTarget.VARIABLE, - ImmutableList.builder().addAll(taxonomyService.getOpalTaxonomies().stream() // - .filter(t -> taxonomyService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) // - .add(taxonomyService.getVariableTaxonomy()) // + ImmutableList.builder().addAll(taxonomiesService.getVariableTaxonomies().stream() + .filter(t -> taxonomiesService.metaTaxonomyContains(t.getName())).collect(Collectors.toList())) + .add(taxonomiesService.getVariableTaxonomy()) .build()); - index(TaxonomyTarget.STUDY, Lists.newArrayList(taxonomyService.getStudyTaxonomy())); - index(TaxonomyTarget.DATASET, Lists.newArrayList(taxonomyService.getDatasetTaxonomy())); - index(TaxonomyTarget.NETWORK, Lists.newArrayList(taxonomyService.getNetworkTaxonomy())); + index(TaxonomyTarget.STUDY, Lists.newArrayList(taxonomiesService.getStudyTaxonomy())); + index(TaxonomyTarget.DATASET, Lists.newArrayList(taxonomiesService.getDatasetTaxonomy())); + index(TaxonomyTarget.NETWORK, Lists.newArrayList(taxonomiesService.getNetworkTaxonomy())); } else { indexer.delete(Indexer.TAXONOMY_INDEX, new String[] {}, ImmutablePair.of("name", event.getTaxonomyName())); Map.Entry termQuery = ImmutablePair.of("taxonomyName", event.getTaxonomyName()); @@ -92,19 +92,19 @@ public void taxonomiesUpdated(TaxonomiesUpdatedEvent event) { switch (taxonomyTarget) { case STUDY: log.info("Study taxonomies were updated"); - taxonomy = taxonomyService.getStudyTaxonomy(); + taxonomy = taxonomiesService.getStudyTaxonomy(); break; case NETWORK: log.info("Network taxonomies were updated"); - taxonomy = taxonomyService.getNetworkTaxonomy(); + taxonomy = taxonomiesService.getNetworkTaxonomy(); break; case DATASET: log.info("Dataset taxonomies were updated"); - taxonomy = taxonomyService.getDatasetTaxonomy(); + taxonomy = taxonomiesService.getDatasetTaxonomy(); break; case VARIABLE: log.info("Variable taxonomies were updated"); - taxonomy = taxonomyService.getVariableTaxonomy(); + taxonomy = taxonomiesService.getVariableTaxonomy(); break; } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java index ce436b2a61..def8f306f6 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/AbstractTaxonomySearchResource.java @@ -17,7 +17,7 @@ import org.obiba.mica.dataset.service.VariableSetService; import org.obiba.mica.micaConfig.service.MicaConfigService; import org.obiba.mica.micaConfig.service.TaxonomyNotFoundException; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.TaxonomyTarget; import org.obiba.mica.taxonomy.EsTaxonomyTermService; import org.obiba.mica.taxonomy.EsTaxonomyVocabularyService; @@ -52,7 +52,7 @@ public class AbstractTaxonomySearchResource { private EsTaxonomyVocabularyService esTaxonomyVocabularyService; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @Inject private MicaConfigService micaConfigService; @@ -63,40 +63,41 @@ public class AbstractTaxonomySearchResource { protected void populate(Opal.TaxonomyDto.Builder tBuilder, Taxonomy taxonomy, Map>> taxoNamesMap, String anonymousUserId) { - taxonomy.getVocabularies().stream().filter(v -> taxoNamesMap.get(taxonomy.getName()).containsKey(v.getName())) - .forEach(voc -> { - Opal.VocabularyDto.Builder vBuilder = Dtos.asDto(voc, false).toBuilder(); - List termNames = taxoNamesMap.get(taxonomy.getName()).get(voc.getName()); - if (voc.hasTerms()) { - List terms = voc.getTerms(); - - // for now, sets are available for variable documents only - if (taxonomy.getName().equals("Mica_" + TaxonomyTarget.VARIABLE.asId()) && voc.getName().equals("sets")) { - List sets = SecurityUtils.getSubject().isAuthenticated() ? - variableSetService.getAllCurrentUser() : - variableSetService.getAllAnonymousUser(anonymousUserId); - List allSetsCurrentUser = sets.stream().map(DocumentSet::getId).collect(Collectors.toList()); - - if (allSetsCurrentUser.size() > 0) { - terms = voc.getTerms().stream().filter(term -> allSetsCurrentUser.contains(term.getName())).collect(Collectors.toList()); - } else { - terms = Lists.newArrayList(); - } + taxonomy.getVocabularies().stream() + .filter(v -> taxoNamesMap.get(taxonomy.getName()).containsKey(v.getName())) + .forEach(voc -> { + Opal.VocabularyDto.Builder vBuilder = Dtos.asDto(voc, false).toBuilder(); + List termNames = taxoNamesMap.get(taxonomy.getName()).get(voc.getName()); + if (voc.hasTerms()) { + List terms = voc.getTerms(); + + // for now, sets are available for variable documents only + if (taxonomy.getName().equals("Mica_" + TaxonomyTarget.VARIABLE.asId()) && voc.getName().equals("sets")) { + List sets = SecurityUtils.getSubject().isAuthenticated() ? + variableSetService.getAllCurrentUser() : + variableSetService.getAllAnonymousUser(anonymousUserId); + List allSetsCurrentUser = sets.stream().map(DocumentSet::getId).collect(Collectors.toList()); + + if (allSetsCurrentUser.size() > 0) { + terms = voc.getTerms().stream().filter(term -> allSetsCurrentUser.contains(term.getName())).collect(Collectors.toList()); + } else { + terms = Lists.newArrayList(); } - - if (termNames.isEmpty()) - vBuilder.addAllTerms(terms.stream().map(Dtos::asDto).collect(Collectors.toList())); - else terms.stream().filter(t -> termNames.contains(t.getName())) - .forEach(term -> vBuilder.addTerms(Dtos.asDto(term))); } - tBuilder.addVocabularies(vBuilder); - }); + + if (termNames.isEmpty()) + vBuilder.addAllTerms(terms.stream().map(Dtos::asDto).collect(Collectors.toList())); + else terms.stream().filter(t -> termNames.contains(t.getName())) + .forEach(term -> vBuilder.addTerms(Dtos.asDto(term))); + } + tBuilder.addVocabularies(vBuilder); + }); } protected List filterVocabularies(TaxonomyTarget target, String query, String locale) { try { - return esTaxonomyVocabularyService.find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), - getFields(locale, VOCABULARY_FIELDS), null, null).getList(); + return esTaxonomyVocabularyService.find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargetedQuery(target, query), + getFields(locale, VOCABULARY_FIELDS), null, null).getList(); } catch (Exception e) { initTaxonomies(); // for a 404 response @@ -109,14 +110,14 @@ protected List filterTerms(TaxonomyTarget target, String query, String l if (vocabularies != null && vocabularies.size() > 0) { // filter on vocabulary names; remove taxonomy prefixes ('Mica_study:') String vocabulariesQuery = vocabularies.stream() - .map(v -> String.format("vocabularyName:%s", v.replaceAll("^([^\\:]+):", ""))) - .collect(Collectors.joining(" AND ")); + .map(v -> String.format("vocabularyName:%s", v.replaceAll("^([^\\:]+):", ""))) + .collect(Collectors.joining(" AND ")); query = Strings.isNullOrEmpty(query) ? vocabulariesQuery : query + " " + vocabulariesQuery; } return esTaxonomyTermService - .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargettedQuery(target, query), getFields(locale, TERM_FIELDS)) - .getList(); + .find(0, MAX_SIZE, DEFAULT_SORT, "asc", null, getTargetedQuery(target, query), getFields(locale, TERM_FIELDS)) + .getList(); } catch (Exception e) { initTaxonomies(); // for a 404 response @@ -127,31 +128,31 @@ protected List filterTerms(TaxonomyTarget target, String query, String l protected List getTaxonomies(TaxonomyTarget target) { switch (target) { case NETWORK: - return Lists.newArrayList(taxonomyService.getNetworkTaxonomy()); + return Lists.newArrayList(taxonomiesService.getNetworkTaxonomy()); case STUDY: - return Lists.newArrayList(taxonomyService.getStudyTaxonomy()); + return Lists.newArrayList(taxonomiesService.getStudyTaxonomy()); case DATASET: - return Lists.newArrayList(taxonomyService.getDatasetTaxonomy()); + return Lists.newArrayList(taxonomiesService.getDatasetTaxonomy()); case TAXONOMY: - return Lists.newArrayList(taxonomyService.getTaxonomyTaxonomy()); + return Lists.newArrayList(taxonomiesService.getTaxonomyTaxonomy()); default: - return taxonomyService.getVariableTaxonomies(); + return taxonomiesService.getAllVariableTaxonomies(); } } protected Taxonomy getTaxonomy(TaxonomyTarget target, String name) { switch (target) { case NETWORK: - return taxonomyService.getNetworkTaxonomy(); + return taxonomiesService.getNetworkTaxonomy(); case STUDY: - return taxonomyService.getStudyTaxonomy(); + return taxonomiesService.getStudyTaxonomy(); case DATASET: - return taxonomyService.getDatasetTaxonomy(); + return taxonomiesService.getDatasetTaxonomy(); case TAXONOMY: - return taxonomyService.getTaxonomyTaxonomy(); + return taxonomiesService.getTaxonomyTaxonomy(); default: - Taxonomy foundTaxonomy = taxonomyService.getVariableTaxonomies().stream().filter(taxonomy -> taxonomy.getName().equals(name)) - .findFirst().orElse(null); + Taxonomy foundTaxonomy = taxonomiesService.getAllVariableTaxonomies().stream().filter(taxonomy -> taxonomy.getName().equals(name)) + .findFirst().orElse(null); if (foundTaxonomy == null) { throw new TaxonomyNotFoundException(name); @@ -169,7 +170,7 @@ protected TaxonomyTarget getTaxonomyTarget(String target) { } } - private String getTargettedQuery(TaxonomyTarget target, String query) { + private String getTargetedQuery(TaxonomyTarget target, String query) { return String.format(Strings.isNullOrEmpty(query) ? "target:%s" : "target:%s AND (%s)", target.name(), query); } @@ -193,6 +194,6 @@ private List getFields(String locale, String... fieldNames) { * Populate taxonomies cache and trigger taxonomies indexing. */ private void initTaxonomies() { - taxonomyService.getOpalTaxonomies(); + taxonomiesService.getVariableTaxonomies(); } } diff --git a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java index f17a782b88..a93d483ac2 100644 --- a/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java +++ b/mica-search/src/main/java/org/obiba/mica/taxonomy/rest/TaxonomiesSearchResource.java @@ -10,6 +10,7 @@ package org.obiba.mica.taxonomy.rest; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -23,10 +24,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; +import com.google.common.collect.Maps; import org.obiba.mica.security.SubjectUtils; import org.obiba.mica.spi.search.TaxonomyTarget; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.taxonomy.TaxonomyResolver; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.domain.taxonomy.TaxonomyEntity; import org.obiba.opal.web.model.Opal; import org.obiba.opal.web.taxonomy.Dtos; import org.slf4j.Logger; @@ -45,45 +49,60 @@ public class TaxonomiesSearchResource extends AbstractTaxonomySearchResource { private static final Logger logger = LoggerFactory.getLogger(TaxonomiesSearchResource.class); @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @GET @Path("/_filter") @Timed public List filter(@Context HttpServletRequest request, @QueryParam("target") @DefaultValue("variable") String target, + @QueryParam("mode") @DefaultValue("search") String mode, @QueryParam("query") String query, @QueryParam("locale") String locale) { TaxonomyTarget taxonomyTarget = getTaxonomyTarget(target); + List taxonomies = getTaxonomies(taxonomyTarget); + + boolean searchMode = "search".equals(mode); + + Map>> taxoNamesMap; + if (searchMode) { + List filteredVocabularies = filterVocabularies(taxonomyTarget, query, locale); + taxoNamesMap = TaxonomyResolver + .asMap(filteredVocabularies, filterTerms(taxonomyTarget, query, locale, filteredVocabularies)); + } else { + taxoNamesMap = Maps.newHashMap(); + for (Taxonomy taxonomy : taxonomies) { + Map> taxoMap = taxonomy.getVocabularies().stream().collect(Collectors.toMap(TaxonomyEntity::getName, vocabulary -> new ArrayList())); + taxoNamesMap.put(taxonomy.getName(), taxoMap); + } + } - List filteredVocabularies = filterVocabularies(taxonomyTarget, query, locale); - - Map>> taxoNamesMap = TaxonomyResolver - .asMap(filteredVocabularies, filterTerms(taxonomyTarget, query, locale, filteredVocabularies)); List results = Lists.newArrayList(); - getTaxonomies(taxonomyTarget).stream().filter(t -> taxoNamesMap.containsKey(t.getName())).forEach(taxo -> { - Opal.TaxonomyDto.Builder tBuilder = Dtos.asDto(taxo, false).toBuilder(); - populate(tBuilder, taxo, taxoNamesMap, SubjectUtils.getAnonymousUserId(request)); - results.add(tBuilder.build()); - }); + taxonomies.stream() + .filter(t -> taxoNamesMap.containsKey(t.getName())) + .forEach(taxo -> { + Opal.TaxonomyDto.Builder tBuilder = Dtos.asDto(taxo, false).toBuilder(); + populate(tBuilder, taxo, taxoNamesMap, SubjectUtils.getAnonymousUserId(request)); + results.add(tBuilder.build()); + }); - return results; + return results; } @GET @Path("/_search") @Timed public List search(@Context HttpServletRequest request, @QueryParam("query") String query, @QueryParam("locale") String locale, - @Nullable @QueryParam("target") String target) { + @Nullable @QueryParam("target") String target) { logger.debug("TaxonomiesSearchResource#search called with query [%s], locale [%s] and target [%s]", query, locale, target); List results = Lists.newArrayList(); List targets = target == null - ? taxonomyService.getTaxonomyTaxonomy().getVocabularies().stream() + ? taxonomiesService.getTaxonomyTaxonomy().getVocabularies().stream() .map(t -> TaxonomyTarget.valueOf(t.getName().toUpperCase())).collect(Collectors.toList()) : Lists.newArrayList(TaxonomyTarget.valueOf(target.toUpperCase())); - targets.forEach(t -> filter(request, t.name(), query, locale).stream() + targets.forEach(t -> filter(request, t.name(), "search", query, locale).stream() .map(taxo -> Opal.TaxonomyBundleDto.newBuilder().setTarget(t.name().toLowerCase()).setTaxonomy(taxo).build()) .forEach(results::add)); return results; diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java b/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java index fe42507de8..c2479efd9a 100644 --- a/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java +++ b/mica-spi/src/main/java/org/obiba/mica/spi/search/SearchEngineService.java @@ -15,7 +15,7 @@ public interface SearchEngineService extends ServicePlugin { /** - * Provides some Mica configurations usefull for the search engine. + * Provides some Mica configurations useful for the search engine. * * @param configurationProvider */ diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java new file mode 100644 index 0000000000..dbeaba2208 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSource.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.mica.web.model.Mica; + +/** + * Helper class for implementing {@link StudyTableSource}. + */ +public abstract class AbstractStudyTableSource implements StudyTableSource { + + private StudyTableContext context; + + @Override + public void setStudyTableContext(StudyTableContext context) { + this.context = context; + } + + protected StudyTableContext getContext() { + return context; + } + + @Override + public boolean providesContingency() { + return false; + } + + @Override + public Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable) { + throw new UnsupportedOperationException("Contingency search not provided by: " + getClass().getSimpleName()); + } + + @Override + public boolean providesVariableSummary() { + return false; + } + + @Override + public Mica.DatasetVariableAggregationDto getVariableSummary(String variableName) { + throw new UnsupportedOperationException("Summary statistics not provided by: " + getClass().getSimpleName()); + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java new file mode 100644 index 0000000000..6cefa5992c --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/AbstractStudyTableSourceService.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import java.util.Properties; + +/** + * Helper class. + */ +public abstract class AbstractStudyTableSourceService implements StudyTableSourceService { + + protected Properties properties; + + protected boolean running; + + @Override + public Properties getProperties() { + return properties; + } + + @Override + public void configure(Properties properties) { + this.properties = properties; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java new file mode 100644 index 0000000000..3d3cce7efb --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/ICategory.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +/** + * Category description, for summary statistics. + */ +public interface ICategory { + + /** + * Get the category name. + * + * @return + */ + String getName(); + + /** + * Get whether this category represents a missing value. + * + * @return + */ + boolean isMissing(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java new file mode 100644 index 0000000000..1ad987469a --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IDataset.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.mica.spi.search.Identified; + +import java.util.Map; + +/** + * Dataset as it is exposed to plugins. + */ +public interface IDataset extends Identified { + + /** + * Get the dataset model. + * + * @return + */ + Map getModel(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java new file mode 100644 index 0000000000..4589de81dd --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IStudy.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.mica.spi.search.Identified; + +import java.util.Map; + +/** + * Study as it is exposed to plugins. + */ +public interface IStudy extends Identified { + + /** + * Get the Opal URL, if any is defined specifically for this study. + * + * @return + */ + String getOpal(); + + /** + * Get the study model. + * + * @return + */ + Map getModel(); +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java new file mode 100644 index 0000000000..c3d4280742 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/IVariable.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import java.util.List; + +/** + * Variable description, for summary statistics. + */ +public interface IVariable { + + /** + * Get the variable name. + * + * @return + */ + String getName(); + + /** + * Get the value type of the variable. + * + * @return + */ + String getValueType(); + + /** + * Whether the variable has categories (may still be categorical without, see boolean value type). + * + * @return + */ + boolean hasCategories(); + + /** + * Helper to get the category names. + * + * @return + */ + List getCategoryNames(); + + /** + * Get a category by its name. + * + * @param name + * @return + */ + ICategory getCategory(String name); +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableFileSourceException.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableFileSourceException.java new file mode 100644 index 0000000000..0f1f67f3d4 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableFileSourceException.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.magma.NoSuchValueTableException; + +public class NoSuchStudyTableFileSourceException extends NoSuchStudyTableSourceException { + + public NoSuchStudyTableFileSourceException(String s) { + super(s); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java new file mode 100644 index 0000000000..3318365424 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/NoSuchStudyTableSourceException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +public class NoSuchStudyTableSourceException extends RuntimeException { + + public NoSuchStudyTableSourceException(String s) { + super(s); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java new file mode 100644 index 0000000000..d8fc3fa841 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableContext.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +public class StudyTableContext { + + private final IDataset dataset; + + private final IStudy study; + + private final int privacyThreshold; + + public StudyTableContext(IDataset dataset, IStudy study, int privacyThreshold) { + this.dataset = dataset; + this.study = study; + this.privacyThreshold = privacyThreshold; + } + + public IDataset getDataset() { + return dataset; + } + + public IStudy getStudy() { + return study; + } + + public int getPrivacyThreshold() { + return privacyThreshold; + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java new file mode 100644 index 0000000000..a3d4703d5b --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileSource.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +/** + * Study table is to be extracted from a file which path applies to the Mica's internal file system. + */ +public interface StudyTableFileSource extends StudyTableSource { + + /** + * The path to the file in Mica's file system, that can be absolute or relative the study's folder considered. + * + * @return + */ + String getPath(); + + /** + * Set the accessor to the input stream that represents the file content. + * + * @param provider + */ + void setStudyTableFileStreamProvider(StudyTableFileStreamProvider provider); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java new file mode 100644 index 0000000000..e8321d73d1 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableFileStreamProvider.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import java.io.InputStream; + +/** + * Define an accessor to a file stream. + */ +public interface StudyTableFileStreamProvider { + + /** + * Get the Mica file input stream on demand. + * + * @return + */ + InputStream getInputStream(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java new file mode 100644 index 0000000000..deece69d60 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSource.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.magma.ValueTable; +import org.obiba.mica.web.model.Mica; + +/** + * Describes the parameters to establish a connection with a Datasource + * and retrieve a ValueTable containing the dataset's variables. + */ +public interface StudyTableSource { + + /** + * Get the {@link ValueTable} implementing the data dictionary and the data values. + * + * @return + */ + ValueTable getValueTable(); + + /** + * Whether crossing variables is supported. + * + * @return + */ + boolean providesContingency(); + + /** + * Make a contingency query and return results. + * + * @param variable + * @param crossVariable + * @return + */ + Mica.DatasetVariableContingencyDto getContingency(IVariable variable, IVariable crossVariable); + + /** + * Whether variable summaries are supported. + * + * @return + */ + boolean providesVariableSummary(); + + /** + * Get a variable summary statistics. + * + * @param variableName + * @return + */ + Mica.DatasetVariableAggregationDto getVariableSummary(String variableName); + + /** + * URN representation of a value table source, indicates the identifier of the value table, in the namespace of the source. + * + * @return + */ + String getURN(); + + /** + * Set context in which the study table is defined. + * + * @param context + */ + void setStudyTableContext(StudyTableContext context); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java new file mode 100644 index 0000000000..f775c248c9 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceService.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import org.obiba.plugins.spi.ServicePlugin; + +/** + * Makes a {@link StudyTableSource} from a URN. + */ +public interface StudyTableSourceService extends ServicePlugin { + + /** + * Whether this service plugin can handle the study table source's URN. + * @param source + * @return + */ + boolean isFor(String source); + + /** + * Make a {@link StudyTableSource} from the source URN. + * + * @param source + * @return + */ + StudyTableSource makeSource(String source); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java new file mode 100644 index 0000000000..e4a358ec62 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/tables/StudyTableSourceServiceLoader.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.tables; + +import com.google.common.collect.Lists; + +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.ServiceLoader; + +/** + * {@link StudyTableSourceService} loader. + */ +public class StudyTableSourceServiceLoader { + + public static Collection get(URLClassLoader classLoader) { + return Lists.newArrayList(ServiceLoader.load(StudyTableSourceService.class, classLoader).iterator()); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java new file mode 100644 index 0000000000..b8aaf31d90 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/AbstractTaxonomiesProviderService.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.opal.core.support.yaml.TaxonomyYaml; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Properties; + +public abstract class AbstractTaxonomiesProviderService implements TaxonomiesProviderService { + + protected Properties properties; + + protected boolean running; + + @Override + public Properties getProperties() { + return properties; + } + + @Override + public void configure(Properties properties) { + this.properties = properties; + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void start() { + this.running = true; + } + + @Override + public void stop() { + this.running = false; + } + + /** + * Read a taxonomy in YAML format from a URL stream. + * + * @param uri + * @return + * @throws TaxonomyImportException + */ + protected Taxonomy readTaxonomy(URL uri) throws TaxonomyImportException { + try (InputStream input = uri.openStream()) { + return readTaxonomy(input); + } catch (Exception e) { + throw new TaxonomyImportException(e); + } + } + + /** + * Read a taxonomy in YAML format from a stream. + * + * @param input + * @return + * @throws TaxonomyImportException + */ + protected Taxonomy readTaxonomy(InputStream input) { + TaxonomyYaml yaml = new TaxonomyYaml(); + return yaml.load(input); + } +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java new file mode 100644 index 0000000000..f75ce0450a --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderService.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import org.obiba.mica.spi.search.TaxonomyTarget; +import org.obiba.opal.core.domain.taxonomy.Taxonomy; +import org.obiba.plugins.spi.ServicePlugin; + +import java.util.List; + +public interface TaxonomiesProviderService extends ServicePlugin { + + boolean isFor(TaxonomyTarget target); + + List getTaxonomies(); + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java new file mode 100644 index 0000000000..3766940303 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomiesProviderServiceLoader.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +import com.google.common.collect.Lists; + +import java.net.URLClassLoader; +import java.util.Collection; +import java.util.ServiceLoader; + +/** + * {@link TaxonomiesProviderService} loader. + */ +public class TaxonomiesProviderServiceLoader { + + public static Collection get(URLClassLoader classLoader) { + return Lists.newArrayList(ServiceLoader.load(TaxonomiesProviderService.class, classLoader).iterator()); + } + +} diff --git a/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java new file mode 100644 index 0000000000..84d15c5c31 --- /dev/null +++ b/mica-spi/src/main/java/org/obiba/mica/spi/taxonomies/TaxonomyImportException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 OBiBa. All rights reserved. + * + * This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.obiba.mica.spi.taxonomies; + +public class TaxonomyImportException extends RuntimeException { + + public TaxonomyImportException(Throwable error) { + super(error); + } + +} diff --git a/mica-web-model/src/main/protobuf/Mica.proto b/mica-web-model/src/main/protobuf/Mica.proto index 4bcd968704..5bb2d5bed3 100644 --- a/mica-web-model/src/main/protobuf/Mica.proto +++ b/mica-web-model/src/main/protobuf/Mica.proto @@ -409,7 +409,7 @@ message MicaConfigDto { repeated string languages = 2; required string defaultCharSet = 3; optional string publicUrl = 4; - required string opal = 5; + optional string opal = 5; optional string version = 6; optional int32 privacyThreshold = 7; repeated string roles = 8; @@ -542,8 +542,8 @@ message DatasetDto { extensions 1000 to max; message StudyTableDto { - required string project = 1; - required string table = 2; + optional string project = 1; + optional string table = 2; repeated LocalizedStringDto name = 3; repeated LocalizedStringDto description = 4; @@ -554,11 +554,12 @@ message DatasetDto { optional StudySummaryDto studySummary = 9; optional int32 weight = 10; repeated LocalizedStringDto additionalInformation = 11; + optional string source = 12; } message HarmonizationTableDto { - required string project = 1; - required string table = 2; + optional string project = 1; + optional string table = 2; repeated LocalizedStringDto name = 3; repeated LocalizedStringDto description = 4; @@ -566,6 +567,7 @@ message DatasetDto { optional StudySummaryDto studySummary = 7; optional int32 weight = 8; repeated LocalizedStringDto additionalInformation = 9; + optional string source = 10; } } @@ -665,6 +667,7 @@ message DatasetVariableResolverDto { repeated LocalizedStringDto populationName = 29; repeated LocalizedStringDto dceName = 30; optional string entityType = 31; + optional string source = 32; } message DatasetVariableSummaryDto { diff --git a/mica-web-model/src/main/protobuf/MicaPlugins.proto b/mica-web-model/src/main/protobuf/MicaPlugins.proto new file mode 100644 index 0000000000..2afd7487fb --- /dev/null +++ b/mica-web-model/src/main/protobuf/MicaPlugins.proto @@ -0,0 +1,49 @@ +package obiba.mica; +option java_package = "org.obiba.mica.web.model"; + +message PluginPackageDto { + required string name = 1; + required string title = 2; + required string description = 3; + required string author = 4; + required string maintainer = 5; + required string license = 6; + optional string website = 7; + required string version = 8; + required string micaVersion = 9; + required string type = 10; + optional string file = 11; + optional bool uninstalled = 12; + extensions 1000 to max; +} + +message PluginPackagesDto { + required string site = 1; + optional string updated = 2; + required bool restart = 3; + repeated PluginPackageDto packages = 4; +} + +message PluginDto { + required string name = 1; + required string title = 2; + required string description = 3; + required string author = 4; + required string maintainer = 5; + required string license = 6; + optional string website = 7; + required string version = 8; + required string micaVersion = 9; + required string type = 10; + required string siteProperties = 11; +} + +enum ServicePluginStatus { + RUNNING = 1; + STOPPED = 2; +} + +message ServicePluginDto { + required string name = 1; + required ServicePluginStatus status = 2; +} diff --git a/mica-webapp/bower.json b/mica-webapp/bower.json index 87b4d7d9d4..eeaccb1177 100644 --- a/mica-webapp/bower.json +++ b/mica-webapp/bower.json @@ -30,7 +30,7 @@ "mica-study-timeline": "https://github.com/obiba/mica-study-timeline.git#1.1.4", "ng-file-upload": "12.2.13", "ng-obiba": "https://github.com/obiba/ng-obiba.git#2.0.2", - "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#4.3.0", + "ng-obiba-mica": "https://github.com/obiba/ng-obiba-mica.git#4.4.0", "filesize": "3.6.0", "ngclipboard": "1.1.3", "angular-utils-pagination": "0.11.1", diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java index 0e682deca0..9baeca25bb 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/DatasetAnalysisController.java @@ -1,13 +1,8 @@ package org.obiba.mica.web.controller; -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; -import javax.servlet.http.HttpServletRequest; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.googlecode.protobuf.format.JsonFormat; import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.Subject; import org.obiba.magma.NoSuchVariableException; @@ -16,7 +11,7 @@ import org.obiba.mica.dataset.domain.DatasetVariable; import org.obiba.mica.dataset.domain.StudyDataset; import org.obiba.mica.dataset.service.PublishedDatasetService; -import org.obiba.mica.micaConfig.service.OpalService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.security.service.SubjectAclService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; @@ -30,9 +25,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.ModelAndView; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; -import com.googlecode.protobuf.format.JsonFormat; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; @Controller public class DatasetAnalysisController extends BaseController { @@ -49,7 +46,7 @@ public class DatasetAnalysisController extends BaseController { private Dtos dtos; @Inject - private OpalService opalService; + private TaxonomiesService taxonomiesService; @Inject private PublishedDatasetService publishedDatasetService; @@ -133,17 +130,11 @@ private Dataset getDataset(String id) { return dataset; } - private List getTaxonomies() { - List taxonomies = null; - try { - taxonomies = opalService.getTaxonomies(); - } catch (Exception e) { - // ignore - } - return taxonomies == null ? Collections.emptyList() : taxonomies; + private List getVariableTaxonomies() { + return taxonomiesService.getVariableTaxonomies(); } private String getDatasetVariableJSON(DatasetVariable variable) { - return JsonFormat.printToString(dtos.asDto(variable, getTaxonomies())); + return JsonFormat.printToString(dtos.asDto(variable, getVariableTaxonomies())); } } diff --git a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java index 3c7805e9f0..9c54a1056a 100644 --- a/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java +++ b/mica-webapp/src/main/java/org/obiba/mica/web/controller/VariableController.java @@ -10,7 +10,7 @@ import org.obiba.mica.dataset.domain.HarmonizationDataset; import org.obiba.mica.dataset.service.CollectedDatasetService; import org.obiba.mica.dataset.service.HarmonizedDatasetService; -import org.obiba.mica.micaConfig.service.TaxonomyService; +import org.obiba.mica.micaConfig.service.TaxonomiesService; import org.obiba.mica.spi.search.Indexer; import org.obiba.mica.spi.search.Searcher; import org.obiba.mica.study.NoSuchStudyException; @@ -63,13 +63,15 @@ public class VariableController extends BaseController { private ObjectMapper objectMapper; @Inject - private TaxonomyService taxonomyService; + private TaxonomiesService taxonomiesService; @GetMapping("/variable/{id:.+}") public ModelAndView variable(@PathVariable String id) { Map params = newParameters(); - DatasetVariable.IdResolver resolver = DatasetVariable.IdResolver.from(id); + String decodedId = DatasetVariable.IdEncoderDecoder.decode(id); + + DatasetVariable.IdResolver resolver = DatasetVariable.IdResolver.from(decodedId); String datasetId = resolver.getDatasetId(); String variableName = resolver.getName(); @@ -83,13 +85,23 @@ public ModelAndView variable(@PathVariable String id) { break; } - DatasetVariable variable = resolver.getType().equals(DatasetVariable.Type.Harmonized) ? getHarmonizedDatasetVariable(resolver.getDatasetId(), id, variableName) : getDatasetVariable(id, variableName); + DatasetVariable variable; + if (resolver.getType().equals(DatasetVariable.Type.Harmonized)) { + try { + variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), resolver.getId(), variableName); + } catch (NoSuchVariableException e) { + // legacy variable id format + variable = getHarmonizedDatasetVariable(resolver.getDatasetId(), decodedId, variableName); + } + } else + variable = getDatasetVariable(resolver.getId(), variableName); + params.put("variable", variable); params.put("type", resolver.getType().toString()); addStudyTableParameters(params, variable); - Map taxonomies = taxonomyService.getVariableTaxonomies().stream() + Map taxonomies = taxonomiesService.getAllVariableTaxonomies().stream() .collect(Collectors.toMap(TaxonomyEntity::getName, e -> e)); // annotations are attributes described by some taxonomies @@ -163,10 +175,11 @@ private DatasetVariable getDatasetVariableInternal(String indexName, String inde } private DatasetVariable getHarmonizedDatasetVariable(String datasetId, String variableId, String variableName) { - String dataSchemaVariableId = DatasetVariable.IdResolver - .encode(datasetId, variableName, DatasetVariable.Type.Dataschema, null, null, null, null); DatasetVariable harmonizedDatasetVariable = getDatasetVariableInternal(Indexer.PUBLISHED_HVARIABLE_INDEX, Indexer.HARMONIZED_VARIABLE_TYPE, variableId, variableName); + + String dataSchemaVariableId = DatasetVariable.IdResolver + .encode(datasetId, variableName, DatasetVariable.Type.Dataschema); DatasetVariable dataSchemaVariable = getDatasetVariableInternal(Indexer.PUBLISHED_VARIABLE_INDEX, Indexer.VARIABLE_TYPE, dataSchemaVariableId, variableName); @@ -208,13 +221,13 @@ private void addStudyTableParameters(Map params, DatasetVariable if (DatasetVariable.OpalTableType.Study.equals(variable.getOpalTableType())) { Optional studyTable = dataset.getStudyTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getProject().equals(st.getProject()) && variable.getTable().equals(st.getTable())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSource().equals(st.getSource())) .findFirst(); if (studyTable.isPresent()) params.put("opalTable", studyTable.get()); } else { Optional harmoStudyTable = dataset.getHarmonizationTables().stream().filter(st -> - variable.getStudyId().equals(st.getStudyId()) && variable.getProject().equals(st.getProject()) && variable.getTable().equals(st.getTable())) + variable.getStudyId().equals(st.getStudyId()) && variable.getSource().equals(st.getSource())) .findFirst(); if (harmoStudyTable.isPresent()) params.put("opalTable", harmoStudyTable.get()); diff --git a/mica-webapp/src/main/resources/_templates/dataset.ftl b/mica-webapp/src/main/resources/_templates/dataset.ftl index 257235f70f..c6a6b00210 100644 --- a/mica-webapp/src/main/resources/_templates/dataset.ftl +++ b/mica-webapp/src/main/resources/_templates/dataset.ftl @@ -118,7 +118,7 @@ ${initiativeIcon}<#else>${studyIcon}"> ${localize(study.acronym)} - <#if showDatasetContingencyLink> + <#if showVariableStatistics && showDatasetContingencyLink> <@message "dataset.crosstab.title"/> diff --git a/mica-webapp/src/main/resources/_templates/libs/settings.ftl b/mica-webapp/src/main/resources/_templates/libs/settings.ftl index fb287edd96..1c8188ec92 100644 --- a/mica-webapp/src/main/resources/_templates/libs/settings.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/settings.ftl @@ -140,6 +140,7 @@ <#assign searchCriteriaMenus = ["variable", "dataset", "study", "network"]/> +<#assign showVariableStatistics = true/> <#assign showHarmonizedVariableSummarySelector = true/> diff --git a/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl b/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl index 15b1e381d4..54532e836a 100644 --- a/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl +++ b/mica-webapp/src/main/resources/_templates/libs/variable-scripts.ftl @@ -119,7 +119,7 @@ }); - <#if user?? || !config.variableSummaryRequiresAuthentication> + <#if showVariableStatistics && (user?? || !config.variableSummaryRequiresAuthentication)> makeSummary(${showHarmonizedVariableSummarySelector?c}); diff --git a/mica-webapp/src/main/resources/_templates/variable.ftl b/mica-webapp/src/main/resources/_templates/variable.ftl index d27c252a3a..11382f1708 100644 --- a/mica-webapp/src/main/resources/_templates/variable.ftl +++ b/mica-webapp/src/main/resources/_templates/variable.ftl @@ -289,34 +289,36 @@ -
-
-
-
-

<@message "summary-statistics"/>

- <#if showDatasetContingencyLink> - <#if variable.nature == "CATEGORICAL"> - - <@message "dataset.crosstab.title"/> - - <#elseif variable.nature == "CONTINUOUS"> - - <@message "dataset.crosstab.title"/> - + <#if showVariableStatistics> +
+
+
+
+

<@message "summary-statistics"/>

+ <#if showDatasetContingencyLink> + <#if variable.nature == "CATEGORICAL"> + + <@message "dataset.crosstab.title"/> + + <#elseif variable.nature == "CONTINUOUS"> + + <@message "dataset.crosstab.title"/> + + - -
-
- <#if user?? || !config.variableSummaryRequiresAuthentication> - <@variableSummary variable=variable/> - <#else> - <@message "sign-in-for-variable-statistics"/> - <@message "sign-in"/> - +
+
+ <#if user?? || !config.variableSummaryRequiresAuthentication> + <@variableSummary variable=variable/> + <#else> + <@message "sign-in-for-variable-statistics"/> + <@message "sign-in"/> + +
-
+ <#if type == "Dataschema">
diff --git a/mica-webapp/src/main/resources/i18n/en.json b/mica-webapp/src/main/resources/i18n/en.json index f0c57295eb..61508c24e6 100644 --- a/mica-webapp/src/main/resources/i18n/en.json +++ b/mica-webapp/src/main/resources/i18n/en.json @@ -59,6 +59,7 @@ "update-comment": "Add a comment to describe the changes.", "copied": "Copied", "copy-to-clipboard": "Copy to clipboard", + "copy-path-to-clipboard": "Copy file path to clipboard", "copy-query": "Copy query", "name": "Name", "default": "Default", @@ -229,8 +230,8 @@ "all-help": "All caches.", "clear": "Clear", "run": "Run", - "opal-taxonomies": "Opal taxonomies", - "opal-taxonomies-help": "Taxonomies from primary Opal server. To be cleared after taxonomies have been updated in Opal.", + "variable-taxonomies": "Variable taxonomies", + "variable-taxonomies-help": "Taxonomies from the primary Opal server (if enabled) and other sources (plugins). To be cleared after taxonomies have been updated.", "mica-config": "Mica configuration", "mica-config-help": "Mica system various settings. Cache is automatically cleared each time the configuration is updated.", "aggregations-metadata": "Aggregations metadata", @@ -558,12 +559,12 @@ "help": "The complete translations will be exported in the Gettext format." } }, - "variables-count-enabled": "Variables statistics enabled", - "variables-count-enabled-help": "Show variables statistics summary in the administration section.", - "projects-count-enabled": "Projects statistics enabled", - "projects-count-enabled-help": "Show projects statistics summary in the administration section.", - "data-access-requests-count-enabled": "Data Access Requests statistics enabled", - "data-access-requests-count-enabled-help": "Show data access requests statistics summary in the administration section." + "variables-count-enabled": "Variables metrics enabled", + "variables-count-enabled-help": "Show variables counts in the administration section and the home page.", + "projects-count-enabled": "Projects metrics enabled", + "projects-count-enabled-help": "Show projects counts in the administration section.", + "data-access-requests-count-enabled": "Data Access Requests metrics enabled", + "data-access-requests-count-enabled-help": "Show data access requests counts in the administration section." }, "publish": { "label": "Publish", @@ -955,7 +956,25 @@ }, "datasource": { "title": "Data Source", - "info": "Reference to the Opal table from which variables and data summaries will be extracted from." + "info": "Source of the data dictionary and summary statistics.", + "info-schema": "Source of the data dictionary.", + "opal": { + "title": "Opal Source", + "info": "Reference to the Opal table from which variables and data summaries will be extracted from.", + "info-schema": "Reference to the Opal table from which variables will be extracted from." + }, + "file": { + "title": "File Source", + "info": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the dataset's folder, then in the associated study's folder. The table name is optional if there is only one described in the file.", + "info-schema": "Path to the file from which table's variables will be extracted. If the path is relative, it is first looked up in the harmonization protocol's folder, then in the associated harmonization initiative's folder. The table name is optional if there is only one described in the file." + }, + "other": { + "title": "Other Source", + "info": "Generic study table source definition.", + "info-schema": "Generic data schema source definition.", + "nid": "Namespace identifier", + "nss": "Specific source name in the namespace" + } }, "delete-dialog": { "title": "Delete Dataset", @@ -976,7 +995,12 @@ }, "study-table": { "title": "Study Table", - "info": "A study table defines which study data collection event it applies to and which Opal table the variables and data summaries are to be extracted from." + "info": "A study table defines which study data collection event it applies to and which Opal table the variables and data summaries are to be extracted from.", + "source": { + "opal": "Opal source", + "file": "File source", + "other": "Other source" + } }, "harmonization-study-table": { "title": "Harmonization Initiative Table", @@ -1006,7 +1030,7 @@ "table-additional-information": "Additional Information", "data-schema": { "title": "Harmonization Protocol", - "info": "The variables of the harmonization protocol are the reference variables. This is a simple data dictionary: no data summaries will be extracted from the Opal table." + "info": "The variables of the harmonization protocol are the reference variables. This is a simple data dictionary: no data summary statistics will be extracted." }, "index": "Index", "add-study-table": "Add Table", @@ -1465,7 +1489,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Navigate the classification system and select the search criteria." + "help": "Navigate the classification system and manage order and visibility of the search criteria." }, "all-null-classifications": "", "all-variable-classifications": "All variable classifications", @@ -1523,6 +1547,10 @@ "message": "Are you sure you want to delete the access of {{arg0}} {{arg1}}" } }, + "up": "Up", + "down": "Down", + "hide": "Hide", + "show": "Show", "draft": "Draft", "published": "Published", "role": "Role", @@ -1575,7 +1603,7 @@ "security": "Security", "system": "System", "content": "Content", - "data-management": "Data Management", + "search-management": "Search", "data-access": "Data Access", "research": "Research", "properties": "Properties", @@ -1613,7 +1641,7 @@ "project-config": "Configure Research Project form.", "file-system": "Manage file system.", "search": "Search in the published repository.", - "classifications": "Browse taxonomies.", + "classifications": "Manage the taxonomies that are used as search criteria.", "translations": "Manage labels translations.", "cart": "Cart of variables.", "sets": "Lists of variables.", @@ -2562,7 +2590,7 @@ }, "entity-statistics-summary": { "title": "Statistics Summary", - "full-title": "Content Statistics Summary", + "full-title": "Content Metrics Summary", "Study": "Individual Studies", "HarmonizationStudy": "Harmonization Initiatives", "StudyDataset": "Collected Datasets", diff --git a/mica-webapp/src/main/resources/i18n/fr.json b/mica-webapp/src/main/resources/i18n/fr.json index afa8787016..aba446266b 100644 --- a/mica-webapp/src/main/resources/i18n/fr.json +++ b/mica-webapp/src/main/resources/i18n/fr.json @@ -59,6 +59,7 @@ "update-comment": "Ajoutez un commentaire pour décrire les modifications.", "copied": "Copié", "copy-to-clipboard": "Copier dans le presse-papier", + "copy-path-to-clipboard": "Copier le chemin du fichier dans le presse-papier", "copy-query": "Copier la requête", "name": "Nom", "default": "Par défaut", @@ -229,8 +230,8 @@ "all-help": "Tous les caches.", "clear": "Effacer", "run": "Exécuter", - "opal-taxonomies": "Taxonomies de Opal", - "opal-taxonomies-help": "Taxonomies du serveur Opal primaire. Doit être effacé après que les taxonomies aient été mises à jour dans Opal.", + "variable-taxonomies": "Taxonomies de variables", + "variable-taxonomies-help": "Taxonomies du serveur Opal primaire (si activé) ainsi que d'autres sources (plugins). Doit être effacé après que les taxonomies aient été mises à jour.", "mica-config": "Configuration de Mica", "mica-config-help": "Divers paramètres de Mica. Ce cache est automatiquement effacé chaque fois que la configuration est mise à jour.", "aggregations-metadata": "Aggregations des meta-données", @@ -558,12 +559,12 @@ "help": "Les traductions complètes seront exportées au format Gettext." } }, - "variables-count-enabled": "Statistiques des variables activées", - "variables-count-enabled-help": "Afficher le résumé des statistiques des variables dans la section d'administration.", - "projects-count-enabled": "Statistiques des projets activées", - "projects-count-enabled-help": "Afficher le résumé des statistiques des projets dans la section d'administration.", - "data-access-requests-count-enabled": "Statistiques des demandes d'accès aux données activées", - "data-access-requests-count-enabled-help": "Afficher le résumé des statistiques des demandes d'accès aux données dans la section d'administration." + "variables-count-enabled": "Métriques des variables activées", + "variables-count-enabled-help": "Afficher les nombres de variables dans la section d'administration.", + "projects-count-enabled": "Métriques des projets activées", + "projects-count-enabled-help": "Afficher les nombres de projets dans la section d'administration.", + "data-access-requests-count-enabled": "Métriques de demandes d'accès aux données activées", + "data-access-requests-count-enabled-help": "Afficher les nombres de demandes d'accès aux données dans la section d'administration." }, "publish": { "label": "Publier", @@ -955,7 +956,25 @@ }, "datasource": { "title": "Source de données", - "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites." + "info": "Source du dictionnaire de données et des statistiques sommaires.", + "info-schema": "Source du dictionnaire de données.", + "opal": { + "title": "Source Opal", + "info": "Référence vers la table Opal de laquelle les variables et les données sommaires seront extraites.", + "info-schema": "Référence vers la table Opal de laquelle les variables seront extraites." + }, + "file": { + "title": "Fichier source", + "info": "Fichier duquel les variables de la table seront extraites. Si le chemin est relatif, le fichier est d'abord cherché dans le dossier de l'ensemble de données, puis dans celui de l'étude associée. Le nom de la table est optionel si le fichier ne décrit qu'une table.", + "info-schema": "Fichier duquel les variables de la table seront extraites. Le chemin du fichier peut-être relatif, le fichier est d'abord cherché dans le dossier du protocole d'harmonisation, puis dans celui de l'initiative d'harmonisation associée. Le nom de la table est optionel si le fichier ne décrit qu'une table." + }, + "other": { + "title": "Autre source", + "info": "Définition générique de la source de la table.", + "info-schema": "Définition générique de la source du schéma de données.", + "nid": "Identifiant de l'espace de nommage", + "nss": "Nom spécifique de la source dans l'espace de nommage" + } }, "delete-dialog": { "title": "Supprimer l'ensemble de données", @@ -976,7 +995,12 @@ }, "study-table": { "title": "Table d'étude", - "info": "La table d'étude définie à quel événement de collecte elle s'applique et à partir de quelle table Opal il faut extraire les variables et les données sommaires." + "info": "La table d'étude définie à quel événement de collecte elle s'applique et à partir de quelle table Opal il faut extraire les variables et les données sommaires.", + "source": { + "opal": "Opal source", + "file": "Fichier source", + "other": "Autre source" + } }, "harmonization-study-table": { "title": "Table d'initiative d'harmonisation", @@ -1006,7 +1030,7 @@ "table-additional-information": "Information additionnelle", "data-schema": { "title": "Protocole d'harmonisation", - "info": "Les variables du protocole d'harmonisation sont les variables de référence. Il s'agit d'un simple dictionnaire de données: aucun résumé de données ne sera extrait de la table Opal." + "info": "Les variables du protocole d'harmonisation sont les variables de référence. Il s'agit d'un simple dictionnaire de données: aucune statistique sommaires ne seront extraites." }, "index": "Index", "add-study-table": "Ajouter une table d'étude", @@ -1465,7 +1489,7 @@ "variables": "Variables", "classifications": { "title": "Classifications", - "help": "Naviguer dans le système de classification et sélectionner les critères de recherche." + "help": "Naviguer dans le système de classification et gérer l'ordre et la visibilité des critères de recherche." }, "all-null-classifications": "", "all-variable-classifications": "Toutes les classifications de variables", @@ -1523,6 +1547,10 @@ "message": "Êtes-vous sûr de vouloir supprimer l'accès de {{arg0}} {{arg1}}" } }, + "up": "Monter", + "down": "Descendre", + "hide": "Cacher", + "show": "Montrer", "draft": "Brouillon", "published": "Publié", "role": "Rôle", @@ -1575,7 +1603,7 @@ "security": "Sécurité", "system": "Système", "content": "Contenu", - "data-management": "Gestion de données", + "search-management": "Recherche", "data-access": "Accès aux données", "research": "Recherche", "properties": "Propriétés", @@ -1613,7 +1641,7 @@ "project-config": "Configurer le formulaire de projet de recherche.", "file-system": "Gérer le système de fichiers.", "search": "Chercher dans le catalogue publié.", - "classifications": "Parcourir les taxonomies.", + "classifications": "Gérer les taxonomies servant à bâtir les critères de recherche.", "translations": "Gérer les traductions des libellés.", "cart": "Panier de variables.", "sets": "Listes de variables.", @@ -2562,7 +2590,7 @@ }, "entity-statistics-summary": { "title": "Résumé des statistiques", - "full-title": "Résumé des statistiques de contenu", + "full-title": "Métriques du contenu", "Study": "Étude individuelles", "HarmonizationStudy": "Initiatives d'harmonisation", "StudyDataset": "Ensemble de données de collecte", diff --git a/mica-webapp/src/main/webapp/app/admin/admin-controller.js b/mica-webapp/src/main/webapp/app/admin/admin-controller.js index 376162ebdf..aa0cea2f64 100644 --- a/mica-webapp/src/main/webapp/app/admin/admin-controller.js +++ b/mica-webapp/src/main/webapp/app/admin/admin-controller.js @@ -152,9 +152,9 @@ mica.admin }); }; - $scope.clearOpalTaxonomies = function () { + $scope.clearVariableTaxonomies = function () { withConfirm(function () { - CacheService.cache.clear({id: 'opalTaxonomies'}); + CacheService.cache.clear({id: 'variableTaxonomies'}); }); }; @@ -261,4 +261,105 @@ mica.admin }); }; + }]) + + .controller('ClassificationsController', ['$timeout', + '$scope', + '$location', + '$translate', + function ($timeout, $scope, + $location, + $translate) { + + //$scope.options = options; + + $scope.taxonomyTypeMap = { //backwards compatibility for pluralized naming in configs. + variable: 'variables', + study: 'studies', + network: 'networks', + dataset: 'datasets' + }; + + $translate(['search.classifications-title', 'search.classifications-link', 'search.faceted-navigation-help']) + .then(function (translation) { + $scope.hasClassificationsTitle = translation['search.classifications-title']; + $scope.hasClassificationsLinkLabel = translation['search.classifications-link']; + $scope.hasFacetedNavigationHelp = translation['search.faceted-navigation-help']; + }); + + var searchTaxonomyDisplay = { + variable: true, //$scope.options.variables.showSearchTab, + dataset: true, //$scope.options.datasets.showSearchTab, + study: true, //$scope.options.studies.showSearchTab, + network: true, //$scope.options.networks.showSearchTab + }; + + $scope.lang = $translate.use(); + + function initSearchTabs() { + function getTabsOrderParam(arg) { + var value = $location.search()[arg]; + + return value && value.split(',') + .filter(function (t) { + return t; + }) + .map(function (t) { + return t.trim(); + }); + } + + const defaultTargetTabsOrderParam = ['variable', 'dataset', 'study', 'network']; + var targetTabsOrderParam = getTabsOrderParam('targetTabsOrder'); + $scope.targetTabsOrder = (targetTabsOrderParam || defaultTargetTabsOrderParam).filter(function (t) { + return searchTaxonomyDisplay[t]; + }); + + if ($location.search().target) { + $scope.target = $location.search().target; + } else if (!$scope.target) { + $scope.target = $scope.targetTabsOrder[0]; + } + } + + var onSelectTerm = function (target, taxonomy, vocabulary, args) { + args = args || {}; + + if (args.text) { + args.text = args.text.replace(/[^a-zA-Z0-9*" _-]/g, ''); + } + + if (angular.isString(args)) { + args = { term: args }; + } + + console.log('onSelectTerm'); + }; + + $scope.navigateToTarget = function (target) { + $location.search('target', target); + $location.search('taxonomy', null); + $location.search('vocabulary', null); + $scope.target = target; + }; + + $scope.onSelectTerm = onSelectTerm; + + $scope.toggleFullscreen = function (fullscreen) { + if ($scope.isFullscreen && $scope.isFullscreen !== fullscreen) { + // in case the ESC key was pressed + $timeout(function() {$scope.isFullscreen = fullscreen;}); + } else { + $scope.isFullscreen = fullscreen; + } + }; + + $scope.isFullscreen = false; + + function init() { + $scope.lang = $translate.use(); + initSearchTabs(); + } + + init(); }]); diff --git a/mica-webapp/src/main/webapp/app/admin/admin-router.js b/mica-webapp/src/main/webapp/app/admin/admin-router.js index 3c46db896b..df97acab2c 100644 --- a/mica-webapp/src/main/webapp/app/admin/admin-router.js +++ b/mica-webapp/src/main/webapp/app/admin/admin-router.js @@ -36,6 +36,13 @@ mica.admin authorizedRoles: ['mica-administrator'] } }) + .when('/admin/classifications', { + templateUrl: 'app/admin/views/classifications.html', + controller: 'ClassificationsController', + access: { + authorizedRoles: ['mica-administrator'] + } + }) .when('/admin/indexing', { templateUrl: 'app/admin/views/indexing.html', controller: 'IndexingController', diff --git a/mica-webapp/src/main/webapp/app/admin/views/admin-view.html b/mica-webapp/src/main/webapp/app/admin/views/admin-view.html index c8eb812fdb..1d67cc29bf 100644 --- a/mica-webapp/src/main/webapp/app/admin/views/admin-view.html +++ b/mica-webapp/src/main/webapp/app/admin/views/admin-view.html @@ -49,18 +49,12 @@

admin-page.help.caching

-
  • - {{'indexing.title' | translate}} -

    - admin-page.help.indexing -

    -
    • -
  • +
  • {{'logs.title' | translate}} @@ -128,24 +122,20 @@

      • -
    • data-management
      • +
    • search-management
      • -
    • - - {{'search.title' | translate}} +
    • + + {{'classifications.title' | translate}}

      - admin-page.help.search + admin-page.help.classifications

    • - - {{'classifications.title' | translate}} - + {{'indexing.title' | translate}}

      - admin-page.help.classifications + admin-page.help.indexing

      • diff --git a/mica-webapp/src/main/webapp/app/admin/views/caching.html b/mica-webapp/src/main/webapp/app/admin/views/caching.html index cbef466389..f983acddb7 100644 --- a/mica-webapp/src/main/webapp/app/admin/views/caching.html +++ b/mica-webapp/src/main/webapp/app/admin/views/caching.html @@ -47,11 +47,11 @@

      - caching.opal-taxonomies - caching.opal-taxonomies-help + caching.variable-taxonomies + caching.variable-taxonomies-help - + diff --git a/mica-webapp/src/main/webapp/app/admin/views/classifications.html b/mica-webapp/src/main/webapp/app/admin/views/classifications.html new file mode 100644 index 0000000000..6cd08ab801 --- /dev/null +++ b/mica-webapp/src/main/webapp/app/admin/views/classifications.html @@ -0,0 +1,37 @@ + + +
      + +

      +
        +
      1. global.menu.admin
        2. +
      2. classifications.title
        3. +
      +

      +

      classifications.help

      + + + + + +
      + + + +
      + +
      diff --git a/mica-webapp/src/main/webapp/app/config/views/config-form.html b/mica-webapp/src/main/webapp/app/config/views/config-form.html index 9e6d732e1b..0dad8b8d39 100644 --- a/mica-webapp/src/main/webapp/app/config/views/config-form.html +++ b/mica-webapp/src/main/webapp/app/config/views/config-form.html @@ -39,8 +39,7 @@

      config.serverIdentif

      config.dataSource

      -
      +
      diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js index 410224a71e..af430f2d90 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-controller.js @@ -37,7 +37,7 @@ mica.dataset 'MicaConfigResource', 'FormServerValidation', 'StudyStatesResource', - 'StudyStateProjectsResource', + 'StudyStateOpalProjectsResource', 'FormDirtyStateObserver', 'EntityFormResource', 'SfOptionsService', @@ -57,7 +57,7 @@ mica.dataset MicaConfigResource, FormServerValidation, StudyStatesResource, - StudyStateProjectsResource, + StudyStateOpalProjectsResource, FormDirtyStateObserver, EntityFormResource, SfOptionsService, @@ -175,20 +175,26 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise.then(function (projects) { - $scope.projects = projects; - var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === $scope.studyTable.project; })[0]; + StudyStateOpalProjectsResource.query({id: $scope.selected.study.id}).$promise + .then(function (projects) { + $scope.projects = projects; + var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === $scope.studyTable.project; })[0]; - if (selectedProject) { - $scope.selected.project = selectedProject; + if (selectedProject) { + $scope.selected.project = selectedProject; - selectedTable = selectedProject.datasource.table.filter(function (t) {return t === $scope.studyTable.table; })[0]; + selectedTable = selectedProject.datasource.table.filter(function (t) {return t === $scope.studyTable.table; })[0]; - if (selectedTable) { - $scope.selected.project.table = selectedTable; + if (selectedTable) { + $scope.selected.project.table = selectedTable; + } } - } - }); + }) + .catch(function (err) { + console.log(err); + $scope.projects = []; + $scope.selected.project = undefined; + }); } }); @@ -317,20 +323,33 @@ mica.dataset return p.name === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project; }).pop(); - if ($scope.selected.project) { - $scope.selected.project.table = $scope.selected.project.datasource.table.filter(function (t) { - return t === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table; - }).pop(); + if (!$scope.selected.project) { + $scope.selected.project = $scope.projects.pop(); + } + + $scope.selected.project.table = $scope.selected.project.datasource.table.filter(function (t) { + return t === dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table; + }).pop(); + + if (!$scope.selected.project.table) { + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); } }); }); } else { - getOpalProjects(); $scope.dataset = { published: false, - 'obiba.mica.HarmonizedDatasetDto.type': {}, + 'obiba.mica.HarmonizedDatasetDto.type': { + harmonizationTable: { + namespace: 'opal' + } + }, model: {} }; + getOpalProjects().then(function() { + $scope.selected.project = $scope.projects.pop(); + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); + }); } $scope.save = function () { @@ -343,8 +362,10 @@ mica.dataset } $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}; - $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project = $scope.selected.project.name; - $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table = $scope.selected.project.table; + if ($scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.namespace === 'opal') { + $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.project = $scope.selected.project.name; + $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table = $scope.selected.project.table; + } $scope.dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.studyId = $scope.selected.study ? $scope.selected.study.id : null; if ($scope.dataset.id) { @@ -387,6 +408,12 @@ mica.dataset initializeForm(); FormDirtyStateObserver.observe($scope); + + $scope.NAMESPACES = [ + 'opal', + 'file', + 'other' + ]; }]) .controller('DatasetViewController', ['$rootScope', @@ -823,7 +850,7 @@ mica.dataset '$filter', 'MicaConfigResource', 'StudyStatesResource', - 'StudyStateProjectsResource', + 'StudyStateOpalProjectsResource', 'LocalizedValues', 'LocalizedSchemaFormService', 'isCommentsRequiredOnDocumentSave', @@ -836,7 +863,7 @@ mica.dataset $filter, MicaConfigResource, StudyStatesResource, - StudyStateProjectsResource, + StudyStateOpalProjectsResource, LocalizedValues, LocalizedSchemaFormService, isCommentsRequiredOnDocumentSave, @@ -861,10 +888,13 @@ mica.dataset $scope.type = tableType; $scope.selected.isHarmonizationTable = tableType === mica.dataset.OPAL_TABLE_TYPES.HARMONIZATION_TABLE; $scope.table = $.extend(true, {}, table); + if (!$scope.table.namespace) { + $scope.table.namespace = 'opal'; + } $scope.table.model = { name: LocalizedValues.arrayToObject(table.name), description: LocalizedValues.arrayToObject(table.description), - additionalInformation: LocalizedValues.arrayToObject(table.additionalInformation) + additionalInformation: LocalizedValues.arrayToObject(table.additionalInformation), }; MicaConfigResource.get(function (micaConfig) { @@ -929,21 +959,32 @@ mica.dataset $scope.$watch('selected.study', function () { if ($scope.selected.study && $scope.selected.study.id) { - StudyStateProjectsResource.query({id: $scope.selected.study.id}).$promise.then(function (projects) { - $scope.projects = projects; - var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === table.project; })[0]; + StudyStateOpalProjectsResource.query({id: $scope.selected.study.id}).$promise + .then(function (projects) { + $scope.projects = projects; + var selectedTable, selectedProject = $scope.projects.filter(function (p) {return p.name === table.project; })[0]; - if (selectedProject) { - $scope.selected.project = selectedProject; + if (selectedProject) { + $scope.selected.project = selectedProject; - selectedTable = selectedProject.datasource.table.filter(function (t) {return t === table.table; })[0]; + selectedTable = selectedProject.datasource.table.filter(function (t) {return t === table.table; })[0]; - if (selectedTable) { - $scope.selected.project.table = selectedTable; + if (selectedTable) { + $scope.selected.project.table = selectedTable; + } + } else { + $scope.selected.project = $scope.projects.pop(); + if ($scope.selected.project) { + $scope.selected.project.table = $scope.selected.project.datasource.table.pop(); + } } - } - }); + }) + .catch(function (err) { + console.log(err); + $scope.projects = []; + $scope.selected.project = undefined; + }); } }); @@ -956,13 +997,27 @@ mica.dataset !$scope.selected.study.population || !$scope.selected.study.population.dataCollectionEvent ? null : $scope.selected.study.population.dataCollectionEvent.id; - angular.extend($scope.table, { - studyId: $scope.selected.study.id, - populationId: populationId, - dataCollectionEventId: dceId, - project: $scope.selected.project.name, - table: $scope.selected.project.table - }); + if ($scope.table.namespace === 'file') { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId + }); + } else if ($scope.table.namespace === 'opal') { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId, + project: $scope.selected.project.name, + table: $scope.selected.project.table + }); + } else { + angular.extend($scope.table, { + studyId: $scope.selected.study.id, + populationId: populationId, + dataCollectionEventId: dceId + }); + } $scope.table.name = LocalizedValues.objectToArray($scope.table.model.name); $scope.table.description = LocalizedValues.objectToArray($scope.table.model.description); @@ -981,4 +1036,10 @@ mica.dataset $uibModalInstance.dismiss('cancel'); }; + $scope.NAMESPACES = [ + 'opal', + 'file', + 'other' + ]; + }]); diff --git a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js index e795af4c36..5887ba4139 100644 --- a/mica-webapp/src/main/webapp/app/dataset/dataset-service.js +++ b/mica-webapp/src/main/webapp/app/dataset/dataset-service.js @@ -48,9 +48,9 @@ mica.dataset }); }]) - .factory('StudyStateProjectsResource', ['$resource', + .factory('StudyStateOpalProjectsResource', ['$resource', function ($resource) { - return $resource(contextPath + '/ws/draft/study-state/:id/projects', {}, { + return $resource(contextPath + '/ws/draft/study-state/:id/opal-projects', {}, { 'get': {method: 'GET', params: {id: '@id'}} }); }]) @@ -372,6 +372,14 @@ mica.dataset serializeOpalTableForRestoringFields(datasetCopy); } + if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(serializeTableSource); + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].studyTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables || []).map(serializeTableSource); + datasetCopy['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = serializeTableSource(dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}); + } else if (typeof dataset['obiba.mica.CollectedDatasetDto.type'] === 'object') { + datasetCopy['obiba.mica.CollectedDatasetDto.type'].studyTable = serializeTableSource(dataset['obiba.mica.CollectedDatasetDto.type'].studyTable || {}); + } + datasetCopy.content = datasetCopy.model ? angular.toJson(datasetCopy.model) : null; delete datasetCopy.model; // NOTICE: must be removed to avoid protobuf exception in dto. return angular.toJson(datasetCopy); @@ -394,6 +402,14 @@ mica.dataset deserializeOpalTableForRestoringFields(dataset); } + if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { + dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(deserializeTableSource); + dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].studyTables || []).map(deserializeTableSource); + dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable = deserializeTableSource(dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable || {}); + } else if (typeof dataset['obiba.mica.CollectedDatasetDto.type'] === 'object') { + dataset['obiba.mica.CollectedDatasetDto.type'].studyTable = deserializeTableSource(dataset['obiba.mica.CollectedDatasetDto.type'].studyTable || {}); + } + return dataset; } @@ -416,6 +432,26 @@ mica.dataset } } + function serializeTableSource(table) { + const result = JSON.parse(JSON.stringify(table)); + + if (result.namespace === 'file') { + result.source = 'urn:file:' + result.path + (result.table ? ':' + result.table : ''); + } else if (result.namespace === 'opal') { + result.source = 'urn:opal:' + result.project + '.' + result.table; + } else { + result.source = 'urn:' + result.nid + ':' + result.nss; + } + delete result.path; + delete result.project; + delete result.table; + delete result.nid; + delete result.nss; + delete result.namespace; + + return result; + } + function deserializeOpalTableForRestoringFields(dataset) { if (typeof dataset['obiba.mica.HarmonizedDatasetDto.type'] === 'object') { dataset.harmonizationTables = (dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTables || []).map(deserializeTable); @@ -435,5 +471,33 @@ mica.dataset } } + function deserializeTableSource(table) { + const result = JSON.parse(JSON.stringify(table)); + + if (result.source) { + if (result.source.startsWith('urn:opal:')) { + const id = result.source.replace('urn:opal:', ''); + result.namespace = 'opal'; + result.project = id.substring(0, id.indexOf('.')); + result.table = id.substring(id.indexOf('.') + 1); + } else if (result.source.startsWith('urn:file:')) { + const id = result.source.replace('urn:file:', ''); + result.namespace = 'file'; + result.path = id.indexOf(':') > 0 ? id.substring(0, id.indexOf(':')) : id; + result.table = id.indexOf(':') > 0 ? id.substring(id.indexOf(':') + 1) : undefined; + } else { + result.namespace = 'other'; + } + const id = result.source.replace('urn:', ''); + const sepIdx = id.indexOf(':'); + result.nid = id.substring(0, sepIdx); + result.nss = id.substring(sepIdx + 1); + } else { + result.namespace = 'opal'; + } + + return result; + } + return this; }]); diff --git a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html index 3f64ebbac5..1d7b7ef6da 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/collected-dataset-view-details.html @@ -54,18 +54,6 @@

      dataset.study-table.title

      {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.dataCollectionEventId}} - - dataset.project - - {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.project}} - - - - dataset.table - - {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} - - dataset.table-name @@ -101,6 +89,72 @@

      dataset.study-table.title

      +
      +

      dataset.datasource.opal.title

      +

      dataset.datasource.opal.info

      + + + + + + + + + + + + +
      dataset.project + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.project}} +
      dataset.table + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} +
      +
      + +
      +

      dataset.datasource.file.title

      +

      dataset.datasource.file.info

      + + + + + + + + + + + + +
      path + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.path}} +
      dataset.table + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.table}} +
      +
      + +
      +

      dataset.datasource.other.title

      +

      dataset.datasource.other.info

      + + + + + + + + + + + + +
      dataset.datasource.other.nid + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.nid}} +
      dataset.datasource.other.nss + {{dataset['obiba.mica.CollectedDatasetDto.type'].studyTable.nss}} +
      +
      +

    diff --git a/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html b/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html index 65e927207c..5d9198eac1 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/dataset-study-table-form.html @@ -57,9 +57,9 @@

    dataset.study-dce.title

    -

    dataset.datasource.title

    +

    dataset.datasource.opal.title

    -

    dataset.datasource.info

    +

    dataset.datasource.opal.info

    diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html index b7aef6bb21..769912ad73 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-dataset-form.html @@ -39,34 +39,95 @@

    dataset.data-schema.title

    dataset.data-schema.info

    -
    - - - {{$select.selected.name}} - - - - -

    dataset.project-dataschema-help

    +

    dataset.datasource.info-schema

    + + + +
    + +

    dataset.datasource.opal.info-schema

    +
    +
    +
    + + + {{$select.selected.name}} + + + + +

    dataset.project-dataschema-help

    +
    +
    +
    +
    + + + {{$select.selected}} + + + + +

    dataset.table-dataschema-help

    +
    +
    +
    +
    -
    - - - {{$select.selected}} - - - - -

    dataset.table-dataschema-help

    +
    +

    dataset.datasource.file.info-schema

    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +
    +
    + +
    +

    dataset.datasource.other.info-schema

    +
    +
    +
    + + +
    +
    +
    +
    + + +
    +
    +

    dataset.harmonization-study.title

    diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html index 9a6d7292da..625df12f65 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonization-study-tables-form.html @@ -21,8 +21,7 @@

    dataset.harmonized-tables.title

    study.label type - dataset.project - dataset.table + dataset.datasource.title dataset.table-name dataset.table-description actions @@ -46,10 +45,8 @@

    dataset.harmonized-tables.title

    {{(wrapper.type === 'studyTable' ? 'search.study.individual' : 'search.study.harmonization') | translate}} - {{wrapper.table.project}} - - - {{wrapper.table.table}} + {{wrapper.table.nid}} + {{wrapper.table.nss}}
      diff --git a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html index 994303fd0b..b9d24a3536 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/harmonized-dataset-view-details.html @@ -16,22 +16,71 @@

      dataset.data-schema.title

      dataset.data-schema.info

      - - - - - - - - - - - -
      dataset.project - {{datasetProject}} -
      dataset.table - {{datasetTable}} -
      + + +
      +

      dataset.datasource.opal.title

      + + + + + + + + + + + + +
      dataset.project + {{datasetProject}} +
      dataset.table + {{datasetTable}} +
      +
      + +
      +

      dataset.datasource.file.title

      + + + + + + + + + + + + +
      path + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.path}} +
      dataset.table + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.table}} +
      +
      + +
      +

      dataset.datasource.other.title

      + + + + + + + + + + + + +
      dataset.datasource.other.nid + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.nid}} +
      dataset.datasource.other.nss + {{dataset['obiba.mica.HarmonizedDatasetDto.type'].harmonizationTable.nss}} +
      +
      +

      dataset.harmonization-study.title

      dataset.harmonization-study.info

      diff --git a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html index 679214b42f..9716140040 100644 --- a/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html +++ b/mica-webapp/src/main/webapp/app/dataset/views/opal-table-modal-form.html @@ -26,38 +26,92 @@

      dataset.datasource.title

      -

      dataset.datasource.info

      -
      -
      -
      - - - {{$select.selected.name}} - - - - + + +
      +

      dataset.datasource.opal.info

      +
      +
      +
      + + + {{$select.selected.name}} + + + + +
      +
      +
      +
      + + + {{$select.selected}} + + + + +
      -
      -
      - - - {{$select.selected}} - - - - +
      + +
      +

      dataset.datasource.file.info

      +
      +
      +
      + + +
      +
      +
      +
      + + +
      +
      +
      +
      + +
      +

      dataset.datasource.other.info

      +
      +
      +
      + + +
      +
      +
      +
      + + +
      diff --git a/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js b/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js index 1a431874c3..fd741c323d 100644 --- a/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js +++ b/mica-webapp/src/main/webapp/app/file-system/file-system-controller.js @@ -29,6 +29,7 @@ mica.fileSystem 'DraftFileSystemSearchResource', 'MicaConfigResource', '$q', + '$timeout', function ($rootScope, $scope, @@ -45,7 +46,8 @@ mica.fileSystem DraftFileAccessResource, DraftFileSystemSearchResource, MicaConfigResource, - $q) { + $q, + $timeout) { function buildClipboardCommand(command, origin, items) { return { @@ -499,6 +501,15 @@ mica.fileSystem }); }; + $scope.showCopiedTooltipStatus = false; + + $scope.showCopiedTooltip = function () { + $scope.showCopiedTooltipStatus = true; + $timeout(function () { + $scope.showCopiedTooltipStatus = false; + }, 1000); + }; + $scope.loadPermissions = function (document) { DraftFilePermissionResource.query({path: document.path}, function onSuccess(response) { $scope.data.permissions = response; diff --git a/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html b/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html index d9b57931b3..c05cb4a789 100644 --- a/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html +++ b/mica-webapp/src/main/webapp/app/file-system/views/documents-table-template.html @@ -77,6 +77,11 @@
      • +
    • + + + +
      • diff --git a/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html b/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html index d5fe8cbb1c..f3fd93a9ca 100644 --- a/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html +++ b/mica-webapp/src/main/webapp/app/file-system/views/file-system-template.html @@ -21,6 +21,11 @@ + + + + +
    • diff --git a/mica-webapp/src/main/webapp/app/network/network-service.js b/mica-webapp/src/main/webapp/app/network/network-service.js index 0798513b3c..d74332c000 100644 --- a/mica-webapp/src/main/webapp/app/network/network-service.js +++ b/mica-webapp/src/main/webapp/app/network/network-service.js @@ -37,13 +37,6 @@ mica.network }); }]) - .factory('DraftNetworkProjectsResource', ['$resource', - function ($resource) { - return $resource(contextPath + '/ws/draft/network/:id/projects', {}, { - 'get': {method: 'GET', errorHandler: true} - }); - }]) - .factory('DraftNetworkPermissionsResource', ['$resource', function ($resource) { return $resource(contextPath + '/ws/draft/network/:id/permissions', {}, { diff --git a/mica-webapp/src/main/webapp/app/search/search.js b/mica-webapp/src/main/webapp/app/search/search.js index 7944fc716c..5a54390eaa 100644 --- a/mica-webapp/src/main/webapp/app/search/search.js +++ b/mica-webapp/src/main/webapp/app/search/search.js @@ -13,10 +13,3 @@ mica.search = angular.module('mica.search', [ 'obiba.mica.search' ]); - -mica.search - .config(['ngObibaMicaSearchTemplateUrlProvider', - function (ngObibaMicaSearchTemplateUrlProvider) { - ngObibaMicaSearchTemplateUrlProvider.setHeaderUrl('search', 'app/search/views/search-view-header.html'); - ngObibaMicaSearchTemplateUrlProvider.setHeaderUrl('classifications', 'app/search/views/classifications-view-header.html'); - }]); diff --git a/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html b/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html deleted file mode 100644 index a7df2ad9e4..0000000000 --- a/mica-webapp/src/main/webapp/app/search/views/classifications-view-header.html +++ /dev/null @@ -1,19 +0,0 @@ - - -
      -

      -
        -
      1. global.menu.admin
        2. -
      2. classifications.title
        3. -
      -

      -

      classifications.help

      -
      diff --git a/mica-webapp/src/main/webapp/app/search/views/search-view-header.html b/mica-webapp/src/main/webapp/app/search/views/search-view-header.html deleted file mode 100644 index dd35b9ef57..0000000000 --- a/mica-webapp/src/main/webapp/app/search/views/search-view-header.html +++ /dev/null @@ -1,19 +0,0 @@ - - -
      -

      -
        -
      1. global.menu.admin
        2. -
      2. search.title
        3. -
      -

      -

      search.help

      -
      diff --git a/mica-webapp/src/main/webapp/app/services.js b/mica-webapp/src/main/webapp/app/services.js index baeae4805f..cb33186eef 100644 --- a/mica-webapp/src/main/webapp/app/services.js +++ b/mica-webapp/src/main/webapp/app/services.js @@ -194,7 +194,7 @@ mica.service('AuthenticationSharedService', ['$rootScope', '$q', '$http', '$cook mica.factory('MetricsService', ['$resource', function ($resource) { - return $resource('jvm', {}, { + return $resource('/jvm', {}, { 'get': { method: 'GET'} }); }]); diff --git a/mica-webapp/src/main/webapp/assets/js/mica-search.js b/mica-webapp/src/main/webapp/assets/js/mica-search.js index 574068563e..06b3d052cc 100644 --- a/mica-webapp/src/main/webapp/assets/js/mica-search.js +++ b/mica-webapp/src/main/webapp/assets/js/mica-search.js @@ -131,11 +131,21 @@ Vue.component('search-criteria', { return ai - bi; }); + const isTermHidden = function(term) { + if (term.attributes) { + const hiddenAttr = term.attributes.filter(attr => attr.key === 'hidden').pop(); + if (hiddenAttr) { + return hiddenAttr.value === 'true'; + } + } + return false; + } + for (let target of filteredTargets) { this.criteriaMenu.items[target.name].title = StringLocalizer.localize(target.title); switch (target.name) { case 'variable': - let level = target.terms[0].terms; + let level = target.terms[0].terms.filter(t => !isTermHidden(t)); const theRest = target.terms.slice(1); if (theRest.length > 0) { @@ -153,7 +163,7 @@ Vue.component('search-criteria', { case 'dataset': case 'study': case 'network': - this.criteriaMenu.items[target.name].menus = target.terms; + this.criteriaMenu.items[target.name].menus = target.terms ? target.terms.filter(t => !isTermHidden(t)) : []; break; } if (this.criteriaMenu.items[target.name].menus && this.criteriaMenu.items[target.name].menus.length > 0) { diff --git a/mica-webapp/src/main/webapp/assets/js/mica-variable.js b/mica-webapp/src/main/webapp/assets/js/mica-variable.js index c512911758..ac0ba49838 100644 --- a/mica-webapp/src/main/webapp/assets/js/mica-variable.js +++ b/mica-webapp/src/main/webapp/assets/js/mica-variable.js @@ -219,7 +219,7 @@ const makeHarmonizedVariablesTable = function() { dceName = dceName + ' -- ' + localizedString(dce.name); } harmonizedVariablesTableBody.append('' + - '' + harmonizedVariable.resolver.name + '' + + '' + harmonizedVariable.resolver.name + '' + '' + studyAnchor(baseStudyTable.studySummary) + '' + '' + getTableInformation(baseStudyTable, studyTableName) + '' + '' + dceName + '' + diff --git a/pom.xml b/pom.xml index e545348593..a936c95b48 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 3.0.0 1.2.9 6.6 + 4.0.1 3.0.0 4.7.2 0.7.9 @@ -422,6 +423,12 @@ cli + + org.obiba.magma + magma-datasource-excel + ${magma.version} + + org.obiba.opal opal-web-model