remove version prefix from paths for local users (#917)

* remove prefix from paths for local users

* ruff

* get

* fix reference data path

* correct usage

* another bug

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -39,6 +39,7 @@ jobs:
           export GRCH37_TO_GRCH38_LIFTOVER_REF_PATH=v03_pipeline/var/test/liftover/grch37_to_grch38.over.chain.gz
           export GRCH38_TO_GRCH37_LIFTOVER_REF_PATH=v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz
           export ACCESS_PRIVATE_REFERENCE_DATASETS=1
+          export INCLUDE_PIPELINE_VERSION_IN_PREFIX=1
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           nosetests --with-coverage --cover-package v03_pipeline/lib v03_pipeline/lib
           coverage report --omit '*test*' --fail-under=75

v03_pipeline/lib/model/environment.py

@@ -44,6 +44,9 @@
 )
 CHECK_SEX_AND_RELATEDNESS = os.environ.get('CHECK_SEX_AND_RELATEDNESS') == '1'
 EXPECT_WES_FILTERS = os.environ.get('EXPECT_WES_FILTERS') == '1'
+INCLUDE_PIPELINE_VERSION_IN_PREFIX = (
+    os.environ.get('INCLUDE_PIPELINE_VERSION_IN_PREFIX') == '1'
+)
 SHOULD_REGISTER_ALLELES = os.environ.get('SHOULD_REGISTER_ALLELES') == '1'
 SHOULD_TRIGGER_HAIL_BACKEND_RELOAD = (
     os.environ.get('SHOULD_TRIGGER_HAIL_BACKEND_RELOAD') == '1'
@@ -56,12 +59,13 @@ class Env:
     ALLELE_REGISTRY_SECRET_NAME: str | None = ALLELE_REGISTRY_SECRET_NAME
     CHECK_SEX_AND_RELATEDNESS: bool = CHECK_SEX_AND_RELATEDNESS
     EXPECT_WES_FILTERS: bool = EXPECT_WES_FILTERS
+    GRCH37_TO_GRCH38_LIFTOVER_REF_PATH: str = GRCH37_TO_GRCH38_LIFTOVER_REF_PATH
+    GRCH38_TO_GRCH37_LIFTOVER_REF_PATH: str = GRCH38_TO_GRCH37_LIFTOVER_REF_PATH
     HAIL_BACKEND_SERVICE_HOSTNAME: str | None = HAIL_BACKEND_SERVICE_HOSTNAME
     HAIL_BACKEND_SERVICE_PORT: int = HAIL_BACKEND_SERVICE_PORT
     HAIL_TMP_DIR: str = HAIL_TMP_DIR
     HAIL_SEARCH_DATA_DIR: str = HAIL_SEARCH_DATA_DIR
-    GRCH37_TO_GRCH38_LIFTOVER_REF_PATH: str = GRCH37_TO_GRCH38_LIFTOVER_REF_PATH
-    GRCH38_TO_GRCH37_LIFTOVER_REF_PATH: str = GRCH38_TO_GRCH37_LIFTOVER_REF_PATH
+    INCLUDE_PIPELINE_VERSION_IN_PREFIX: bool = INCLUDE_PIPELINE_VERSION_IN_PREFIX
     LOADING_DATASETS_DIR: str = LOADING_DATASETS_DIR
     PRIVATE_REFERENCE_DATASETS_DIR: str = PRIVATE_REFERENCE_DATASETS_DIR
     PROJECT_ID: str | None = PROJECT_ID

v03_pipeline/lib/paths.py

@@ -19,9 +19,15 @@ def _pipeline_prefix(
     reference_genome: ReferenceGenome,
     dataset_type: DatasetType,
 ) -> str:
+    if Env.INCLUDE_PIPELINE_VERSION_IN_PREFIX:
+        return os.path.join(
+            root,
+            PipelineVersion.V3_1.value,
+            reference_genome.value,
+            dataset_type.value,
+        )
     return os.path.join(
         root,
-        PipelineVersion.V3_1.value,
         reference_genome.value,
         dataset_type.value,
     )
@@ -30,16 +36,24 @@ def _pipeline_prefix(
 def _v03_reference_data_prefix(
     access_control: AccessControl,
     reference_genome: ReferenceGenome,
+    dataset_type: DatasetType,
 ) -> str:
     root = (
         Env.PRIVATE_REFERENCE_DATASETS_DIR
         if access_control == AccessControl.PRIVATE
         else Env.REFERENCE_DATASETS_DIR
     )
+    if Env.INCLUDE_PIPELINE_VERSION_IN_PREFIX:
+        return os.path.join(
+            root,
+            PipelineVersion.V03.value,
+            reference_genome.value,
+            dataset_type.value,
+        )
     return os.path.join(
         root,
-        PipelineVersion.V03.value,
         reference_genome.value,
+        dataset_type.value,
     )
 
 
@@ -52,8 +66,8 @@ def cached_reference_dataset_query_path(
         _v03_reference_data_prefix(
             AccessControl.PUBLIC,
             reference_genome,
+            dataset_type,
         ),
-        dataset_type.value,
         'cached_reference_dataset_queries',
         f'{cached_reference_dataset_query.value}.ht',
     )
@@ -267,8 +281,8 @@ def valid_reference_dataset_collection_path(
         _v03_reference_data_prefix(
             reference_dataset_collection.access_control,
             reference_genome,
+            dataset_type,
         ),
-        dataset_type.value,
         'reference_datasets',
         f'{reference_dataset_collection.value}.ht',
     )

v03_pipeline/lib/paths_test.py

@@ -61,6 +61,16 @@ def test_family_table_path(self) -> None:
                 ),
                 'gs://seqr-datasets/v3.1/GRCh37/SNV_INDEL/families/WES/franklin.ht',
             )
+            mock_env.INCLUDE_PIPELINE_VERSION_IN_PREFIX = False
+            self.assertEqual(
+                family_table_path(
+                    ReferenceGenome.GRCh37,
+                    DatasetType.SNV_INDEL,
+                    SampleType.WES,
+                    'franklin',
+                ),
+                'gs://seqr-datasets/GRCh37/SNV_INDEL/families/WES/franklin.ht',
+            )
 
     def test_valid_filters_path(self) -> None:
         self.assertEqual(

v03_pipeline/lib/test/mocked_dataroot_testcase.py

@@ -13,7 +13,7 @@ def setUp(self) -> None:
         self.mock_env = patcher.start()
         self.addCleanup(patcher.stop)  # https://stackoverflow.com/a/37534051
         for field_name in Env.__dataclass_fields__:
-            if 'DATA' in field_name or 'DIR' in field_name:
+            if field_name.endswith('_DIR'):
                 setattr(self.mock_env, field_name, tempfile.TemporaryDirectory().name)
 
     def tearDown(self) -> None: