Merge pull request #176 from broadinstitute/namespace-packages

nawatts · web-flow · commit e1138f26d77a · 2020-03-04T10:02:43.000-05:00
Restructure as namespace packages
diff --git a/docs/README.md b/docs/README.md
@@ -1,11 +1,11 @@
 # Documentation
 
-Documentation for gnomad_hail is generated using [Sphinx](https://www.sphinx-doc.org/en/master/).
+Documentation for gnomad is generated using [Sphinx](https://www.sphinx-doc.org/en/master/).
 
 To build the documentation, run:
 
 ```
-cd /path/to/gnomad_hail
+cd /path/to/gnomad
 pip install -r requirements.txt
 pip install -r docs/requirements.txt
 ./docs/build.sh
diff --git a/docs/conf.py b/docs/conf.py
@@ -6,11 +6,11 @@
 
 from directives import AutoModuleSummary
 
-# Add gnomad_hail to import path.
+# Add gnomad to import path.
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 
-project = "gnomad_hail"
+project = "gnomad"
 version = release = "master"
 
 extensions = [
@@ -113,4 +113,4 @@ def get_annotation_module(annotation):
 
 
 def setup(app):
-    app.add_directive("gnomadhail_automodulesummary", AutoModuleSummary)
+    app.add_directive("gnomad_automodulesummary", AutoModuleSummary)
diff --git a/docs/generate_api_reference.py b/docs/generate_api_reference.py
@@ -11,11 +11,11 @@
 import re
 import sys
 
-# Add gnomad_hail to import path.
+# Add gnomad to import path.
 sys.path.insert(0, str(pathlib.Path(os.path.abspath(__file__)).parent.parent))
 
 ROOT_PACKAGE_PATH = str(
-    pathlib.Path(os.path.abspath(__file__)).parent.parent / "gnomad_hail"
+    pathlib.Path(os.path.abspath(__file__)).parent.parent / "gnomad"
 )
 
 DOCS_DIRECTORY = str(pathlib.Path(os.path.abspath(__file__)).parent)
@@ -69,7 +69,7 @@ def format_title(title):
 
 {module_doc}
 
-.. gnomadhail_automodulesummary:: {module_name}
+.. gnomad_automodulesummary:: {module_name}
 
 .. automodule:: {module_name}
 """
@@ -133,4 +133,14 @@ def write_package_doc(package_name):
 
 
 if __name__ == "__main__":
-    write_package_doc("gnomad_hail")
+    packages = ["resources", "utils"]
+    for pkg in packages:
+        write_package_doc(f"gnomad.{pkg}")
+
+    root_doc = PACKAGE_DOC_TEMPLATE.format(
+        title=format_title("gnomad"),
+        package_doc="",
+        module_links="\n    ".join(f"{pkg} <{pkg}/index>" for pkg in packages),
+    )
+
+    write_file(os.path.join(DOCS_DIRECTORY, "api_reference", "index.rst"), root_doc)
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,5 +1,5 @@
-gnomad_hail
-===========
+gnomad
+======
 
 This package contains a number of `Hail <https://hail.is>`_ utility functions
 and scripts for the `gnomAD project <https://gnomad.broadinstitute.org>`_ and
diff --git a/gnomad/resources/__init__.py b/gnomad/resources/__init__.py
diff --git a/gnomad/resources/grch37/__init__.py b/gnomad/resources/grch37/__init__.py
diff --git a/gnomad/resources/grch37/gnomad.py b/gnomad/resources/grch37/gnomad.py
@@ -1,4 +1,4 @@
-from gnomad_hail.resources.resource_utils import (
+from gnomad.resources.resource_utils import (
     DataException,
     PedigreeResource,
     TableResource,
diff --git a/gnomad/resources/grch37/gnomad_ld.py b/gnomad/resources/grch37/gnomad_ld.py
@@ -1,5 +1,5 @@
-from gnomad_hail.resources.resource_utils import TableResource, BlockMatrixResource
-from gnomad_hail.resources.grch37.gnomad import CURRENT_EXOME_RELEASE, CURRENT_GENOME_RELEASE
+from gnomad.resources.resource_utils import TableResource, BlockMatrixResource
+from gnomad.resources.grch37.gnomad import CURRENT_EXOME_RELEASE, CURRENT_GENOME_RELEASE
 from typing import Optional
 
 
diff --git a/gnomad/resources/grch37/reference_data.py b/gnomad/resources/grch37/reference_data.py
@@ -1,4 +1,4 @@
-from gnomad_hail.resources.resource_utils import (
+from gnomad.resources.resource_utils import (
     MatrixTableResource,
     TableResource,
     VersionedMatrixTableResource,
diff --git a/gnomad/resources/grch38/__init__.py b/gnomad/resources/grch38/__init__.py
diff --git a/gnomad/resources/grch38/gnomad.py b/gnomad/resources/grch38/gnomad.py
@@ -1,4 +1,4 @@
-from gnomad_hail.resources.resource_utils import (
+from gnomad.resources.resource_utils import (
     TableResource,
     VersionedMatrixTableResource,
     MatrixTableResource,
diff --git a/gnomad/resources/grch38/reference_data.py b/gnomad/resources/grch38/reference_data.py
@@ -1,12 +1,12 @@
-from gnomad_hail.resources.resource_utils import (
+from gnomad.resources.resource_utils import (
     TableResource,
     VersionedTableResource,
     MatrixTableResource,
     VersionedMatrixTableResource,
     import_sites_vcf,
     NO_CHR_TO_CHR_CONTIG_RECODING
 )
-from gnomad_hail.utils.generic import vep_or_lookup_vep
+from gnomad.utils.generic import vep_or_lookup_vep
 import hail as hl
 
 from hail import Table
diff --git a/gnomad/resources/import_resources.py b/gnomad/resources/import_resources.py
@@ -2,9 +2,9 @@
 import textwrap
 from inspect import getmembers
 from typing import Dict, Tuple, Optional
-from gnomad_hail.resources.resource_utils import BaseResource, BaseVersionedResource
-import gnomad_hail.resources.grch37 as grch37
-import gnomad_hail.resources.grch38 as grch38
+from gnomad.resources.resource_utils import BaseResource, BaseVersionedResource
+import gnomad.resources.grch37 as grch37
+import gnomad.resources.grch38 as grch38
 import argparse
 
 
@@ -16,11 +16,11 @@ def get_module_importable_resources(module, prefix: Optional[str] = None) -> Dic
     keys: {prefix}.{resource_name}.{version} (with prefix only present if `prefix` is set, and `version` only present for versioned resources)
     values: ({resource_name}[ version {version}], resource) with resource_name set to the variable name in the module and the version present for versioned resources.
 
-     The following example will generate a dict with all the resources in gnomad_hail.resources.grch37 that can be imported:
+     The following example will generate a dict with all the resources in gnomad.resources.grch37 that can be imported:
 
     .. code-block:: python
 
-        import gnomad_hail.resources.grch37 as grch37
+        import gnomad.resources.grch37 as grch37
         grch37_resources = get_module_importable_resources(grch37, prefix='grch37')
     
     :param module: Input module
diff --git a/gnomad/resources/resource_utils.py b/gnomad/resources/resource_utils.py
@@ -1,10 +1,13 @@
+import logging
 from typing import Optional, Dict, Any, Callable, List
-from gnomad_hail.utils.gnomad_functions import logger
 import hail as hl
 from hail.linalg import BlockMatrix
 from abc import ABC, abstractmethod
 
 
+logger = logging.getLogger("gnomad.resources")
+
+
 # Resource classes
 class BaseResource(ABC):
     """
diff --git a/gnomad/utils/__init__.py b/gnomad/utils/__init__.py
diff --git a/gnomad/utils/annotations.py b/gnomad/utils/annotations.py
@@ -1,8 +1,16 @@
-from gnomad_hail import *
-import itertools
+import logging
+from typing import Dict, List, Optional, Set, Tuple, Union
+
+import hail as hl
+
+from gnomad.utils.generic import filter_to_autosomes
+from gnomad.utils.gnomad_functions import annotate_adj
 
 # TODO: Use import below when relatedness PR goes in
-# from gnomad_hail.utils.relatedness import SIBLINGS
+# from gnomad.utils.relatedness import SIBLINGS
+
+
+logger = logging.getLogger("gnomad.utils")
 
 
 def pop_max_expr(
@@ -531,7 +539,7 @@ def low_qual_expr(
         ref: hl.expr.StringExpression,
         alt: hl.expr.StringExpression,
         qual_approx: hl.expr.NumericExpression,
-    ) -> BooleanExpression:
+    ) -> hl.expr.BooleanExpression:
         return hl.cond(
             hl.is_snp(ref, alt),
             qual_approx < snv_phred_threshold + snv_phred_het_prior,
@@ -893,7 +901,7 @@ def default_generate_sib_stats(
 
     This function takes a hail Table with a row for each pair of individuals i,j in the data that are related (it's OK to have unrelated samples too).
     The `relationship_col` should be a column specifying the relationship between each two samples as defined by
-    the constants in `gnomad_hail.utils.relatedness`. This relationship_col will be used to filter to only pairs of
+    the constants in `gnomad.utils.relatedness`. This relationship_col will be used to filter to only pairs of
     samples that are annotated as `SIBLINGS`.
 
     :param mt: Input Matrix table
diff --git a/gnomad/utils/constants.py b/gnomad/utils/constants.py
diff --git a/gnomad/utils/generic.py b/gnomad/utils/generic.py
@@ -11,7 +11,7 @@
 from hail.utils.misc import divide_null
 from .gnomad_functions import logger
 import os
-from gnomad_hail.resources.resource_utils import DataException
+from gnomad.resources.resource_utils import DataException
 
 INFO_VCF_AS_PIPE_DELIMITED_FIELDS = ['AS_QUALapprox', 'AS_VarDP', 'AS_MQ_DP', 'AS_RAW_MQ', 'AS_SB_TABLE']
 
@@ -245,9 +245,9 @@ def filter_low_conf_regions(mt: Union[hl.MatrixTable, hl.Table], filter_lcr: boo
     """
     build = get_reference_genome(mt.locus).name
     if build == "GRCh37":
-        import gnomad_hail.resources.grch37.reference_data as resources
+        import gnomad.resources.grch37.reference_data as resources
     elif build == "GRCh38":
-        import gnomad_hail.resources.grch38.reference_data as resources
+        import gnomad.resources.grch38.reference_data as resources
 
     criteria = []
     if filter_lcr:
diff --git a/gnomad/utils/gnomad_functions.py b/gnomad/utils/gnomad_functions.py
@@ -165,7 +165,7 @@ def liftover_using_gnomad_map(ht, data_type):
     :param data_type: one of "exomes" or "genomes" which to map across
     :return: Lifted over table
     """
-    from gnomad_hail.resources.grch37.gnomad import liftover
+    from gnomad.resources.grch37.gnomad import liftover
     lift_ht = liftover(data_type).ht()
     ht = ht.key_by(original_locus=ht.locus, original_alleles=ht.alleles).drop('locus', 'alleles')
     return lift_ht.annotate(**ht[(lift_ht.original_locus, lift_ht.original_alleles)]).key_by('locus', 'alleles')
diff --git a/gnomad/utils/ld.py b/gnomad/utils/ld.py
@@ -1,5 +1,5 @@
-from gnomad_hail.resources.grch37.gnomad_ld import ld_index, ld_matrix
-from gnomad_hail.resources.grch37.gnomad import public_release
+from gnomad.resources.grch37.gnomad_ld import ld_index, ld_matrix
+from gnomad.resources.grch37.gnomad import public_release
 from hail.linalg import BlockMatrix
 import hail as hl
 
diff --git a/gnomad/utils/liftover.py b/gnomad/utils/liftover.py
@@ -1,8 +1,8 @@
 import argparse
 from typing import Union
 import hail as hl
-from gnomad_hail.resources.grch37.gnomad import public_release
-from gnomad_hail.utils.generic import flip_base, get_reference_genome
+from gnomad.resources.grch37.gnomad import public_release
+from gnomad.utils.generic import flip_base, get_reference_genome
 import logging
 from os.path import dirname, basename
 import sys
diff --git a/gnomad/utils/plotting.py b/gnomad/utils/plotting.py
diff --git a/gnomad/utils/relatedness.py b/gnomad/utils/relatedness.py
@@ -1,9 +1,13 @@
 import hail as hl
-from gnomad_hail import logger
+import logging
 from typing import Dict, List, Tuple, Set, Union, Iterable, Optional
 from collections import defaultdict
 import random
 
+
+logger = logging.getLogger("gnomad.utils")
+
+
 UNRELATED = 'Unrelated'
 """
 String representation for a pair of unrelated individuals in this module.
diff --git a/gnomad/utils/rf.py b/gnomad/utils/rf.py
@@ -1,4 +1,7 @@
-from gnomad_hail import *
+import logging
+import pprint
+from typing import Dict, List, Tuple
+
 import pyspark.sql
 from pyspark.sql import SparkSession
 from pyspark.ml.feature import *
@@ -9,6 +12,10 @@
 from pyspark.sql.functions import udf, col  # pylint: disable=no-name-in-module
 from pyspark.sql.types import ArrayType, DoubleType
 
+
+logger = logging.getLogger("gnomad.utils")
+
+
 def run_rf_test(
         mt: hl.MatrixTable,
         output: str = '/tmp'
@@ -227,7 +234,7 @@ def test_model(
     # Print results
     df = pd.DataFrame(test_results)
     df = df.pivot(index=label, columns=prediction_col_name, values='n')
-    logger.info("Testing results:\n{}".format(pformat(df)))
+    logger.info("Testing results:\n{}".format(pprint.pformat(df)))
     logger.info("Accuracy: {}".format(
         sum([x.n for x in test_results if x[label] == x[prediction_col_name]]) /
         sum([x.n for x in test_results])
diff --git a/gnomad/utils/sample_qc.py b/gnomad/utils/sample_qc.py
@@ -2,7 +2,7 @@
 from .generic import *
 from .gnomad_functions import logger, filter_to_adj
 from .sparse_mt import impute_sex_ploidy
-from gnomad_hail.utils.relatedness import get_duplicated_samples
+from gnomad.utils.relatedness import get_duplicated_samples
 
 
 def filter_rows_for_qc(
diff --git a/gnomad/utils/slack.py b/gnomad/utils/slack.py
@@ -6,7 +6,7 @@ def get_slack_info():
     from slackclient import SlackClient
     # import os
     try:
-        from gnomad_hail.slack_creds import slack_token  # pylint: disable=import-error,no-name-in-module
+        from gnomad.slack_creds import slack_token  # pylint: disable=import-error,no-name-in-module
     except Exception:
         return None
 
diff --git a/gnomad/utils/sparse_mt.py b/gnomad/utils/sparse_mt.py
diff --git a/gnomad/utils/variant_qc.py b/gnomad/utils/variant_qc.py
@@ -1,6 +1,6 @@
 from .generic import *
-import gnomad_hail.resources.grch37 as grch37_resources
-import gnomad_hail.resources.grch38 as grch38_resources
+import gnomad.resources.grch37 as grch37_resources
+import gnomad.resources.grch38 as grch38_resources
 
 
 def compute_quantile_bin(
diff --git a/gnomad_hail/__init__.py b/gnomad_hail/__init__.py
diff --git a/init_scripts/master-init.sh b/init_scripts/master-init.sh
@@ -12,7 +12,7 @@ if [[ "${ROLE}" == 'Master' ]]; then
     git clone https://github.com/broadinstitute/gnomad_methods.git /home/gnomad_hail
 
     mkdir -p /home/hail/
-    ln -s ln -s /home/gnomad_hail/gnomad_hail /home/hail
+    ln -s ln -s /home/gnomad_hail/gnomad /home/hail
 
     cd /home/gnomad_hail/init_scripts
     chmod +x gnomad-init.sh sparklyr-init.sh
diff --git a/lint b/lint
@@ -2,4 +2,4 @@
 
 cd "$(dirname "$0")"
 
-pylint "$@" gnomad_hail
+pylint "$@" gnomad
diff --git a/setup.py b/setup.py
@@ -17,7 +17,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/broadinstitute/gnomad_methods",
-    packages=setuptools.find_packages(),
+    packages=setuptools.find_namespace_packages(include=["gnomad.*"]),
     project_urls={
         "Documentation": "https://broadinstitute.github.io/gnomad_methods/",
         "Source Code": "https://github.com/broadinstitute/gnomad_methods",

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-from gnomad_hail.resources.resource_utils import (`
	`1`	`+from gnomad.resources.resource_utils import (`
`2`	`2`	`DataException,`
`3`	`3`	`PedigreeResource,`
`4`	`4`	`TableResource,`