Switch to namespace packages

Author: nawatts

docs/generate_api_reference.py

@@ -133,4 +133,14 @@ def write_package_doc(package_name):
 
 
 if __name__ == "__main__":
-    write_package_doc("gnomad")
+    packages = ["resources", "utils"]
+    for pkg in packages:
+        write_package_doc(f"gnomad.{pkg}")
+
+    root_doc = PACKAGE_DOC_TEMPLATE.format(
+        title=format_title("gnomad"),
+        package_doc="",
+        module_links="\n    ".join(f"{pkg} <{pkg}/index>" for pkg in packages),
+    )
+
+    write_file(os.path.join(DOCS_DIRECTORY, "api_reference", "index.rst"), root_doc)

gnomad/__init__.py

@@ -1,10 +1,13 @@
+import logging
 from typing import Optional, Dict, Any, Callable, List
-from gnomad.utils.gnomad_functions import logger
 import hail as hl
 from hail.linalg import BlockMatrix
 from abc import ABC, abstractmethod
 
 
+logger = logging.getLogger("gnomad.resources")
+
+
 # Resource classes
 class BaseResource(ABC):
     """

gnomad/resources/resource_utils.py

@@ -1,10 +1,18 @@
-from gnomad import *
-import itertools
+import logging
+from typing import Dict, List, Optional, Set, Tuple, Union
+
+import hail as hl
+
+from gnomad.utils.generic import filter_to_autosomes
+from gnomad.utils.gnomad_functions import annotate_adj
 
 # TODO: Use import below when relatedness PR goes in
 # from gnomad.utils.relatedness import SIBLINGS
 
 
+logger = logging.getLogger("gnomad.utils")
+
+
 def pop_max_expr(
     freq: hl.expr.ArrayExpression,
     freq_meta: hl.expr.ArrayExpression,
@@ -531,7 +539,7 @@ def low_qual_expr(
         ref: hl.expr.StringExpression,
         alt: hl.expr.StringExpression,
         qual_approx: hl.expr.NumericExpression,
-    ) -> BooleanExpression:
+    ) -> hl.expr.BooleanExpression:
         return hl.cond(
             hl.is_snp(ref, alt),
             qual_approx < snv_phred_threshold + snv_phred_het_prior,

gnomad/utils/annotations.py

@@ -1,9 +1,13 @@
 import hail as hl
-from gnomad import logger
+import logging
 from typing import Dict, List, Tuple, Set, Union, Iterable, Optional
 from collections import defaultdict
 import random
 
+
+logger = logging.getLogger("gnomad.utils")
+
+
 UNRELATED = 'Unrelated'
 """
 String representation for a pair of unrelated individuals in this module.

gnomad/utils/relatedness.py

@@ -1,4 +1,7 @@
-from gnomad import *
+import logging
+import pprint
+from typing import Dict, List, Tuple
+
 import pyspark.sql
 from pyspark.sql import SparkSession
 from pyspark.ml.feature import *
@@ -9,6 +12,10 @@
 from pyspark.sql.functions import udf, col  # pylint: disable=no-name-in-module
 from pyspark.sql.types import ArrayType, DoubleType
 
+
+logger = logging.getLogger("gnomad.utils")
+
+
 def run_rf_test(
         mt: hl.MatrixTable,
         output: str = '/tmp'
@@ -227,7 +234,7 @@ def test_model(
     # Print results
     df = pd.DataFrame(test_results)
     df = df.pivot(index=label, columns=prediction_col_name, values='n')
-    logger.info("Testing results:\n{}".format(pformat(df)))
+    logger.info("Testing results:\n{}".format(pprint.pformat(df)))
     logger.info("Accuracy: {}".format(
         sum([x.n for x in test_results if x[label] == x[prediction_col_name]]) /
         sum([x.n for x in test_results])

gnomad/utils/rf.py

@@ -17,7 +17,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/broadinstitute/gnomad_methods",
-    packages=setuptools.find_packages(),
+    packages=setuptools.find_namespace_packages(include=["gnomad.*"]),
     project_urls={
         "Documentation": "https://broadinstitute.github.io/gnomad_methods/",
         "Source Code": "https://github.com/broadinstitute/gnomad_methods",