diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..d04df8946 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +exclude = ga4gh/*_pb2.py,ga4gh/datamodel/obo_parser.py,docs,ez_setup.py diff --git a/.gitignore b/.gitignore index 61c5228d0..af596b7c8 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ tests/data/registry.db *.rar *.tar *.zip +*.whl # Logs and databases # ###################### diff --git a/.travis.yml b/.travis.yml index ef0a0cff2..0cdc3e273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,6 @@ install: - ga4gh_repo --version before_script: - # the following two lines are to prevent travis from - # verbosely complaining about a security problem; see issue 54 - - mkdir $HOME/.python-eggs - - chmod og-w $HOME/.python-eggs - pip install -r dev-requirements.txt - python scripts/build_test_data.py @@ -33,14 +29,16 @@ before_script: # run_tests.py runs everything under the script: tag so only put commands # under it that we want to run (and want to be able to run) as local tests script: - - flake8 *.py tests ga4gh scripts - --exclude=ez_setup.py,ga4gh/*_pb2.py,ga4gh/datamodel/obo_parser.py + - flake8 client_dev.py configtest_dev.py convert_error_code.py + ga2sam_dev.py ga2vcf_dev.py repo_dev.py server_dev.py + setup.py + tests ga4gh scripts - nosetests --with-coverage --cover-package ga4gh --cover-inclusive --cover-min-percentage 80 --cover-branches --cover-erase - make clean -C docs - make -C docs -# run codecov ! after_success: - - bash <(curl -s https://codecov.io/bash) \ No newline at end of file +# run codecov + - bash <(curl -s https://codecov.io/bash) diff --git a/dev-requirements.txt b/dev-requirements.txt index bf2072e97..1f3ca0425 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -16,16 +16,18 @@ PyYAML==3.11 # do not import them in any code that can be reached via # an executable that we ship +mock +nose +pep8 +flake8 +coverage + PyVCF==0.6.7 -coverage==3.7.1 -flake8==2.3.0 # Set due to conflict with pep8 in version 2.4.0 freezegun==0.3.6 guppy==0.1.10 -mock==1.0.0 -nose==1.3.7 -pep8==1.6.2 snakefood==1.4 Sphinx==1.3.1 + # Requirements for OIDC provider pyaml==15.03.1 cherrypy==3.2.4 diff --git a/ga4gh/cli/__init__.py b/ga4gh/cli/__init__.py index e2173e5d6..5983113f0 100644 --- a/ga4gh/cli/__init__.py +++ b/ga4gh/cli/__init__.py @@ -5,9 +5,6 @@ from __future__ import print_function from __future__ import unicode_literals -import argparse -import operator - import ga4gh import ga4gh.protocol as protocol @@ -20,52 +17,6 @@ AVRO_LONG_MAX = 2**31 - 1 -class SortedHelpFormatter(argparse.HelpFormatter): - """ - An argparse HelpFormatter that sorts the flags and subcommands - in alphabetical order - """ - def add_arguments(self, actions): - """ - Sort the flags alphabetically - """ - actions = sorted( - actions, key=operator.attrgetter('option_strings')) - super(SortedHelpFormatter, self).add_arguments(actions) - - def _iter_indented_subactions(self, action): - """ - Sort the subcommands alphabetically - """ - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - self._indent() - if isinstance(action, argparse._SubParsersAction): - for subaction in sorted( - get_subactions(), key=lambda x: x.dest): - yield subaction - else: - for subaction in get_subactions(): - yield subaction - self._dedent() - - -def addSubparser(subparsers, subcommand, description): - parser = subparsers.add_parser( - subcommand, description=description, help=description) - return parser - - -def createArgumentParser(description): - parser = argparse.ArgumentParser( - description=description, - formatter_class=SortedHelpFormatter) - return parser - - def addVersionArgument(parser): # TODO argparse strips newlines from version output versionString = ( diff --git a/ga4gh/cli/client.py b/ga4gh/cli/client.py index c24c08bf6..f178d7c55 100644 --- a/ga4gh/cli/client.py +++ b/ga4gh/cli/client.py @@ -16,6 +16,8 @@ import ga4gh.exceptions as exceptions import ga4gh.protocol as protocol +import ga4gh_common.cli as common_cli + def verbosityToLogLevel(verbosity): """ @@ -1194,7 +1196,7 @@ def addHelpParser(subparsers): def addVariantsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "variants-search", "Search for variants") parser.set_defaults(runner=SearchVariantsRunner) addUrlArgument(parser) @@ -1204,7 +1206,7 @@ def addVariantsSearchParser(subparsers): def addVariantSetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "variantsets-search", "Search for variantSets") parser.set_defaults(runner=SearchVariantSetsRunner) addOutputFormatArgument(parser) @@ -1240,42 +1242,42 @@ def addVariantAnnotationSetsSearchParser(subparsers): def addVariantAnnotationSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "variantannotationsets-get", "Get a variantAnnotationSet") parser.set_defaults(runner=GetVariantAnnotationSetRunner) addGetArguments(parser) def addVariantSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "variantsets-get", "Get a variantSet") parser.set_defaults(runner=GetVariantSetRunner) addGetArguments(parser) def addFeaturesGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "features-get", "Get a feature by ID") parser.set_defaults(runner=GetFeatureRunner) addGetArguments(parser) def addFeatureSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "featuresets-get", "Get a featureSet by ID") parser.set_defaults(runner=GetFeatureSetRunner) addGetArguments(parser) def addBioSamplesGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "biosamples-get", "Get a biosample by ID") parser.set_defaults(runner=GetBioSampleRunner) addGetArguments(parser) def addIndividualsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "individuals-get", "Get a individual by ID") parser.set_defaults(runner=GetIndividualRunner) addGetArguments(parser) @@ -1337,7 +1339,7 @@ def addFeatureSetsSearchParser(subparsers): def addReferenceSetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "referencesets-search", "Search for referenceSets") parser.set_defaults(runner=SearchReferenceSetsRunner) addUrlArgument(parser) @@ -1352,7 +1354,7 @@ def addReferenceSetsSearchParser(subparsers): def addReferencesSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "references-search", "Search for references") parser.set_defaults(runner=SearchReferencesRunner) addUrlArgument(parser) @@ -1365,7 +1367,7 @@ def addReferencesSearchParser(subparsers): def addReadGroupSetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "readgroupsets-search", "Search for readGroupSets") parser.set_defaults(runner=SearchReadGroupSetsRunner) addUrlArgument(parser) @@ -1378,7 +1380,7 @@ def addReadGroupSetsSearchParser(subparsers): def addCallSetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "callsets-search", "Search for callSets") parser.set_defaults(runner=SearchCallSetsRunner) addUrlArgument(parser) @@ -1391,7 +1393,7 @@ def addCallSetsSearchParser(subparsers): def addReadsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "reads-search", "Search for reads") parser.set_defaults(runner=SearchReadsRunner) addOutputFormatArgument(parser) @@ -1400,14 +1402,14 @@ def addReadsSearchParser(subparsers): def addDatasetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "datasets-get", "Get a dataset") parser.set_defaults(runner=GetDatasetRunner) addGetArguments(parser) def addDatasetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "datasets-search", "Search for datasets") parser.set_defaults(runner=SearchDatasetsRunner) addUrlArgument(parser) @@ -1430,49 +1432,49 @@ def addReadsSearchParserArguments(parser): def addReferenceSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "referencesets-get", "Get a referenceset") parser.set_defaults(runner=GetReferenceSetRunner) addGetArguments(parser) def addReferencesGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "references-get", "Get a reference") parser.set_defaults(runner=GetReferenceRunner) addGetArguments(parser) def addReadGroupSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "readgroupsets-get", "Get a read group set") parser.set_defaults(runner=GetReadGroupSetRunner) addGetArguments(parser) def addReadGroupsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "readgroups-get", "Get a read group") parser.set_defaults(runner=GetReadGroupRunner) addGetArguments(parser) def addCallSetsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "callsets-get", "Get a callSet") parser.set_defaults(runner=GetCallSetRunner) addGetArguments(parser) def addVariantsGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "variants-get", "Get a variant") parser.set_defaults(runner=GetVariantRunner) addGetArguments(parser) def addRnaQuantificationSetGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "rnaquantificationsets-get", "Get a rna quantification set") parser.set_defaults(runner=GetRnaQuantificationSetRunner) @@ -1480,21 +1482,21 @@ def addRnaQuantificationSetGetParser(subparsers): def addRnaQuantificationGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "rnaquantifications-get", "Get a rna quantification") parser.set_defaults(runner=GetRnaQuantificationRunner) addGetArguments(parser) def addExpressionLevelGetParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "expressionlevels-get", "Get a expression level") parser.set_defaults(runner=GetExpressionLevelRunner) addGetArguments(parser) def addReferencesBasesListParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "references-list-bases", "List bases of a reference") parser.add_argument( "--outputFormat", "-O", choices=['text', 'fasta'], default="text", @@ -1557,7 +1559,7 @@ def addExpressionLevelsSearchParser(subparsers): def addGenotypePhenotypeSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "genotypephenotype-search", "Search for genotype to phenotype associations") parser.set_defaults(runner=SearchGenotypePhenotypeRunner) @@ -1569,7 +1571,7 @@ def addGenotypePhenotypeSearchParser(subparsers): def addPhenotypeSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "phenotype-search", "Search for phenotypes") parser.set_defaults(runner=SearchPhenotypeRunner) addUrlArgument(parser) @@ -1580,7 +1582,7 @@ def addPhenotypeSearchParser(subparsers): def addPhenotypeAssociationSetsSearchParser(subparsers): - parser = cli.addSubparser( + parser = common_cli.addSubparser( subparsers, "phenotypeassociationsets-search", "Search for phenotypeassociationsets") parser.set_defaults(runner=SearchPhenotypeAssociationSetsRunner) @@ -1591,7 +1593,7 @@ def addPhenotypeAssociationSetsSearchParser(subparsers): def getClientParser(): - parser = cli.createArgumentParser("GA4GH reference client") + parser = common_cli.createArgumentParser("GA4GH reference client") addClientGlobalOptions(parser) subparsers = parser.add_subparsers(title='subcommands',) addHelpParser(subparsers) diff --git a/ga4gh/cli/configtest.py b/ga4gh/cli/configtest.py index 8621c9320..962e62ea0 100644 --- a/ga4gh/cli/configtest.py +++ b/ga4gh/cli/configtest.py @@ -11,6 +11,8 @@ import ga4gh.cli as cli import ga4gh.configtest as configtest +import ga4gh_common.cli as common_cli + class SimplerResult(unittest.TestResult): """ @@ -29,7 +31,7 @@ def addFailure(self, test, err): def configtest_main(parser=None): if parser is None: - parser = cli.createArgumentParser( + parser = common_cli.createArgumentParser( "GA4GH server configuration validator") parser.add_argument( "--config", "-c", default='DevelopmentConfig', type=str, diff --git a/ga4gh/cli/ga2sam.py b/ga4gh/cli/ga2sam.py index 172ebcc8b..791404591 100644 --- a/ga4gh/cli/ga2sam.py +++ b/ga4gh/cli/ga2sam.py @@ -9,6 +9,8 @@ import ga4gh.cli.client as cli_client import ga4gh.converters as converters +import ga4gh_common.cli as common_cli + class Ga2SamRunner(cli_client.SearchReadsRunner): """ @@ -31,7 +33,7 @@ def run(self): def getGa2SamParser(): - parser = cli.createArgumentParser("GA4GH SAM conversion tool") + parser = common_cli.createArgumentParser("GA4GH SAM conversion tool") cli_client.addClientGlobalOptions(parser) cli_client.addUrlArgument(parser) parser.add_argument( diff --git a/ga4gh/cli/ga2vcf.py b/ga4gh/cli/ga2vcf.py index 39016a78f..d8ca9097c 100644 --- a/ga4gh/cli/ga2vcf.py +++ b/ga4gh/cli/ga2vcf.py @@ -9,6 +9,8 @@ import ga4gh.cli.client as cli_client import ga4gh.converters as converters +import ga4gh_common.cli as common_cli + class Ga2VcfRunner(cli_client.SearchVariantsRunner): """ @@ -35,7 +37,7 @@ def run(self): def getGa2VcfParser(): - parser = cli.createArgumentParser(( + parser = common_cli.createArgumentParser(( "GA4GH VCF conversion tool. Converts variant information " "stored in a GA4GH repository into VCF format.")) cli_client.addClientGlobalOptions(parser) diff --git a/ga4gh/cli/repomanager.py b/ga4gh/cli/repomanager.py index 469751e1a..68190c188 100644 --- a/ga4gh/cli/repomanager.py +++ b/ga4gh/cli/repomanager.py @@ -27,6 +27,8 @@ import ga4gh.exceptions as exceptions import ga4gh.repo.rnaseq2ga as rnaseq2ga +import ga4gh_common.cli as common_cli + def getNameFromPath(filePath): """ @@ -712,29 +714,29 @@ def addRnaFeatureTypeOption(cls, subparser): @classmethod def getParser(cls): - parser = cli.createArgumentParser( + parser = common_cli.createArgumentParser( "GA4GH data repository management tool") subparsers = parser.add_subparsers(title='subcommands',) cli.addVersionArgument(parser) - initParser = cli.addSubparser( + initParser = common_cli.addSubparser( subparsers, "init", "Initialize a data repository") initParser.set_defaults(runner="init") cls.addRepoArgument(initParser) cls.addForceOption(initParser) - verifyParser = cli.addSubparser( + verifyParser = common_cli.addSubparser( subparsers, "verify", "Verifies the repository by examing all data files") verifyParser.set_defaults(runner="verify") cls.addRepoArgument(verifyParser) - listParser = cli.addSubparser( + listParser = common_cli.addSubparser( subparsers, "list", "List the contents of the repo") listParser.set_defaults(runner="list") cls.addRepoArgument(listParser) - addDatasetParser = cli.addSubparser( + addDatasetParser = common_cli.addSubparser( subparsers, "add-dataset", "Add a dataset to the data repo") addDatasetParser.set_defaults(runner="addDataset") cls.addRepoArgument(addDatasetParser) @@ -742,7 +744,7 @@ def getParser(cls): cls.addDatasetInfoArgument(addDatasetParser) cls.addDescriptionOption(addDatasetParser, "dataset") - removeDatasetParser = cli.addSubparser( + removeDatasetParser = common_cli.addSubparser( subparsers, "remove-dataset", "Remove a dataset from the data repo") removeDatasetParser.set_defaults(runner="removeDataset") @@ -751,7 +753,7 @@ def getParser(cls): cls.addForceOption(removeDatasetParser) objectType = "reference set" - addReferenceSetParser = cli.addSubparser( + addReferenceSetParser = common_cli.addSubparser( subparsers, "add-referenceset", "Add a reference set to the data repo") addReferenceSetParser.set_defaults(runner="addReferenceSet") @@ -778,7 +780,7 @@ def getParser(cls): "--sourceUri", default=None, help="The source URI") - removeReferenceSetParser = cli.addSubparser( + removeReferenceSetParser = common_cli.addSubparser( subparsers, "remove-referenceset", "Remove a reference set from the repo") removeReferenceSetParser.set_defaults(runner="removeReferenceSet") @@ -789,7 +791,7 @@ def getParser(cls): cls.addForceOption(removeReferenceSetParser) objectType = "ReadGroupSet" - addReadGroupSetParser = cli.addSubparser( + addReadGroupSetParser = common_cli.addSubparser( subparsers, "add-readgroupset", "Add a read group set to the data repo") addReadGroupSetParser.set_defaults(runner="addReadGroupSet") @@ -810,7 +812,7 @@ def getParser(cls): "file name. If the dataFile is a remote URL the path to " "a local file containing the BAM index must be provided")) - addOntologyParser = cli.addSubparser( + addOntologyParser = common_cli.addSubparser( subparsers, "add-ontology", "Adds an ontology in OBO format to the repo. Currently, " "a sequence ontology (SO) instance is required to translate " @@ -825,7 +827,7 @@ def getParser(cls): cls.addRelativePathOption(addOntologyParser) cls.addNameOption(addOntologyParser, "ontology") - removeOntologyParser = cli.addSubparser( + removeOntologyParser = common_cli.addSubparser( subparsers, "remove-ontology", "Remove an ontology from the repo") removeOntologyParser.set_defaults(runner="removeOntology") @@ -833,7 +835,7 @@ def getParser(cls): cls.addOntologyNameArgument(removeOntologyParser) cls.addForceOption(removeOntologyParser) - removeReadGroupSetParser = cli.addSubparser( + removeReadGroupSetParser = common_cli.addSubparser( subparsers, "remove-readgroupset", "Remove a read group set from the repo") removeReadGroupSetParser.set_defaults(runner="removeReadGroupSet") @@ -843,7 +845,7 @@ def getParser(cls): cls.addForceOption(removeReadGroupSetParser) objectType = "VariantSet" - addVariantSetParser = cli.addSubparser( + addVariantSetParser = common_cli.addSubparser( subparsers, "add-variantset", "Add a variant set to the data repo based on one or " "more VCF files. ") @@ -877,7 +879,7 @@ def getParser(cls): "If the supplied VCF file contains annotations, create the " "corresponding VariantAnnotationSet.")) - removeVariantSetParser = cli.addSubparser( + removeVariantSetParser = common_cli.addSubparser( subparsers, "remove-variantset", "Remove a variant set from the repo") removeVariantSetParser.set_defaults(runner="removeVariantSet") @@ -886,7 +888,7 @@ def getParser(cls): cls.addVariantSetNameArgument(removeVariantSetParser) cls.addForceOption(removeVariantSetParser) - addFeatureSetParser = cli.addSubparser( + addFeatureSetParser = common_cli.addSubparser( subparsers, "add-featureset", "Add a feature set to the data repo") addFeatureSetParser.set_defaults(runner="addFeatureSet") cls.addRepoArgument(addFeatureSetParser) @@ -900,7 +902,7 @@ def getParser(cls): cls.addSequenceOntologyNameOption(addFeatureSetParser, "feature set") cls.addClassNameOption(addFeatureSetParser, "feature set") - removeFeatureSetParser = cli.addSubparser( + removeFeatureSetParser = common_cli.addSubparser( subparsers, "remove-featureset", "Remove a feature set from the repo") removeFeatureSetParser.set_defaults(runner="removeFeatureSet") @@ -909,7 +911,7 @@ def getParser(cls): cls.addFeatureSetNameArgument(removeFeatureSetParser) cls.addForceOption(removeFeatureSetParser) - addBioSampleParser = cli.addSubparser( + addBioSampleParser = common_cli.addSubparser( subparsers, "add-biosample", "Add a BioSample to the dataset") addBioSampleParser.set_defaults(runner="addBioSample") cls.addRepoArgument(addBioSampleParser) @@ -917,7 +919,7 @@ def getParser(cls): cls.addBioSampleNameArgument(addBioSampleParser) cls.addBioSampleArgument(addBioSampleParser) - removeBioSampleParser = cli.addSubparser( + removeBioSampleParser = common_cli.addSubparser( subparsers, "remove-biosample", "Remove a BioSample from the repo") removeBioSampleParser.set_defaults(runner="removeBioSample") @@ -926,7 +928,7 @@ def getParser(cls): cls.addBioSampleNameArgument(removeBioSampleParser) cls.addForceOption(removeBioSampleParser) - addIndividualParser = cli.addSubparser( + addIndividualParser = common_cli.addSubparser( subparsers, "add-individual", "Add an Individual to the dataset") addIndividualParser.set_defaults(runner="addIndividual") cls.addRepoArgument(addIndividualParser) @@ -934,7 +936,7 @@ def getParser(cls): cls.addIndividualNameArgument(addIndividualParser) cls.addIndividualArgument(addIndividualParser) - removeIndividualParser = cli.addSubparser( + removeIndividualParser = common_cli.addSubparser( subparsers, "remove-individual", "Remove an Individual from the repo") removeIndividualParser.set_defaults(runner="removeIndividual") @@ -944,7 +946,7 @@ def getParser(cls): cls.addForceOption(removeIndividualParser) objectType = "RnaQuantification" - addRnaQuantificationParser = cli.addSubparser( + addRnaQuantificationParser = common_cli.addSubparser( subparsers, "add-rnaquantification", "Add an RNA quantification to the data repo") addRnaQuantificationParser.set_defaults( @@ -968,7 +970,7 @@ def getParser(cls): cls.addRnaFeatureTypeOption(addRnaQuantificationParser) objectType = "RnaQuantificationSet" - initRnaQuantificationSetParser = cli.addSubparser( + initRnaQuantificationSetParser = common_cli.addSubparser( subparsers, "init-rnaquantificationset", "Initializes an RNA quantification set") initRnaQuantificationSetParser.set_defaults( @@ -978,7 +980,7 @@ def getParser(cls): initRnaQuantificationSetParser, "The path to the resulting Quantification Set") - addRnaQuantificationSetParser = cli.addSubparser( + addRnaQuantificationSetParser = common_cli.addSubparser( subparsers, "add-rnaquantificationset", "Add an RNA quantification set to the data repo") addRnaQuantificationSetParser.set_defaults( @@ -992,7 +994,7 @@ def getParser(cls): addRnaQuantificationSetParser, objectType) cls.addNameOption(addRnaQuantificationSetParser, objectType) - removeRnaQuantificationSetParser = cli.addSubparser( + removeRnaQuantificationSetParser = common_cli.addSubparser( subparsers, "remove-rnaquantificationset", "Remove an RNA quantification set from the repo") removeRnaQuantificationSetParser.set_defaults( @@ -1003,7 +1005,7 @@ def getParser(cls): removeRnaQuantificationSetParser) cls.addForceOption(removeRnaQuantificationSetParser) - addPhenotypeAssociationSetParser = cli.addSubparser( + addPhenotypeAssociationSetParser = common_cli.addSubparser( subparsers, "add-phenotypeassociationset", "Adds phenotypes in ttl format to the repo.") addPhenotypeAssociationSetParser.set_defaults( @@ -1017,7 +1019,7 @@ def getParser(cls): addPhenotypeAssociationSetParser, "PhenotypeAssociationSet") - removePhenotypeAssociationSetParser = cli.addSubparser( + removePhenotypeAssociationSetParser = common_cli.addSubparser( subparsers, "remove-phenotypeassociationset", "Remove an phenotypes from the repo") removePhenotypeAssociationSetParser.set_defaults( diff --git a/ga4gh/cli/server.py b/ga4gh/cli/server.py index c281e07fa..d08dc12a3 100644 --- a/ga4gh/cli/server.py +++ b/ga4gh/cli/server.py @@ -10,6 +10,8 @@ import ga4gh.cli as cli import ga4gh.frontend as frontend +import ga4gh_common.cli as common_cli + def addServerOptions(parser): parser.add_argument( @@ -35,7 +37,7 @@ def addServerOptions(parser): def getServerParser(): - parser = cli.createArgumentParser("GA4GH reference server") + parser = common_cli.createArgumentParser("GA4GH reference server") addServerOptions(parser) return parser diff --git a/oidc-provider/simple_op/src/provider/authn/__init__.py b/oidc-provider/simple_op/src/provider/authn/__init__.py index ac8929110..4c32a6013 100644 --- a/oidc-provider/simple_op/src/provider/authn/__init__.py +++ b/oidc-provider/simple_op/src/provider/authn/__init__.py @@ -3,6 +3,7 @@ __author__ = 'regu0004' + class AuthnModule(UserAuthnMethod): # override in subclass specifying suitable url endpoint to POST user input diff --git a/oidc-provider/simple_op/src/provider/authn/two_factor.py b/oidc-provider/simple_op/src/provider/authn/two_factor.py index 254d8ea2d..fbfb150a9 100644 --- a/oidc-provider/simple_op/src/provider/authn/two_factor.py +++ b/oidc-provider/simple_op/src/provider/authn/two_factor.py @@ -95,4 +95,4 @@ def _send_mail(self, code, receiver): s = smtplib.SMTP(self.smtp_server) s.sendmail(self.outgoing_sender, [receiver], msg.as_string()) - s.quit() \ No newline at end of file + s.quit() diff --git a/oidc-provider/simple_op/src/provider/authn/user_pass.py b/oidc-provider/simple_op/src/provider/authn/user_pass.py index 8879584d2..e777bcb4f 100644 --- a/oidc-provider/simple_op/src/provider/authn/user_pass.py +++ b/oidc-provider/simple_op/src/provider/authn/user_pass.py @@ -8,7 +8,8 @@ class UserPass(AuthnModule): url_endpoint = "/user_pass/verify" - def __init__(self, db, template_env, template="user_pass.jinja2", **kwargs): + def __init__( + self, db, template_env, template="user_pass.jinja2", **kwargs): super(UserPass, self).__init__(None) self.template_env = template_env self.template = template @@ -31,7 +32,7 @@ def __call__(self, *args, **kwargs): def verify(self, *args, **kwargs): username = kwargs["username"] if username in self.user_db and self.user_db[username] == kwargs[ - "password"]: + "password"]: return username, True else: return self.FAILED_AUTHN diff --git a/oidc-provider/simple_op/src/provider/authn/util.py b/oidc-provider/simple_op/src/provider/authn/util.py index 4884b9d29..822d7da0f 100644 --- a/oidc-provider/simple_op/src/provider/authn/util.py +++ b/oidc-provider/simple_op/src/provider/authn/util.py @@ -10,4 +10,4 @@ def __getitem__(self, item): return self._db[item] def __contains__(self, item): - return item in self._db \ No newline at end of file + return item in self._db diff --git a/oidc-provider/simple_op/src/provider/authn/yubikey.py b/oidc-provider/simple_op/src/provider/authn/yubikey.py index 40f3ce3b8..d16b2a39b 100644 --- a/oidc-provider/simple_op/src/provider/authn/yubikey.py +++ b/oidc-provider/simple_op/src/provider/authn/yubikey.py @@ -12,9 +12,10 @@ class YubicoOTP(AuthnModule): url_endpoint = "/yubi_otp/verify" - def __init__(self, yubikey_db, validation_server, client_id, template_env, - secret_key=None, verify_ssl=True, template="yubico_otp.jinja2", - **kwargs): + def __init__( + self, yubikey_db, validation_server, client_id, template_env, + secret_key=None, verify_ssl=True, template="yubico_otp.jinja2", + **kwargs): super(YubicoOTP, self).__init__(None) self.template_env = template_env self.template = template @@ -58,4 +59,5 @@ def verify(self, *args, **kwargs): return self.yubikey_db[yubikey_public_id], True else: logger.error( - "No response from the servers or received other negative status code") + "No response from the servers " + "or received other negative status code") diff --git a/oidc-provider/simple_op/src/provider/server/server.py b/oidc-provider/simple_op/src/provider/server/server.py index bfd073a2b..10111db83 100644 --- a/oidc-provider/simple_op/src/provider/server/server.py +++ b/oidc-provider/simple_op/src/provider/server/server.py @@ -140,7 +140,8 @@ def _webfinger(provider, request, **kwargs): def make_static_handler(static_dir): def static(environ, start_response): path = environ['PATH_INFO'] - full_path = os.path.join(static_dir, os.path.normpath(path).lstrip("/")) + full_path = os.path.join( + static_dir, os.path.normpath(path).lstrip("/")) if os.path.exists(full_path): with open(full_path, 'rb') as f: @@ -174,8 +175,8 @@ def main(): template_dirs = settings["server"].get("template_dirs", "templates") jinja_env = Environment(loader=FileSystemLoader(template_dirs)) - authn_broker, auth_routing = setup_authentication_methods(settings["authn"], - jinja_env) + authn_broker, auth_routing = setup_authentication_methods( + settings["authn"], jinja_env) # Setup userinfo userinfo_conf = settings["userinfo"] @@ -231,4 +232,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/oidc-provider/simple_op/src/run.py b/oidc-provider/simple_op/src/run.py index 253f16060..4a296bc18 100644 --- a/oidc-provider/simple_op/src/run.py +++ b/oidc-provider/simple_op/src/run.py @@ -1,4 +1,4 @@ from provider.server.server import main if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/requirements.txt b/requirements.txt index 38709fc67..f933ddcde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,8 @@ # these libraries are the set listed by pipdeptree -f -w # that are dependencies of libraries listed in the next section +ga4gh-common==0.0.4 + Werkzeug==0.11.5 MarkupSafe==0.23 itsdangerous==0.24 diff --git a/scripts/alignment_file.py b/scripts/alignment_file.py new file mode 100644 index 000000000..a8c95b3c2 --- /dev/null +++ b/scripts/alignment_file.py @@ -0,0 +1,87 @@ +""" +Tools for alignment files +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import pysam + +import ga4gh_common.utils as utils + + +class AlignmentFileConstants(object): + """ + A container class for constants dealing with alignment files + """ + SAM = "SAM" + BAM = "BAM" + BAI = "BAI" + + +class AlignmentFileTool(object): + """ + Helps with operations on BAM and SAM files + """ + def __init__(self, inputFileFormat, outputFileFormat): + self.inputFileFormat = inputFileFormat + self.outputFileFormat = outputFileFormat + self.args = None + + def parseArgs(self): + description = "{} to {} conversion tool".format( + self.inputFileFormat, self.outputFileFormat) + parser = argparse.ArgumentParser( + description=description) + inputHelpText = "the name of the {} file to read".format( + self.inputFileFormat) + parser.add_argument( + "inputFile", help=inputHelpText) + outputHelpText = "the name of the {} file to write".format( + self.outputFileFormat) + defaultOutputFilePath = "out.{}".format( + self.outputFileFormat.lower()) + parser.add_argument( + "--outputFile", "-o", default=defaultOutputFilePath, + help=outputHelpText) + parser.add_argument( + "--numLines", "-n", default=10, + help="the number of lines to write") + parser.add_argument( + "--skipIndexing", default=False, action='store_true', + help="don't create an index file") + args = parser.parse_args() + self.args = args + + def convert(self): + # set flags + if self.inputFileFormat == AlignmentFileConstants.SAM: + inputFlags = "r" + elif self.inputFileFormat == AlignmentFileConstants.BAM: + inputFlags = "rb" + if self.outputFileFormat == AlignmentFileConstants.SAM: + outputFlags = "wh" + elif self.outputFileFormat == AlignmentFileConstants.BAM: + outputFlags = "wb" + # open files + inputFile = pysam.AlignmentFile( + self.args.inputFile, inputFlags) + outputFile = pysam.AlignmentFile( + self.args.outputFile, outputFlags, header=inputFile.header) + outputFilePath = outputFile.filename + utils.log("Creating alignment file '{}'".format(outputFilePath)) + # write new file + for _ in xrange(self.args.numLines): + alignedSegment = inputFile.next() + outputFile.write(alignedSegment) + # clean up + inputFile.close() + outputFile.close() + # create index file + if (not self.args.skipIndexing and + self.outputFileFormat == AlignmentFileConstants.BAM): + indexFilePath = "{}.{}".format( + outputFilePath, AlignmentFileConstants.BAI.lower()) + utils.log("Creating index file '{}'".format(indexFilePath)) + pysam.index(outputFilePath) diff --git a/scripts/bam2bam.py b/scripts/bam2bam.py index 9810c6eba..3e630305a 100644 --- a/scripts/bam2bam.py +++ b/scripts/bam2bam.py @@ -5,7 +5,7 @@ from __future__ import print_function from __future__ import unicode_literals -import utils +import ga4gh_common.utils as utils @utils.Timed() diff --git a/scripts/bam2sam.py b/scripts/bam2sam.py index 1dbdd1dea..d20a21bf1 100644 --- a/scripts/bam2sam.py +++ b/scripts/bam2sam.py @@ -5,7 +5,7 @@ from __future__ import print_function from __future__ import unicode_literals -import utils +import ga4gh_common.utils as utils @utils.Timed() diff --git a/scripts/build_test_data.py b/scripts/build_test_data.py index 88076de46..c61779482 100644 --- a/scripts/build_test_data.py +++ b/scripts/build_test_data.py @@ -9,7 +9,7 @@ import glob import os.path -import utils +import ga4gh_common.utils as utils def run(*args): diff --git a/scripts/demo_example.py b/scripts/demo_example.py index 502cdcf98..f62a3c602 100755 --- a/scripts/demo_example.py +++ b/scripts/demo_example.py @@ -6,8 +6,9 @@ from __future__ import print_function from __future__ import unicode_literals -import utils -utils.ga4ghImportGlue() +import glue + +glue.ga4ghImportGlue() import ga4gh.client as client # NOQA diff --git a/scripts/download_example_data.py b/scripts/download_example_data.py index 93c3a5cf7..d3800a913 100644 --- a/scripts/download_example_data.py +++ b/scripts/download_example_data.py @@ -17,8 +17,10 @@ import pysam -import utils -utils.ga4ghImportGlue() +import ga4gh_common.utils as utils +import glue + +glue.ga4ghImportGlue() # We need to turn off QA because of the import glue import ga4gh.datarepo as datarepo # NOQA diff --git a/scripts/file_downloader.py b/scripts/file_downloader.py new file mode 100644 index 000000000..44699b662 --- /dev/null +++ b/scripts/file_downloader.py @@ -0,0 +1,100 @@ +""" +Simple tool to download files +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import sys + +import humanize + +import requests.packages.urllib3 +requests.packages.urllib3.disable_warnings() + + +class FileDownloader(object): + """ + Base class for file downloaders of different protocols + """ + defaultStream = sys.stdout + + def __init__(self, url, path, stream=defaultStream): + self.url = url + self.path = path + self.basename = path + self.basenameLength = len(self.basename) + self.stream = stream + self.bytesReceived = 0 + self.displayIndex = 0 + self.displayWindowSize = 20 + self.fileSize = None + self.displayCounter = 0 + + def _printStartDownloadMessage(self): + self.stream.write("Downloading '{}' to '{}'\n".format( + self.url, self.path)) + + def _cleanUp(self): + self.stream.write("\n") + self.stream.flush() + + def _getFileNameDisplayString(self): + if self.basenameLength <= self.displayWindowSize: + return self.basename + else: + return self.basename # TODO scrolling window here + + def _updateDisplay(self, modulo=1): + self.displayCounter += 1 + if self.displayCounter % modulo != 0: + return + fileName = self._getFileNameDisplayString() + if self.fileSize is None: + displayString = "{} bytes received: {}\r" + bytesReceived = humanize.filesize.naturalsize( + self.bytesReceived) + self.stream.write(displayString.format( + fileName, bytesReceived)) + else: + # TODO contentlength seems to slightly under-report how many + # bytes we have to download... hence the min functions + percentage = min(self.bytesReceived / self.fileSize, 1) + numerator = humanize.filesize.naturalsize( + min(self.bytesReceived, self.fileSize)) + denominator = humanize.filesize.naturalsize( + self.fileSize) + displayString = "{} {:<6.2%} ({:>9} / {:<9})\r" + self.stream.write(displayString.format( + fileName, percentage, numerator, denominator)) + self.stream.flush() + + +class HttpFileDownloader(FileDownloader): + """ + Provides a wget-like file download and terminal display for HTTP + """ + defaultChunkSize = 1048576 # 1MB + + def __init__(self, url, path, chunkSize=defaultChunkSize, + stream=FileDownloader.defaultStream): + super(HttpFileDownloader, self).__init__( + url, path, stream) + self.chunkSize = chunkSize + + def download(self): + self._printStartDownloadMessage() + response = requests.get(self.url, stream=True) + response.raise_for_status() + try: + contentLength = int(response.headers['content-length']) + self.fileSize = contentLength + except KeyError: + # chunked transfer encoding + pass + with open(self.path, 'wb') as outputFile: + for chunk in response.iter_content(chunk_size=self.chunkSize): + self.bytesReceived += self.chunkSize + self._updateDisplay() + outputFile.write(chunk) + self._cleanUp() diff --git a/scripts/generate_fasta.py b/scripts/generate_fasta.py index cc9e4a888..60fa04141 100644 --- a/scripts/generate_fasta.py +++ b/scripts/generate_fasta.py @@ -12,7 +12,7 @@ import os import random -import utils +import ga4gh_common.utils as utils class FastaGenerator(object): diff --git a/scripts/generate_gff3_db.py b/scripts/generate_gff3_db.py index e31bd87f7..d18c98bd5 100644 --- a/scripts/generate_gff3_db.py +++ b/scripts/generate_gff3_db.py @@ -16,8 +16,10 @@ import json import sqlite3 -import utils -utils.ga4ghImportGlue() +import ga4gh_common.utils as utils +import glue + +glue.ga4ghImportGlue() import ga4gh.gff3 as gff3 # NOQA # TODO: Shift this to use the Gff3DbBackend class. diff --git a/scripts/glue.py b/scripts/glue.py new file mode 100644 index 000000000..8c3b2e460 --- /dev/null +++ b/scripts/glue.py @@ -0,0 +1,20 @@ +""" +Glue to enable script access to ga4gh packages +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import os +import sys + + +def ga4ghImportGlue(): + """ + Call this method before importing a ga4gh module in the scripts dir. + Otherwise, you will be using the installed package instead of + the development package. + Assumes a certain directory structure. + """ + path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + sys.path.append(path) diff --git a/scripts/prepare_compliance_data.py b/scripts/prepare_compliance_data.py index dab6a0972..cfec18f1b 100644 --- a/scripts/prepare_compliance_data.py +++ b/scripts/prepare_compliance_data.py @@ -13,13 +13,17 @@ import shutil import json import pysam -import utils import generate_gff3_db import tempfile import zipfile import glob -utils.ga4ghImportGlue() +import file_downloader + +import ga4gh_common.utils as utils +import glue + +glue.ga4ghImportGlue() # We need to turn off QA because of the import glue import ga4gh # NOQA @@ -77,7 +81,7 @@ def __init__(self, inputDirectory, outputDirectory, force): assert(os.path.exists(self.tempdir)) url = "https://github.com/ga4gh/compliance/archive/master.zip" filePath = os.path.join(self.tempdir, 'compliance-master.zip') - downloader = utils.HttpFileDownloader(url, filePath) + downloader = file_downloader.HttpFileDownloader(url, filePath) downloader.download() utils.log("Extracting test data...") with zipfile.ZipFile(filePath, "r") as z: diff --git a/scripts/run_tests.py b/scripts/run_tests.py deleted file mode 100644 index 047ec9b4c..000000000 --- a/scripts/run_tests.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Script to imitate a Travis CI run -""" - -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import subprocess - -import yaml - -import utils -import build_test_data - - -class TravisSimulator(object): - - logStrPrefix = '***' - - yamlFileLocation = '.travis.yml' - - def parseTestCommands(self): - yamlFile = file(self.yamlFileLocation) - yamlData = yaml.load(yamlFile) - return yamlData['script'] - - def runTests(self): - testCommands = self.parseTestCommands() - for command in testCommands: - self.log('Running: "{}"'.format(command)) - subprocess.check_call(command, shell=True) - self.log('SUCCESS') - - def log(self, logStr): - utils.log("{0} {1}".format(self.logStrPrefix, logStr)) - - -if __name__ == '__main__': - build_test_data.buildTestData() - travisSimulator = TravisSimulator() - travisSimulator.runTests() diff --git a/scripts/server_benchmark.py b/scripts/server_benchmark.py index 9238c78c6..67994fc1c 100644 --- a/scripts/server_benchmark.py +++ b/scripts/server_benchmark.py @@ -13,8 +13,9 @@ import guppy -import utils -utils.ga4ghImportGlue() +import glue + +glue.ga4ghImportGlue() import ga4gh.backend as backend # noqa import ga4gh.protocol as protocol # noqa import ga4gh.datarepo as datarepo # noqa diff --git a/scripts/split_fasta.py b/scripts/split_fasta.py index 5720d524a..1bc986ca6 100644 --- a/scripts/split_fasta.py +++ b/scripts/split_fasta.py @@ -8,7 +8,7 @@ import argparse import os -import utils +import ga4gh_common.utils as utils def parseArgs(): diff --git a/scripts/update_data.py b/scripts/update_data.py index 1b770a712..9e2dc13a2 100644 --- a/scripts/update_data.py +++ b/scripts/update_data.py @@ -10,7 +10,9 @@ import tarfile import time -import utils +import file_downloader + +import ga4gh_common.utils as utils dataDirPath = 'ga4gh-example-data' @@ -43,7 +45,7 @@ def archiveExistingData(): def downloadData(): url = ("https://github.com/ga4gh/server/releases/" "download/data/ga4gh-example-data-v3.2.tar") - fileDownloader = utils.HttpFileDownloader(url, tarballPath) + fileDownloader = file_downloader.HttpFileDownloader(url, tarballPath) fileDownloader.download() utils.log("Downloading finished") diff --git a/scripts/utils.py b/scripts/utils.py deleted file mode 100644 index fbf6bc43b..000000000 --- a/scripts/utils.py +++ /dev/null @@ -1,293 +0,0 @@ -""" -Utilities for scripts -""" -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import functools -import os -import shlex -import subprocess -import sys -import time - -import humanize -import requests -import yaml -import pysam - -import requests.packages.urllib3 -requests.packages.urllib3.disable_warnings() - - -def getPathOfExecutable(executable): - """ - Returns the full path of the executable, or None if the executable - can not be found. - """ - exe_paths = os.environ['PATH'].split(':') - for exe_path in exe_paths: - exe_file = os.path.join(exe_path, executable) - if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): - return exe_file - return None - - -def requireExecutables(executables): - """ - Check that all of the given executables are on the path. - If at least one of them is not, exit the script and inform - the user of the missing requirement(s). - """ - missingExecutables = [] - for executable in executables: - if getPathOfExecutable(executable) is None: - missingExecutables.append(executable) - if len(missingExecutables) > 0: - log("In order to run this script, the following " - "executables need to be on the path:") - for missingExecutable in missingExecutables: - print(missingExecutable) - exit(1) - - -def ga4ghImportGlue(): - """ - Call this method before importing a ga4gh module in the scripts dir. - Otherwise, you will be using the installed package instead of - the development package. - Assumes a certain directory structure. - """ - path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - sys.path.append(path) - - -def log(message): - print(message) - - -class Timed(object): - """ - Decorator that times a method, reporting runtime at finish - """ - def __call__(self, func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - self.start = time.time() - result = func(*args, **kwargs) - self.end = time.time() - self._report() - return result - return wrapper - - def _report(self): - delta = self.end - self.start - timeString = humanize.time.naturaldelta(delta) - log("Finished in {} ({:.2f} seconds)".format(timeString, delta)) - - -class FileDownloader(object): - """ - Base class for file downloaders of different protocols - """ - defaultStream = sys.stdout - - def __init__(self, url, path, stream=defaultStream): - self.url = url - self.path = path - self.basename = path - self.basenameLength = len(self.basename) - self.stream = stream - self.bytesReceived = 0 - self.displayIndex = 0 - self.displayWindowSize = 20 - self.fileSize = None - self.displayCounter = 0 - - def _printStartDownloadMessage(self): - self.stream.write("Downloading '{}' to '{}'\n".format( - self.url, self.path)) - - def _cleanUp(self): - self.stream.write("\n") - self.stream.flush() - - def _getFileNameDisplayString(self): - if self.basenameLength <= self.displayWindowSize: - return self.basename - else: - return self.basename # TODO scrolling window here - - def _updateDisplay(self, modulo=1): - self.displayCounter += 1 - if self.displayCounter % modulo != 0: - return - fileName = self._getFileNameDisplayString() - if self.fileSize is None: - displayString = "{} bytes received: {}\r" - bytesReceived = humanize.filesize.naturalsize( - self.bytesReceived) - self.stream.write(displayString.format( - fileName, bytesReceived)) - else: - # TODO contentlength seems to slightly under-report how many - # bytes we have to download... hence the min functions - percentage = min(self.bytesReceived / self.fileSize, 1) - numerator = humanize.filesize.naturalsize( - min(self.bytesReceived, self.fileSize)) - denominator = humanize.filesize.naturalsize( - self.fileSize) - displayString = "{} {:<6.2%} ({:>9} / {:<9})\r" - self.stream.write(displayString.format( - fileName, percentage, numerator, denominator)) - self.stream.flush() - - -class HttpFileDownloader(FileDownloader): - """ - Provides a wget-like file download and terminal display for HTTP - """ - defaultChunkSize = 1048576 # 1MB - - def __init__(self, url, path, chunkSize=defaultChunkSize, - stream=FileDownloader.defaultStream): - super(HttpFileDownloader, self).__init__( - url, path, stream) - self.chunkSize = chunkSize - - def download(self): - self._printStartDownloadMessage() - response = requests.get(self.url, stream=True) - response.raise_for_status() - try: - contentLength = int(response.headers['content-length']) - self.fileSize = contentLength - except KeyError: - # chunked transfer encoding - pass - with open(self.path, 'wb') as outputFile: - for chunk in response.iter_content(chunk_size=self.chunkSize): - self.bytesReceived += self.chunkSize - self._updateDisplay() - outputFile.write(chunk) - self._cleanUp() - - -def runCommandSplits(splits, silent=False): - """ - Run a shell command given the command's parsed command line - """ - try: - if silent: - with open(os.devnull, 'w') as devnull: - subprocess.check_call(splits, stdout=devnull, stderr=devnull) - else: - subprocess.check_call(splits) - except OSError, e: - if e.errno == 2: # cmd not found - raise Exception( - "Can't find command while trying to run {}".format(splits)) - else: - raise - - -def runCommand(command, silent=False): - """ - Run a shell command - """ - splits = shlex.split(command) - runCommandSplits(splits, silent=silent) - - -def getAuthValues(filePath='scripts/auth.yml'): - """ - Return the script authentication file as a dictionary - """ - return getYamlDocument(filePath) - - -def getYamlDocument(filePath): - """ - Return a yaml file's contents as a dictionary - """ - with open(filePath) as stream: - doc = yaml.load(stream) - return doc - - -class AlignmentFileConstants(object): - """ - A container class for constants dealing with alignment files - """ - SAM = "SAM" - BAM = "BAM" - BAI = "BAI" - - -class AlignmentFileTool(object): - """ - Helps with operations on BAM and SAM files - """ - def __init__(self, inputFileFormat, outputFileFormat): - self.inputFileFormat = inputFileFormat - self.outputFileFormat = outputFileFormat - self.args = None - - def parseArgs(self): - description = "{} to {} conversion tool".format( - self.inputFileFormat, self.outputFileFormat) - parser = argparse.ArgumentParser( - description=description) - inputHelpText = "the name of the {} file to read".format( - self.inputFileFormat) - parser.add_argument( - "inputFile", help=inputHelpText) - outputHelpText = "the name of the {} file to write".format( - self.outputFileFormat) - defaultOutputFilePath = "out.{}".format( - self.outputFileFormat.lower()) - parser.add_argument( - "--outputFile", "-o", default=defaultOutputFilePath, - help=outputHelpText) - parser.add_argument( - "--numLines", "-n", default=10, - help="the number of lines to write") - parser.add_argument( - "--skipIndexing", default=False, action='store_true', - help="don't create an index file") - args = parser.parse_args() - self.args = args - - def convert(self): - # set flags - if self.inputFileFormat == AlignmentFileConstants.SAM: - inputFlags = "r" - elif self.inputFileFormat == AlignmentFileConstants.BAM: - inputFlags = "rb" - if self.outputFileFormat == AlignmentFileConstants.SAM: - outputFlags = "wh" - elif self.outputFileFormat == AlignmentFileConstants.BAM: - outputFlags = "wb" - # open files - inputFile = pysam.AlignmentFile( - self.args.inputFile, inputFlags) - outputFile = pysam.AlignmentFile( - self.args.outputFile, outputFlags, header=inputFile.header) - outputFilePath = outputFile.filename - log("Creating alignment file '{}'".format(outputFilePath)) - # write new file - for _ in xrange(self.args.numLines): - alignedSegment = inputFile.next() - outputFile.write(alignedSegment) - # clean up - inputFile.close() - outputFile.close() - # create index file - if (not self.args.skipIndexing and - self.outputFileFormat == AlignmentFileConstants.BAM): - indexFilePath = "{}.{}".format( - outputFilePath, AlignmentFileConstants.BAI.lower()) - log("Creating index file '{}'".format(indexFilePath)) - pysam.index(outputFilePath) diff --git a/setup.py b/setup.py index 93aef571a..bd740c4f0 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +# Don't import __future__ packages here; they make setup fail + # First, we try to use setuptools. If it's not available locally, # we fall back on ez_setup. try: @@ -22,16 +24,13 @@ install_requires.append(pinnedVersion) setup( + # END BOILERPLATE name="ga4gh", description="A reference implementation of the ga4gh API", - license='Apache License 2.0', - long_description=long_description, packages=["ga4gh", "ga4gh.datamodel", "ga4gh.templates"], - include_package_data=True, zip_safe=False, - author="Global Alliance for Genomics and Health", - author_email="theglobalalliance@genomicsandhealth.org", url="https://github.com/ga4gh/server", + use_scm_version={"write_to": "ga4gh/_version.py"}, entry_points={ 'console_scripts': [ 'ga4gh_client=ga4gh.cli.client:client_main', @@ -42,6 +41,13 @@ 'ga4gh_repo=ga4gh.cli.repomanager:repo_main', ] }, + # BEGIN BOILERPLATE + long_description=long_description, + install_requires=install_requires, + license='Apache License 2.0', + include_package_data=True, + author="Global Alliance for Genomics and Health", + author_email="theglobalalliance@genomicsandhealth.org", classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Science/Research', @@ -50,11 +56,7 @@ 'Programming Language :: Python :: 2.7', 'Topic :: Scientific/Engineering :: Bio-Informatics', ], - keywords='genomics reference', - install_requires=install_requires, + keywords=['genomics', 'reference'], # Use setuptools_scm to set the version number automatically from Git setup_requires=['setuptools_scm'], - use_scm_version={ - "write_to": "ga4gh/_version.py" - }, ) diff --git a/tests/datadriven/test_reads.py b/tests/datadriven/test_reads.py index bdc94cb43..daba4cbe6 100644 --- a/tests/datadriven/test_reads.py +++ b/tests/datadriven/test_reads.py @@ -16,7 +16,7 @@ import ga4gh.protocol as protocol import ga4gh.datarepo as datarepo import tests.datadriven as datadriven -import tests.utils as utils +import ga4gh_common.utils as utils import tests.paths as paths import pysam diff --git a/tests/datadriven/test_variants.py b/tests/datadriven/test_variants.py index 77205bb0c..a19f5c25e 100644 --- a/tests/datadriven/test_variants.py +++ b/tests/datadriven/test_variants.py @@ -18,7 +18,7 @@ import ga4gh.protocol as protocol import ga4gh.exceptions as exceptions import tests.datadriven as datadriven -import tests.utils as utils +import ga4gh_common.utils as utils import tests.paths as paths diff --git a/tests/end_to_end/client.py b/tests/end_to_end/client.py index bfc632b1e..e537f306a 100644 --- a/tests/end_to_end/client.py +++ b/tests/end_to_end/client.py @@ -9,7 +9,7 @@ import subprocess import tempfile -import tests.utils as utils +import ga4gh_common.utils as utils class ClientForTesting(object): diff --git a/tests/end_to_end/server.py b/tests/end_to_end/server.py index 94b62609a..5662476e8 100644 --- a/tests/end_to_end/server.py +++ b/tests/end_to_end/server.py @@ -13,7 +13,7 @@ import requests -import tests.utils as utils +import ga4gh_common.utils as utils ga4ghPort = 8001 diff --git a/tests/end_to_end/test_client_json.py b/tests/end_to_end/test_client_json.py index 0ca4a272e..5e24809b0 100644 --- a/tests/end_to_end/test_client_json.py +++ b/tests/end_to_end/test_client_json.py @@ -16,7 +16,7 @@ import ga4gh.cli.client as cli_client import ga4gh.protocol as protocol import ga4gh.datarepo as datarepo -import tests.utils as utils +import ga4gh_common.utils as utils import tests.paths as paths import freezegun diff --git a/tests/paths.py b/tests/paths.py index 926d7665d..efc26a401 100644 --- a/tests/paths.py +++ b/tests/paths.py @@ -8,6 +8,18 @@ import os +packageName = 'ga4gh' + + +def getProjectRootFilePath(): + # assumes we're in a directory one level below the project root + return os.path.dirname(os.path.dirname(__file__)) + + +def getGa4ghFilePath(): + return os.path.join(getProjectRootFilePath(), packageName) + + testDir = 'tests' testDataDir = os.path.join(testDir, 'data') testDataRepo = os.path.join(testDataDir, 'registry.db') diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0f66633b6..fa4d5415c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -18,7 +18,7 @@ import ga4gh.protocol as protocol import google.protobuf.descriptor as descriptor import google.protobuf.internal.python_message as python_message -import tests.utils as utils +import ga4gh_common.utils as utils class TestServerArguments(unittest.TestCase): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 2e413915b..d354d9233 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -14,7 +14,7 @@ import ga4gh.backend as backend import ga4gh.client as client import ga4gh.datarepo as datarepo -import tests.utils as utils +import ga4gh_common.utils as utils import ga4gh.exceptions as exceptions diff --git a/tests/unit/test_converters.py b/tests/unit/test_converters.py index aeabbfb56..0ceea16b1 100644 --- a/tests/unit/test_converters.py +++ b/tests/unit/test_converters.py @@ -15,7 +15,7 @@ import ga4gh.converters as converters import ga4gh.datarepo as datarepo import tests.paths as paths -import tests.utils as utils +import ga4gh_common.utils as utils class TestSamConverter(unittest.TestCase): diff --git a/tests/unit/test_data_interface.py b/tests/unit/test_data_interface.py index 06ae3103d..a5d619c8c 100644 --- a/tests/unit/test_data_interface.py +++ b/tests/unit/test_data_interface.py @@ -11,7 +11,7 @@ import ga4gh.datarepo as datarepo import ga4gh.backend as backend import tests.paths as paths -import tests.utils as utils +import ga4gh_common.utils as utils class TestInterfacingLayer(unittest.TestCase): diff --git a/tests/unit/test_imports.py b/tests/unit/test_imports.py index 4b69e12c2..c1c289471 100644 --- a/tests/unit/test_imports.py +++ b/tests/unit/test_imports.py @@ -21,7 +21,7 @@ from snakefood.fallback.collections import defaultdict from snakefood.roots import find_roots, relfile -import tests.utils as utils +import tests.paths as paths class TestImports(unittest.TestCase): @@ -248,7 +248,7 @@ def _allModules(self): def _checkConfiguration(self): # each module that exists in the file tree appears in moduleGroupNames pythonFiles = [] - for root, dirnames, filenames in os.walk(utils.getGa4ghFilePath()): + for root, dirnames, filenames in os.walk(paths.getGa4ghFilePath()): for filename in fnmatch.filter(filenames, '*.py'): pythonFilename = os.path.relpath( os.path.join(root, filename)) @@ -444,7 +444,7 @@ def __init__(self): self.optsIgnores = ['.svn', 'CVS', 'build', '.hg', '.git'] self.optsPrintRoots = None self.optsFollow = True - self.args = [utils.packageName] + self.args = [paths.packageName] def scan(self): """ diff --git a/tests/unit/test_oidc.py b/tests/unit/test_oidc.py index cc96007fa..2c91aebcc 100644 --- a/tests/unit/test_oidc.py +++ b/tests/unit/test_oidc.py @@ -16,7 +16,7 @@ import oic.oic.message as message import ga4gh.frontend as frontend -import tests.utils as utils +import ga4gh_common.utils as utils RANDSTR = 'abc' OICCODE = ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py deleted file mode 100644 index c0a05f803..000000000 --- a/tests/unit/test_utils.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Unit tests for test utility functions. -""" -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import unittest -import sys -import time - -import tests.utils as utils - - -class TestRepeat(unittest.TestCase): - """ - Test the Repeat decorator - """ - # lowest possible positive number - sleepSeconds = sys.float_info.min - - @utils.Repeat(sleepSeconds) - def repeating(self): - self.i += 1 - return self.i < self.endIterationAt - - def testRepeat(self): - self.i = 0 - self.endIterationAt = 3 - self.repeating() - self.assertEqual(self.i, self.endIterationAt) - - -class TestTimeout(unittest.TestCase): - """ - Test the Timeout decorator - """ - # lowest positive value signal.alarm allows - timeoutSecondsLow = 1 - - @utils.Timeout(timeoutSecondsLow) - def timingOut(self): - time.sleep(60) # much longer than the timeout - - @utils.Timeout() - def notTimingOut(self): - return 0 - - def testTimeoutException(self): - with self.assertRaises(utils.TimeoutException): - self.timingOut() - - def testTimeoutNoException(self): - self.assertEquals(self.notTimingOut(), 0) - # no exception thrown - - -class TestCaptureOutput(unittest.TestCase): - """ - Test that the captureOutput correctly returns the value of stdout - and stderr for a function. - """ - - def testCapture(self): - stdoutValue = "stdout" - stderrValue = "stderr" - - def func(): - print(stdoutValue, file=sys.stdout, end="") - print(stderrValue, file=sys.stderr, end="") - stdout, stderr = utils.captureOutput(func) - self.assertEqual(stdout, stdoutValue) - self.assertEqual(stderr, stderrValue) - - # Empty stdout - def func(): - print(stderrValue, file=sys.stderr, end="") - stdout, stderr = utils.captureOutput(func) - self.assertEqual(stdout, "") - self.assertEqual(stderr, stderrValue) - - # Empty stderr - def func(): - print(stdoutValue, file=sys.stdout, end="") - stdout, stderr = utils.captureOutput(func) - self.assertEqual(stdout, stdoutValue) - self.assertEqual(stderr, "") - - def testArgs(self): - def func(one, two, three, keywordOne=None, keywordTwo=None): - print(one, two, three, keywordOne, keywordTwo, file=sys.stdout) - stdout, stderr = utils.captureOutput( - func, "1", "2", "3", keywordTwo="5", keywordOne="4") - self.assertEqual(stdout, "1 2 3 4 5\n") - self.assertEqual(stderr, "") diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index 3d3c681f6..000000000 --- a/tests/utils.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Functions and utility classes for testing. -""" -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import StringIO -import functools -import humanize -import itertools -import os -import signal -import sys -import time -import contextlib - - -packageName = 'ga4gh' - - -def captureOutput(func, *args, **kwargs): - """ - Runs the specified function and arguments, and returns the - tuple (stdout, stderr) as strings. - """ - stdout = sys.stdout - sys.stdout = StringIO.StringIO() - stderr = sys.stderr - sys.stderr = StringIO.StringIO() - try: - func(*args, **kwargs) - stdoutOutput = sys.stdout.getvalue() - stderrOutput = sys.stderr.getvalue() - finally: - sys.stdout.close() - sys.stdout = stdout - sys.stderr.close() - sys.stderr = stderr - return stdoutOutput, stderrOutput - - -def zipLists(*lists): - """ - Checks to see if all of the lists are the same length, and throws - an AssertionError otherwise. Returns the zipped lists. - """ - length = len(lists[0]) - for i, list_ in enumerate(lists[1:]): - if len(list_) != length: - msg = "List at index {} has length {} != {}".format( - i + 1, len(list_), length) - raise AssertionError(msg) - return zip(*lists) - - -def getLinesFromLogFile(stream): - stream.flush() - stream.seek(0) - lines = stream.readlines() - return lines - - -def getProjectRootFilePath(): - # assumes we're in a directory one level below the project root - return os.path.dirname(os.path.dirname(__file__)) - - -def getGa4ghFilePath(): - return os.path.join(getProjectRootFilePath(), packageName) - - -def powerset(iterable, maxSets=None): - """ - powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3) - - See https://docs.python.org/2/library/itertools.html#recipes - """ - s = list(iterable) - return itertools.islice(itertools.chain.from_iterable( - itertools.combinations(s, r) for r in range(len(s) + 1)), - 0, maxSets) - - -# ---------------- Decorators ---------------- - - -class TimeoutException(Exception): - """ - A process has taken too long to execute - """ - - -class Timed(object): - """ - Decorator that times a method, reporting runtime at finish - """ - def __call__(self, func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - self.start = time.time() - result = func(*args, **kwargs) - self.end = time.time() - self._report() - return result - return wrapper - - def _report(self): - delta = self.end - self.start - timeString = humanize.time.naturaldelta(delta) - print("Finished in {} ({} seconds)".format(timeString, delta)) - - -class Repeat(object): - """ - A decorator to use for repeating a tagged function. - The tagged function should return true if it wants to run again, - and false if it wants to stop repeating. - """ - defaultSleepSeconds = 0.1 - - def __init__(self, sleepSeconds=defaultSleepSeconds): - self.sleepSeconds = sleepSeconds - - def __call__(self, func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - while func(*args, **kwargs): - time.sleep(self.sleepSeconds) - return wrapper - - -class Timeout(object): - """ - A decorator to use for only allowing a function to run - for a limited amount of time - """ - defaultTimeoutSeconds = 60 - - def __init__(self, timeoutSeconds=defaultTimeoutSeconds): - self.timeoutSeconds = timeoutSeconds - - def __call__(self, func): - - def _handle_timeout(signum, frame): - raise TimeoutException() - - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - # set the alarm and execute func - signal.signal(signal.SIGALRM, _handle_timeout) - signal.alarm(self.timeoutSeconds) - result = func(*args, **kwargs) - finally: - # clear the alarm - signal.alarm(0) - return result - return wrapper - - -@contextlib.contextmanager -def suppressOutput(): - # I would like to use sys.stdout.fileno() and sys.stderr.fileno() - # here instead of literal fd numbers, but nose does something like - # sys.stdout = StringIO.StringIO() when the -s flag is not enabled - # (to capture test output so it doesn't get entangled with nose's - # display) so the sys.stdout and sys.stderr objects are not able to - # be accessed here. - try: - # redirect stdout and stderr to /dev/null - devnull = open(os.devnull, 'w') - origStdoutFd = 1 - origStderrFd = 2 - origStdout = os.dup(origStdoutFd) - origStderr = os.dup(origStderrFd) - sys.stdout.flush() - sys.stderr.flush() - os.dup2(devnull.fileno(), origStdoutFd) - os.dup2(devnull.fileno(), origStderrFd) - # enter the wrapped code - yield - finally: - # restore original file streams - devnull.flush() - os.dup2(origStdout, origStdoutFd) - os.dup2(origStderr, origStderrFd) - # clean up - os.close(origStdout) - os.close(origStderr) - devnull.close()