From fe777d21a524ff99fd92b26c834e4f475d932de8 Mon Sep 17 00:00:00 2001 From: Danny Colligan Date: Thu, 26 Jan 2017 18:38:06 -0500 Subject: [PATCH] Remove protocol.py Issue https://github.com/BD2KGenomics/ga4gh-integration/issues/53 --- ga4gh/server/backend.py | 6 +- ga4gh/server/cli/__init__.py | 3 +- ga4gh/server/converters.py | 3 +- ga4gh/server/datamodel/__init__.py | 3 +- ga4gh/server/datamodel/bio_metadata.py | 3 +- ga4gh/server/datamodel/datasets.py | 2 +- ga4gh/server/datamodel/genotype_phenotype.py | 3 +- .../genotype_phenotype_featureset.py | 3 +- ga4gh/server/datamodel/ontologies.py | 3 +- ga4gh/server/datamodel/reads.py | 2 +- ga4gh/server/datamodel/references.py | 2 +- ga4gh/server/datamodel/rna_quantification.py | 3 +- .../server/datamodel/sequence_annotations.py | 2 +- ga4gh/server/datamodel/variants.py | 2 +- ga4gh/server/datarepo.py | 3 +- ga4gh/server/exceptions.py | 2 +- ga4gh/server/frontend.py | 4 +- ga4gh/server/protocol.py | 325 ------------------ ga4gh/server/response_builder.py | 95 +++++ scripts/server_benchmark.py | 3 +- tests/datadriven/__init__.py | 2 +- tests/datadriven/test_genotype_phenotype.py | 3 +- tests/datadriven/test_ontologies.py | 2 +- tests/datadriven/test_reads.py | 2 +- tests/datadriven/test_references.py | 3 +- tests/datadriven/test_rna_quantification.py | 3 +- tests/datadriven/test_sequence_annotations.py | 3 +- tests/datadriven/test_variant_annotations.py | 3 +- tests/datadriven/test_variants.py | 2 +- tests/end_to_end/test_client_json.py | 4 +- tests/end_to_end/test_g2p.py | 3 +- tests/end_to_end/test_repo_manager.py | 2 +- tests/end_to_end/test_sequence_annotations.py | 3 +- tests/unit/test_auth0.py | 6 +- tests/unit/test_bio_metadata.py | 3 +- tests/unit/test_cli.py | 3 +- tests/unit/test_client.py | 3 +- tests/unit/test_exceptions.py | 3 +- tests/unit/test_faultydata.py | 3 +- tests/unit/test_imports.py | 5 +- tests/unit/test_protocol_errors.py | 3 +- tests/unit/test_reads.py | 3 +- tests/unit/test_response_generators.py | 3 +- tests/unit/test_schemas.py | 14 +- tests/unit/test_simulated_stack.py | 3 +- tests/unit/test_variant_annotations.py | 3 +- tests/unit/test_views.py | 3 +- 47 files changed, 179 insertions(+), 386 deletions(-) delete mode 100644 ga4gh/server/protocol.py create mode 100644 ga4gh/server/response_builder.py diff --git a/ga4gh/server/backend.py b/ga4gh/server/backend.py index df5d39ce5..c316b7020 100644 --- a/ga4gh/server/backend.py +++ b/ga4gh/server/backend.py @@ -8,8 +8,10 @@ import ga4gh.server.datamodel as datamodel import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import ga4gh.server.paging as paging +import ga4gh.server.response_builder as response_builder + +import ga4gh.schemas.protocol as protocol class Backend(object): @@ -579,7 +581,7 @@ def runSearchRequest( request.page_size = self._defaultPageSize if request.page_size < 0: raise exceptions.BadPageSizeException(request.page_size) - responseBuilder = protocol.SearchResponseBuilder( + responseBuilder = response_builder.SearchResponseBuilder( responseClass, request.page_size, self._maxResponseLength) nextPageToken = None for obj, nextPageToken in objectGenerator(request): diff --git a/ga4gh/server/cli/__init__.py b/ga4gh/server/cli/__init__.py index e65be862d..e664a7d79 100644 --- a/ga4gh/server/cli/__init__.py +++ b/ga4gh/server/cli/__init__.py @@ -6,7 +6,8 @@ from __future__ import unicode_literals import ga4gh.server -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol # the maximum value of a long type in avro = 2**63 - 1 # (64 bit signed integer) diff --git a/ga4gh/server/converters.py b/ga4gh/server/converters.py index d875700c3..e96e5100a 100644 --- a/ga4gh/server/converters.py +++ b/ga4gh/server/converters.py @@ -11,7 +11,8 @@ import pysam import ga4gh.server.datamodel.reads as reads -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class AbstractConverter(object): diff --git a/ga4gh/server/datamodel/__init__.py b/ga4gh/server/datamodel/__init__.py index 444af1fe7..532795eb1 100644 --- a/ga4gh/server/datamodel/__init__.py +++ b/ga4gh/server/datamodel/__init__.py @@ -13,7 +13,8 @@ import os import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class PysamFileHandleCache(object): diff --git a/ga4gh/server/datamodel/bio_metadata.py b/ga4gh/server/datamodel/bio_metadata.py index b807856ff..f5291ccf3 100644 --- a/ga4gh/server/datamodel/bio_metadata.py +++ b/ga4gh/server/datamodel/bio_metadata.py @@ -9,9 +9,10 @@ import json import ga4gh.server.datamodel as datamodel -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions +import ga4gh.schemas.protocol as protocol + class Biosample(datamodel.DatamodelObject): """ diff --git a/ga4gh/server/datamodel/datasets.py b/ga4gh/server/datamodel/datasets.py index 990ccc199..91f26fa07 100644 --- a/ga4gh/server/datamodel/datasets.py +++ b/ga4gh/server/datamodel/datasets.py @@ -10,12 +10,12 @@ import ga4gh.server.datamodel.sequence_annotations as sequence_annotations import ga4gh.server.datamodel.variants as variants import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import ga4gh.server.datamodel.bio_metadata as biodata import ga4gh.server.datamodel.genotype_phenotype as g2p import ga4gh.server.datamodel.rna_quantification as rnaQuantification import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol class Dataset(datamodel.DatamodelObject): diff --git a/ga4gh/server/datamodel/genotype_phenotype.py b/ga4gh/server/datamodel/genotype_phenotype.py index 4bab58bb8..001c80797 100644 --- a/ga4gh/server/datamodel/genotype_phenotype.py +++ b/ga4gh/server/datamodel/genotype_phenotype.py @@ -11,7 +11,8 @@ import ga4gh.server.datamodel as datamodel import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol # annotation keys TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' diff --git a/ga4gh/server/datamodel/genotype_phenotype_featureset.py b/ga4gh/server/datamodel/genotype_phenotype_featureset.py index 29e3b7f5a..c58e644b8 100644 --- a/ga4gh/server/datamodel/genotype_phenotype_featureset.py +++ b/ga4gh/server/datamodel/genotype_phenotype_featureset.py @@ -12,10 +12,11 @@ from rdflib import RDF import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import ga4gh.server.datamodel.sequence_annotations as sequence_annotations import ga4gh.server.datamodel.genotype_phenotype as g2p +import ga4gh.schemas.protocol as protocol + # annotation keys TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' LABEL = 'http://www.w3.org/2000/01/rdf-schema#label' diff --git a/ga4gh/server/datamodel/ontologies.py b/ga4gh/server/datamodel/ontologies.py index 0798e72ba..5f9e8b022 100644 --- a/ga4gh/server/datamodel/ontologies.py +++ b/ga4gh/server/datamodel/ontologies.py @@ -8,10 +8,11 @@ import collections import os.path -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import ga4gh.server.datamodel.obo_parser as obo_parser +import ga4gh.schemas.protocol as protocol + SEQUENCE_ONTOLOGY_PREFIX = "SO" diff --git a/ga4gh/server/datamodel/reads.py b/ga4gh/server/datamodel/reads.py index 349816a01..2b11433a2 100644 --- a/ga4gh/server/datamodel/reads.py +++ b/ga4gh/server/datamodel/reads.py @@ -16,9 +16,9 @@ import ga4gh.server.datamodel as datamodel import ga4gh.server.datamodel.references as references import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol def parseMalformedBamHeader(headerDict): diff --git a/ga4gh/server/datamodel/references.py b/ga4gh/server/datamodel/references.py index 9c73538b2..8003d0477 100644 --- a/ga4gh/server/datamodel/references.py +++ b/ga4gh/server/datamodel/references.py @@ -13,10 +13,10 @@ import pysam import ga4gh.server.datamodel as datamodel -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol DEFAULT_REFERENCESET_NAME = "Default" diff --git a/ga4gh/server/datamodel/rna_quantification.py b/ga4gh/server/datamodel/rna_quantification.py index 88090081c..6cd6ab18d 100644 --- a/ga4gh/server/datamodel/rna_quantification.py +++ b/ga4gh/server/datamodel/rna_quantification.py @@ -7,10 +7,11 @@ from __future__ import unicode_literals import ga4gh.server.datamodel as datamodel -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import ga4gh.server.sqlite_backend as sqlite_backend +import ga4gh.schemas.protocol as protocol + """ The RNA Quantifications associated with a GA4GH dataset reside in a sqlite diff --git a/ga4gh/server/datamodel/sequence_annotations.py b/ga4gh/server/datamodel/sequence_annotations.py index 51141dc14..4afc4fe32 100644 --- a/ga4gh/server/datamodel/sequence_annotations.py +++ b/ga4gh/server/datamodel/sequence_annotations.py @@ -9,12 +9,12 @@ import json import random -import ga4gh.server.protocol as protocol import ga4gh.server.datamodel as datamodel import ga4gh.server.sqlite_backend as sqlite_backend import ga4gh.server.exceptions as exceptions import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol # Note to self: There's the Feature ID as understood in a GFF3 file, # the Feature ID that is its server-assigned compoundId, and the diff --git a/ga4gh/server/datamodel/variants.py b/ga4gh/server/datamodel/variants.py index e763843e0..cc585bba1 100644 --- a/ga4gh/server/datamodel/variants.py +++ b/ga4gh/server/datamodel/variants.py @@ -16,12 +16,12 @@ import pysam -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import ga4gh.server.datamodel as datamodel import ga4gh.schemas.pb as pb import ga4gh.schemas.ga4gh.common_pb2 as common_pb2 +import ga4gh.schemas.protocol as protocol ANNOTATIONS_VEP_V82 = "VEP_v82" diff --git a/ga4gh/server/datarepo.py b/ga4gh/server/datarepo.py index 1671e52c6..e300eabb3 100644 --- a/ga4gh/server/datarepo.py +++ b/ga4gh/server/datarepo.py @@ -22,10 +22,11 @@ import ga4gh.server.datamodel.genotype_phenotype_featureset as g2pFeatureset import ga4gh.server.datamodel.rna_quantification as rna_quantification import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import repo.models as m +import ga4gh.schemas.protocol as protocol + MODE_READ = 'r' MODE_WRITE = 'w' diff --git a/ga4gh/server/exceptions.py b/ga4gh/server/exceptions.py index 3f8e7da8f..89be6ac90 100644 --- a/ga4gh/server/exceptions.py +++ b/ga4gh/server/exceptions.py @@ -11,7 +11,7 @@ import inspect import json -import ga4gh.server.protocol as protocol +import ga4gh.schemas.protocol as protocol def getExceptionClass(errorCode): diff --git a/ga4gh/server/frontend.py b/ga4gh/server/frontend.py index 5172aba18..3e493f7a7 100644 --- a/ga4gh/server/frontend.py +++ b/ga4gh/server/frontend.py @@ -28,12 +28,12 @@ import ga4gh.server import ga4gh.server.backend as backend import ga4gh.server.datamodel as datamodel -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import ga4gh.server.datarepo as datarepo - import ga4gh.server.auth as auth +import ga4gh.schemas.protocol as protocol + MIMETYPE = "application/json" SEARCH_ENDPOINT_METHODS = ['POST', 'OPTIONS'] diff --git a/ga4gh/server/protocol.py b/ga4gh/server/protocol.py deleted file mode 100644 index 2ce98ac84..000000000 --- a/ga4gh/server/protocol.py +++ /dev/null @@ -1,325 +0,0 @@ -""" -Definitions of the GA4GH protocol types. -""" -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import datetime -import json -import inspect -import array -from sys import modules - -import google.protobuf.json_format as json_format -import google.protobuf.message as message - -import ga4gh.schemas.pb as pb - -from ga4gh.schemas._protocol_version import version # noqa -from ga4gh.schemas.ga4gh.common_pb2 import * # noqa -from ga4gh.schemas.ga4gh.metadata_pb2 import * # noqa -from ga4gh.schemas.ga4gh.metadata_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.read_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.reads_pb2 import * # noqa -from ga4gh.schemas.ga4gh.reference_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.references_pb2 import * # noqa -from ga4gh.schemas.ga4gh.variant_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.variants_pb2 import * # noqa -from ga4gh.schemas.ga4gh.allele_annotations_pb2 import * # noqa -from ga4gh.schemas.ga4gh.allele_annotation_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.sequence_annotations_pb2 import * # noqa -from ga4gh.schemas.ga4gh.sequence_annotation_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.bio_metadata_pb2 import * # noqa -from ga4gh.schemas.ga4gh.bio_metadata_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.genotype_phenotype_pb2 import * # noqa -from ga4gh.schemas.ga4gh.genotype_phenotype_service_pb2 import * # noqa -from ga4gh.schemas.ga4gh.rna_quantification_pb2 import * # noqa -from ga4gh.schemas.ga4gh.rna_quantification_service_pb2 import * # noqa - -import ga4gh.schemas.ga4gh.common_pb2 as common - -# A map of response objects to the name of the attribute used to -# store the values returned. -_valueListNameMap = { - SearchVariantSetsResponse: "variant_sets", # noqa - SearchVariantsResponse: "variants", # noqa - SearchDatasetsResponse: "datasets", # noqa - SearchReferenceSetsResponse: "reference_sets", # noqa - SearchReferencesResponse: "references", # noqa - SearchReadGroupSetsResponse: "read_group_sets", # noqa - SearchReadsResponse: "alignments", # noqa - SearchCallSetsResponse: "call_sets", # noqa - SearchVariantAnnotationSetsResponse: "variant_annotation_sets", # noqa - SearchVariantAnnotationsResponse: "variant_annotations", # noqa - SearchFeatureSetsResponse: "feature_sets", # noqa - SearchFeaturesResponse: "features", # noqa - SearchBiosamplesResponse: "biosamples", # noqa - SearchIndividualsResponse: "individuals", # noqa - SearchPhenotypeAssociationSetsResponse: "phenotype_association_sets", # noqa - SearchPhenotypesResponse: "phenotypes", # noqa - SearchGenotypePhenotypeResponse: "associations", # noqa - SearchRnaQuantificationSetsResponse: "rna_quantification_sets", # noqa - SearchRnaQuantificationsResponse: "rna_quantifications", # noqa - SearchExpressionLevelsResponse: "expression_levels", # noqa -} - - -def setAttribute(values, value): - """ - Takes the values of an attribute value list and attempts to append - attributes of the proper type, inferred from their Python type. - """ - if isinstance(value, int): - values.add().int32_value = value - elif isinstance(value, float): - values.add().double_value = value - elif isinstance(value, long): - values.add().int64_value = value - elif isinstance(value, str): - values.add().string_value = value - elif isinstance(value, bool): - values.add().bool_value = value - elif isinstance(value, (list, tuple, array.array)): - for v in value: - setAttribute(values, v) - elif isinstance(value, dict): - for key in value: - setAttribute(values.add().attributes.attr[key].values, value[key]) - else: - values.add().string_value = str(value) - - -def encodeValue(value): - if isinstance(value, (list, tuple)): - return [common.AttributeValue(string_value=str(v)) for v in value] - else: - return [common.AttributeValue(string_value=str(value))] - - -def getValueListName(protocolResponseClass): - """ - Returns the name of the attribute in the specified protocol class - that is used to hold the values in a search response. - """ - return _valueListNameMap[protocolResponseClass] - - -def convertDatetime(t): - """ - Converts the specified datetime object into its appropriate protocol - value. This is the number of milliseconds from the epoch. - """ - epoch = datetime.datetime.utcfromtimestamp(0) - delta = t - epoch - millis = delta.total_seconds() * 1000 - return int(millis) - - -def getValueFromValue(value): - """ - Extract the currently set field from a Value structure - """ - if type(value) != common.AttributeValue: - raise TypeError( - "Expected a AttributeValue, but got {}".format(type(value))) - if value.WhichOneof("value") is None: - raise AttributeError("Nothing set for {}".format(value)) - return getattr(value, value.WhichOneof("value")) - - -def toJson(protoObject, indent=None): - """ - Serialises a protobuf object as json - """ - # Using the internal method because this way we can reformat the JSON - js = json_format.MessageToDict(protoObject, False) - return json.dumps(js, indent=indent) - - -def toJsonDict(protoObject): - """ - Converts a protobuf object to the raw attributes - i.e. a key/value dictionary - """ - return json.loads(toJson(protoObject)) - - -def fromJson(json, protoClass): - """ - Deserialise json into an instance of protobuf class - """ - return json_format.Parse(json, protoClass()) - - -def validate(json, protoClass): - """ - Check that json represents data that could be used to make - a given protobuf class - """ - try: - fromJson(json, protoClass) - # The json conversion automatically validates - return True - except Exception: - return False - - -class SearchResponseBuilder(object): - """ - A class to allow sequential building of SearchResponse objects. - """ - def __init__(self, responseClass, pageSize, maxBufferSize): - """ - Allocates a new SearchResponseBuilder for the specified - responseClass, user-requested pageSize and the system mandated - maxBufferSize (in bytes). The maxBufferSize is an - approximate limit on the overall length of the serialised - response. - """ - self._responseClass = responseClass - self._pageSize = pageSize - self._maxBufferSize = maxBufferSize - self._numElements = 0 - self._nextPageToken = None - self._protoObject = responseClass() - self._valueListName = getValueListName(responseClass) - self._bufferSize = self._protoObject.ByteSize() - - def getPageSize(self): - """ - Returns the page size for this SearchResponseBuilder. This is the - user-requested maximum size for the number of elements in the - value list. - """ - return self._pageSize - - def getMaxBufferSize(self): - """ - Returns the maximum internal buffer size for responses, which - corresponds to total length (in bytes) of the serialised protobuf - objects. This will always be less than the size of JSON output. - """ - return self._maxBufferSize - - def getNextPageToken(self): - """ - Returns the value of the nextPageToken for this - SearchResponseBuilder. - """ - return self._nextPageToken - - def setNextPageToken(self, nextPageToken): - """ - Sets the nextPageToken to the specified value. - """ - self._nextPageToken = nextPageToken - - def addValue(self, protocolElement): - """ - Appends the specified protocolElement to the value list for this - response. - """ - self._numElements += 1 - self._bufferSize += protocolElement.ByteSize() - attr = getattr(self._protoObject, self._valueListName) - obj = attr.add() - obj.CopyFrom(protocolElement) - - def isFull(self): - """ - Returns True if the response buffer is full, and False otherwise. - The buffer is full if either (1) the number of items in the value - list is >= pageSize or (2) the total length of the serialised - elements in the page is >= maxBufferSize. - - If page_size or max_response_length were not set in the request - then they're not checked. - """ - return ( - (self._pageSize > 0 and self._numElements >= self._pageSize) or - (self._bufferSize >= self._maxBufferSize) - ) - - def getSerializedResponse(self): - """ - Returns a string version of the SearchResponse that has - been built by this SearchResponseBuilder. - """ - self._protoObject.next_page_token = pb.string(self._nextPageToken) - s = toJson(self._protoObject) - return s - - -def getProtocolClasses(superclass=message.Message): - """ - Returns all the protocol classes that are subclasses of the - specified superclass. Only 'leaf' classes are returned, - corresponding directly to the classes defined in the protocol. - """ - # We keep a manual list of the superclasses that we define here - # so we can filter them out when we're getting the protocol - # classes. - superclasses = set([message.Message]) - thisModule = modules[__name__] - subclasses = [] - for name, class_ in inspect.getmembers(thisModule): - if ((inspect.isclass(class_) and - issubclass(class_, superclass) and - class_ not in superclasses)): - subclasses.append(class_) - return subclasses - - -postMethods = \ - [('/callsets/search', - SearchCallSetsRequest, # noqa - SearchCallSetsResponse), # noqa - ('/datasets/search', - SearchDatasetsRequest, # noqa - SearchDatasetsResponse), # noqa - ('/readgroupsets/search', - SearchReadGroupSetsRequest, # noqa - SearchReadGroupSetsResponse), # noqa - ('/reads/search', - SearchReadsRequest, # noqa - SearchReadsResponse), # noqa - ('/references/search', - SearchReferencesRequest, # noqa - SearchReferencesResponse), # noqa - ('/referencesets/search', - SearchReferenceSetsRequest, # noqa - SearchReferenceSetsResponse), # noqa - ('/variants/search', - SearchVariantsRequest, # noqa - SearchVariantsResponse), # noqa - ('/datasets/search', - SearchDatasetsRequest, # noqa - SearchDatasetsResponse), # noqa - ('/callsets/search', - SearchCallSetsRequest, # noqa - SearchCallSetsResponse), # noqa - ('/featuresets/search', - SearchFeatureSetsRequest, # noqa - SearchFeatureSetsResponse), # noqa - ('/features/search', - SearchFeaturesRequest, # noqa - SearchFeaturesResponse), # noqa - ('/variantsets/search', - SearchVariantSetsRequest, # noqa - SearchVariantSetsResponse), # noqa - ('/variantannotations/search', - SearchVariantAnnotationsRequest, # noqa - SearchVariantAnnotationSetsResponse), # noqa - ('/variantannotationsets/search', - SearchVariantAnnotationSetsRequest, # noqa - SearchVariantAnnotationSetsResponse), # noqa - ('/rnaquantificationsets/search', - SearchRnaQuantificationSetsRequest, # noqa - SearchRnaQuantificationSetsResponse), # noqa - ('/rnaquantifications/search', - SearchRnaQuantificationsRequest, # noqa - SearchRnaQuantificationsResponse), # noqa - ('/expressionlevels/search', - SearchExpressionLevelsRequest, # noqa - SearchExpressionLevelsResponse)] # noqa diff --git a/ga4gh/server/response_builder.py b/ga4gh/server/response_builder.py new file mode 100644 index 000000000..2f238fec3 --- /dev/null +++ b/ga4gh/server/response_builder.py @@ -0,0 +1,95 @@ +""" +Class that builds the responses to the client +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol + + +class SearchResponseBuilder(object): + """ + A class to allow sequential building of SearchResponse objects. + """ + def __init__(self, responseClass, pageSize, maxBufferSize): + """ + Allocates a new SearchResponseBuilder for the specified + responseClass, user-requested pageSize and the system mandated + maxBufferSize (in bytes). The maxBufferSize is an + approximate limit on the overall length of the serialised + response. + """ + self._responseClass = responseClass + self._pageSize = pageSize + self._maxBufferSize = maxBufferSize + self._numElements = 0 + self._nextPageToken = None + self._protoObject = responseClass() + self._valueListName = protocol.getValueListName(responseClass) + self._bufferSize = self._protoObject.ByteSize() + + def getPageSize(self): + """ + Returns the page size for this SearchResponseBuilder. This is the + user-requested maximum size for the number of elements in the + value list. + """ + return self._pageSize + + def getMaxBufferSize(self): + """ + Returns the maximum internal buffer size for responses, which + corresponds to total length (in bytes) of the serialised protobuf + objects. This will always be less than the size of JSON output. + """ + return self._maxBufferSize + + def getNextPageToken(self): + """ + Returns the value of the nextPageToken for this + SearchResponseBuilder. + """ + return self._nextPageToken + + def setNextPageToken(self, nextPageToken): + """ + Sets the nextPageToken to the specified value. + """ + self._nextPageToken = nextPageToken + + def addValue(self, protocolElement): + """ + Appends the specified protocolElement to the value list for this + response. + """ + self._numElements += 1 + self._bufferSize += protocolElement.ByteSize() + attr = getattr(self._protoObject, self._valueListName) + obj = attr.add() + obj.CopyFrom(protocolElement) + + def isFull(self): + """ + Returns True if the response buffer is full, and False otherwise. + The buffer is full if either (1) the number of items in the value + list is >= pageSize or (2) the total length of the serialised + elements in the page is >= maxBufferSize. + + If page_size or max_response_length were not set in the request + then they're not checked. + """ + return ( + (self._pageSize > 0 and self._numElements >= self._pageSize) or + (self._bufferSize >= self._maxBufferSize) + ) + + def getSerializedResponse(self): + """ + Returns a string version of the SearchResponse that has + been built by this SearchResponseBuilder. + """ + self._protoObject.next_page_token = pb.string(self._nextPageToken) + s = protocol.toJson(self._protoObject) + return s diff --git a/scripts/server_benchmark.py b/scripts/server_benchmark.py index 42685aff2..0e7fc4702 100644 --- a/scripts/server_benchmark.py +++ b/scripts/server_benchmark.py @@ -15,9 +15,10 @@ import glue +import ga4gh.schemas.protocol as protocol + glue.ga4ghImportGlue() import ga4gh.server.backend as backend # noqa -import ga4gh.server.protocol as protocol # noqa import ga4gh.server.datarepo as datarepo # noqa diff --git a/tests/datadriven/__init__.py b/tests/datadriven/__init__.py index b8d445795..4507eab11 100644 --- a/tests/datadriven/__init__.py +++ b/tests/datadriven/__init__.py @@ -13,9 +13,9 @@ import inspect import google.protobuf.json_format as json_format -import ga4gh.server.protocol as protocol import ga4gh.schemas.pb as pb +import ga4gh.schemas.protocol as protocol def _wrapTestMethod(method): diff --git a/tests/datadriven/test_genotype_phenotype.py b/tests/datadriven/test_genotype_phenotype.py index b9c25ecb1..02405c87e 100644 --- a/tests/datadriven/test_genotype_phenotype.py +++ b/tests/datadriven/test_genotype_phenotype.py @@ -10,10 +10,11 @@ import ga4gh.server.datamodel.genotype_phenotype as genotype_phenotype import ga4gh.server.datamodel.datasets as datasets -import ga4gh.server.protocol as protocol import tests.datadriven as datadriven import tests.paths as paths +import ga4gh.schemas.protocol as protocol + def testG2P(): testDataDir = os.path.join( diff --git a/tests/datadriven/test_ontologies.py b/tests/datadriven/test_ontologies.py index 65110ac3f..0f421c44c 100644 --- a/tests/datadriven/test_ontologies.py +++ b/tests/datadriven/test_ontologies.py @@ -11,13 +11,13 @@ # accessing ontology information, since this is the method we use # in the main code. However, other libraries have very heavy dependencies. import ga4gh.server.datamodel.obo_parser as obo_parser -import ga4gh.server.protocol as protocol import ga4gh.server.datamodel.ontologies as ontologies import tests.datadriven as datadriven import tests.paths as paths from ga4gh.schemas.ga4gh.common_pb2 import OntologyTerm +import ga4gh.schemas.protocol as protocol def testReferenceSets(): diff --git a/tests/datadriven/test_reads.py b/tests/datadriven/test_reads.py index 2250c1c5c..ef6840b2e 100644 --- a/tests/datadriven/test_reads.py +++ b/tests/datadriven/test_reads.py @@ -13,12 +13,12 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.datamodel.reads as reads import ga4gh.server.datamodel.references as references -import ga4gh.server.protocol as protocol import ga4gh.server.datarepo as datarepo import tests.datadriven as datadriven import tests.paths as paths import ga4gh.common.utils as utils +import ga4gh.schemas.protocol as protocol import pysam diff --git a/tests/datadriven/test_references.py b/tests/datadriven/test_references.py index 5dbb61093..4979757ea 100644 --- a/tests/datadriven/test_references.py +++ b/tests/datadriven/test_references.py @@ -16,11 +16,12 @@ import pysam import ga4gh.server.datamodel.references as references -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import tests.datadriven as datadriven import tests.paths as paths +import ga4gh.schemas.protocol as protocol + def testReferenceSets(): testDataDir = os.path.join(paths.testDataDir, "referenceSets") diff --git a/tests/datadriven/test_rna_quantification.py b/tests/datadriven/test_rna_quantification.py index 8c2935c71..842da85cd 100644 --- a/tests/datadriven/test_rna_quantification.py +++ b/tests/datadriven/test_rna_quantification.py @@ -10,10 +10,11 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.datamodel.references as references import ga4gh.server.datamodel.rna_quantification as rna_quantification -import ga4gh.server.protocol as protocol import tests.datadriven as datadriven import tests.paths as paths +import ga4gh.schemas.protocol as protocol + _datasetName = "ds" diff --git a/tests/datadriven/test_sequence_annotations.py b/tests/datadriven/test_sequence_annotations.py index 42674e8de..80a967954 100644 --- a/tests/datadriven/test_sequence_annotations.py +++ b/tests/datadriven/test_sequence_annotations.py @@ -10,10 +10,11 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.datamodel.references as references import ga4gh.server.datamodel.sequence_annotations as sequence_annotations -import ga4gh.server.protocol as protocol import tests.datadriven as datadriven import tests.paths as paths +import ga4gh.schemas.protocol as protocol + _datasetName = "ds" _discontinuousTestData = { diff --git a/tests/datadriven/test_variant_annotations.py b/tests/datadriven/test_variant_annotations.py index ad941ce11..68f8c470f 100644 --- a/tests/datadriven/test_variant_annotations.py +++ b/tests/datadriven/test_variant_annotations.py @@ -15,10 +15,11 @@ import ga4gh.server.datamodel.variants as variants import ga4gh.server.datamodel.references as references import ga4gh.server.datamodel.ontologies as ontologies -import ga4gh.server.protocol as protocol import tests.datadriven as datadriven import tests.paths as paths +import ga4gh.schemas.protocol as protocol + def testVariantAnnotationSets(): testDataDir = "tests/data/datasets/dataset1/variants" diff --git a/tests/datadriven/test_variants.py b/tests/datadriven/test_variants.py index 4972ca1a8..d5341e784 100644 --- a/tests/datadriven/test_variants.py +++ b/tests/datadriven/test_variants.py @@ -15,12 +15,12 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.datamodel.references as references import ga4gh.server.datamodel.variants as variants -import ga4gh.server.protocol as protocol import ga4gh.server.exceptions as exceptions import tests.datadriven as datadriven import tests.paths as paths import ga4gh.common.utils as utils +import ga4gh.schemas.protocol as protocol def testVariantSets(): diff --git a/tests/end_to_end/test_client_json.py b/tests/end_to_end/test_client_json.py index 40eed0f29..fe971ad37 100644 --- a/tests/end_to_end/test_client_json.py +++ b/tests/end_to_end/test_client_json.py @@ -10,17 +10,17 @@ import datetime import json import unittest +import freezegun import ga4gh.client.client as client import ga4gh.server.backend as backend import ga4gh.client.cli as cli_client -import ga4gh.server.protocol as protocol import ga4gh.server.datarepo as datarepo import ga4gh.common.utils as utils import tests.paths as paths import server -import freezegun +import ga4gh.schemas.protocol as protocol def setUpModule(): diff --git a/tests/end_to_end/test_g2p.py b/tests/end_to_end/test_g2p.py index ce69fabb0..39481958d 100644 --- a/tests/end_to_end/test_g2p.py +++ b/tests/end_to_end/test_g2p.py @@ -4,10 +4,11 @@ import unittest import ga4gh.server.datamodel as datamodel -import ga4gh.server.protocol as protocol import ga4gh.server.frontend as frontend import tests.paths as paths +import ga4gh.schemas.protocol as protocol + class TestG2P(unittest.TestCase): exampleUrl = 'www.example.com' diff --git a/tests/end_to_end/test_repo_manager.py b/tests/end_to_end/test_repo_manager.py index 6657f0c7e..622c5df20 100644 --- a/tests/end_to_end/test_repo_manager.py +++ b/tests/end_to_end/test_repo_manager.py @@ -14,7 +14,7 @@ import ga4gh.server.cli.repomanager as cli_repomanager import tests.paths as paths -import ga4gh.server.protocol as protocol +import ga4gh.schemas.protocol as protocol class RepoManagerEndToEndTest(unittest.TestCase): diff --git a/tests/end_to_end/test_sequence_annotations.py b/tests/end_to_end/test_sequence_annotations.py index 4440886d2..c802cc265 100644 --- a/tests/end_to_end/test_sequence_annotations.py +++ b/tests/end_to_end/test_sequence_annotations.py @@ -8,10 +8,11 @@ import unittest import logging -import ga4gh.server.protocol as protocol import ga4gh.server.frontend as frontend import tests.paths as paths +import ga4gh.schemas.protocol as protocol + class TestSequenceAnnotations(unittest.TestCase): exampleUrl = 'www.example.com' diff --git a/tests/unit/test_auth0.py b/tests/unit/test_auth0.py index 162f22b7d..b40b9223c 100644 --- a/tests/unit/test_auth0.py +++ b/tests/unit/test_auth0.py @@ -5,17 +5,15 @@ from __future__ import print_function from __future__ import unicode_literals -import ga4gh.server.protocol as protocol - +import shutil import unittest import logging import ga4gh.server.frontend as frontend import ga4gh.server.exceptions as exceptions - import ga4gh.server.auth as auth -import shutil +import ga4gh.schemas.protocol as protocol class TestAuth0(unittest.TestCase): diff --git a/tests/unit/test_bio_metadata.py b/tests/unit/test_bio_metadata.py index 5502a8835..615c58d34 100644 --- a/tests/unit/test_bio_metadata.py +++ b/tests/unit/test_bio_metadata.py @@ -10,7 +10,8 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.exceptions as exceptions import ga4gh.server.datamodel.bio_metadata as bioMetadata -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestIndividuals(unittest.TestCase): diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 99bd43e58..135000602 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -12,7 +12,8 @@ import ga4gh.server.cli.repomanager as cli_repomanager import ga4gh.server.cli.ga2vcf as cli_ga2vcf import ga4gh.server.cli.ga2sam as cli_ga2sam -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestServerArguments(unittest.TestCase): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 4f357c035..7146b2a70 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -6,15 +6,14 @@ from __future__ import unicode_literals import unittest - import json -import ga4gh.server.protocol as protocol import ga4gh.server.backend as backend import ga4gh.server.datarepo as datarepo import ga4gh.client.client as client import ga4gh.common.utils as utils +import ga4gh.schemas.protocol as protocol class DatamodelObjectWrapper(object): diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 7caf1f706..bb36c3f2c 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -11,7 +11,8 @@ import ga4gh.server.exceptions as exceptions import ga4gh.server.frontend as frontend -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestExceptionHandler(unittest.TestCase): diff --git a/tests/unit/test_faultydata.py b/tests/unit/test_faultydata.py index 99a90bc72..769f07883 100644 --- a/tests/unit/test_faultydata.py +++ b/tests/unit/test_faultydata.py @@ -11,7 +11,8 @@ import ga4gh.server.datamodel.datasets as datasets import ga4gh.server.datamodel.variants as variants import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class FaultyVariantDataTest(unittest.TestCase): diff --git a/tests/unit/test_imports.py b/tests/unit/test_imports.py index 6949f5220..a793b0d38 100644 --- a/tests/unit/test_imports.py +++ b/tests/unit/test_imports.py @@ -171,6 +171,7 @@ class ImportGraphLayerChecker(object): 'ga4gh/server/backend.py', 'ga4gh/server/datarepo.py', 'ga4gh/server/paging.py', + 'ga4gh/server/response_builder.py', ], 'exceptions': [ 'ga4gh/server/exceptions.py', @@ -194,9 +195,6 @@ class ImportGraphLayerChecker(object): 'ga4gh/server/converters.py', 'ga4gh/server/configtest.py', ], - 'protocol': [ - 'ga4gh/server/protocol.py', - ], 'config': [ 'ga4gh/server/serverconfig.py', ], @@ -217,7 +215,6 @@ class ImportGraphLayerChecker(object): ['repo'], ['exceptions'], ['config'], - ['protocol'], ] def __init__(self, graph): diff --git a/tests/unit/test_protocol_errors.py b/tests/unit/test_protocol_errors.py index d6a630652..662f721e6 100644 --- a/tests/unit/test_protocol_errors.py +++ b/tests/unit/test_protocol_errors.py @@ -9,7 +9,8 @@ import ga4gh.server.frontend as frontend import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestFrontendErrors(unittest.TestCase): diff --git a/tests/unit/test_reads.py b/tests/unit/test_reads.py index b366080d4..c65683be7 100644 --- a/tests/unit/test_reads.py +++ b/tests/unit/test_reads.py @@ -9,7 +9,8 @@ import unittest import ga4gh.server.datamodel.reads as reads -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestParseMalformedBamHeader(unittest.TestCase): diff --git a/tests/unit/test_response_generators.py b/tests/unit/test_response_generators.py index b316702d1..0a958e60d 100644 --- a/tests/unit/test_response_generators.py +++ b/tests/unit/test_response_generators.py @@ -12,9 +12,10 @@ import ga4gh.server.datamodel.reads as reads import ga4gh.server.datamodel.variants as variants import ga4gh.server.exceptions as exceptions -import ga4gh.server.protocol as protocol import ga4gh.server.datarepo as datarepo +import ga4gh.schemas.protocol as protocol + def generateVariant(): variant = protocol.Variant() diff --git a/tests/unit/test_schemas.py b/tests/unit/test_schemas.py index 3e466c594..023ca4ab4 100644 --- a/tests/unit/test_schemas.py +++ b/tests/unit/test_schemas.py @@ -4,7 +4,8 @@ import unittest -import ga4gh.server.protocol as protocol +import ga4gh.server.response_builder as response_builder +import ga4gh.schemas.protocol as protocol def getValueListName(cls): @@ -17,7 +18,6 @@ class SearchResponseBuilderTest(unittest.TestCase): Tests the SearchResponseBuilder class to ensure that it behaves correctly. """ - def testIntegrity(self): # Verifies that the values we put in are exactly what we get # back across all subclasses of SearchResponse @@ -26,7 +26,7 @@ def testIntegrity(self): instance = class_() valueList = getattr(instance, getValueListName(class_)) valueList.add() - builder = protocol.SearchResponseBuilder( + builder = response_builder.SearchResponseBuilder( class_, len(valueList), 2 ** 32) for value in valueList: builder.addValue(value) @@ -41,7 +41,7 @@ def testPageSizeOverflow(self): responseClass = protocol.SearchVariantsResponse valueClass = protocol.Variant for pageSize in range(1, 10): - builder = protocol.SearchResponseBuilder( + builder = response_builder.SearchResponseBuilder( responseClass, pageSize, 2 ** 32) self.assertEqual(builder.getPageSize(), pageSize) self.assertFalse(builder.isFull()) @@ -61,7 +61,7 @@ def testPageSizeExactFill(self): responseClass = protocol.SearchVariantsResponse valueClass = protocol.Variant for pageSize in range(1, 10): - builder = protocol.SearchResponseBuilder( + builder = response_builder.SearchResponseBuilder( responseClass, pageSize, 2 ** 32) self.assertEqual(builder.getPageSize(), pageSize) while not builder.isFull(): @@ -81,7 +81,7 @@ def testMaxBufferSizeOverridesPageSize(self): typicalValueLength = typicalValue.ByteSize() for numValues in range(1, 10): maxBufferSize = numValues * typicalValueLength - builder = protocol.SearchResponseBuilder( + builder = response_builder.SearchResponseBuilder( responseClass, 1000, maxBufferSize) self.assertEqual( maxBufferSize, builder.getMaxBufferSize()) @@ -94,7 +94,7 @@ def testMaxBufferSizeOverridesPageSize(self): def testNextPageToken(self): responseClass = protocol.SearchVariantsResponse - builder = protocol.SearchResponseBuilder( + builder = response_builder.SearchResponseBuilder( responseClass, 100, 2 ** 32) # If not set, pageToken should be empty string self.assertIsNone(builder.getNextPageToken()) diff --git a/tests/unit/test_simulated_stack.py b/tests/unit/test_simulated_stack.py index 3864ffa66..a9954976e 100644 --- a/tests/unit/test_simulated_stack.py +++ b/tests/unit/test_simulated_stack.py @@ -16,7 +16,8 @@ import ga4gh.server.datamodel.variants as variants import ga4gh.server.datamodel.sequence_annotations as sequence_annotations import ga4gh.server.frontend as frontend -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestSimulatedStack(unittest.TestCase): diff --git a/tests/unit/test_variant_annotations.py b/tests/unit/test_variant_annotations.py index b060bcda5..0e05ab8c6 100644 --- a/tests/unit/test_variant_annotations.py +++ b/tests/unit/test_variant_annotations.py @@ -9,13 +9,14 @@ import hashlib import unittest -import ga4gh.server.protocol as protocol import ga4gh.server.datarepo as datarepo import ga4gh.server.datamodel.variants as variants import ga4gh.server.datamodel.datasets as datasets import tests.paths as paths +import ga4gh.schemas.protocol as protocol + class TestHtslibVariantAnnotationSet(unittest.TestCase): """ diff --git a/tests/unit/test_views.py b/tests/unit/test_views.py index 8a6bff347..b03fdfe4d 100644 --- a/tests/unit/test_views.py +++ b/tests/unit/test_views.py @@ -12,7 +12,8 @@ import ga4gh.server.datamodel as datamodel import ga4gh.server.frontend as frontend -import ga4gh.server.protocol as protocol + +import ga4gh.schemas.protocol as protocol class TestFrontend(unittest.TestCase):