Merge pull request #61 from blarghmatey/py3k_fixes

Updated library and unit tests to run on Python 2 and Python 3

Author: tylertreat

.travis.yml

@@ -1,12 +1,12 @@
 language: python
-python:
-  - "2.7"
-
 install:
-  - pip install -r requirements.txt
-  - pip install -r requirements_dev.txt
-
-script: make test
-
+  - pip install tox
+script: tox -e $TOXENV
 notifications:
   email: false
+env:
+  - TOXENV=py27
+  - TOXENV=py33
+  - TOXENV=py34
+  - TOXENV=nightly
+  - TOXENV=pypy

bigquery/__init__.py

@@ -1,5 +1,6 @@
-from client import get_client
-from client import (
+from __future__ import absolute_import
+from .client import get_client
+from .client import (
     BIGQUERY_SCOPE,
     BIGQUERY_SCOPE_READ_ONLY,
     JOB_CREATE_IF_NEEDED,
@@ -14,4 +15,4 @@
     JOB_ENCODING_ISO_8859_1
 )
 
-from schema_builder import schema_from_record
+from .schema_builder import schema_from_record

bigquery/client.py

@@ -1,21 +1,19 @@
 import calendar
+import json
+import logging
 from collections import defaultdict
 from datetime import datetime, timedelta
-from time import sleep
-from time import time
 from hashlib import sha256
-import json
-import logging
+from time import sleep, time
 
+import httplib2
+import six
 from apiclient.discovery import build
 from apiclient.errors import HttpError
-import httplib2
 
+from bigquery.errors import (BigQueryTimeoutException, JobExecutingException,
+                             JobInsertException, UnfinishedQueryException)
 from bigquery.schema_builder import schema_from_record
-from bigquery.errors import (
-    JobExecutingException, JobInsertException,
-    UnfinishedQueryException, BigQueryTimeoutException
-)
 
 BIGQUERY_SCOPE = 'https://www.googleapis.com/auth/bigquery'
 BIGQUERY_SCOPE_READ_ONLY = 'https://www.googleapis.com/auth/bigquery.readonly'
@@ -154,7 +152,7 @@ def _submit_query_job(self, query_data):
                 projectId=self.project_id, body=query_data).execute()
         except HttpError as e:
             if query_data.get("dryRun", False):
-                return None, json.loads(e.content)
+                return None, json.loads(e.content.decode('utf8'))
             raise
 
         job_id = query_reply['jobReference'].get('jobId')
@@ -266,7 +264,7 @@ def get_table_schema(self, dataset, table):
                 projectId=self.project_id,
                 tableId=table,
                 datasetId=dataset).execute()
-        except HttpError, e:
+        except HttpError as e:
             if int(e.resp['status']) == 404:
                 logging.warn('Table %s.%s does not exist', dataset, table)
                 return None
@@ -651,7 +649,7 @@ def import_data_from_uris(
                               skip_leading_rows=skip_leading_rows,
                               quote=quote)
             non_null_values = dict((k, v) for k, v
-                                   in all_values.items()
+                                   in list(all_values.items())
                                    if v)
             raise Exception("Parameters field_delimiter, allow_jagged_rows, "
                             "allow_quoted_newlines, quote and "
@@ -837,6 +835,7 @@ def wait_for_job(self, job, interval=5, timeout=60):
         Waits until the job indicated by job_resource is done or has failed
         Args:
             job: dict, representing a BigQuery job resource
+                 or str, representing a BigQuery job id
             interval: optional float polling interval in seconds, default = 5
             timeout: optional float timeout in seconds, default = 60
         Returns:
@@ -848,7 +847,9 @@ def wait_for_job(self, job, interval=5, timeout=60):
             BigQueryTimeoutException on timeout
         """
         complete = False
-        job_id = job['jobReference']['jobId']
+        job_id = str(job if isinstance(job,
+                                       (six.binary_type, six.text_type, int))
+                     else job['jobReference']['jobId'])
         job_resource = None
 
         start_time = time()
@@ -1048,7 +1049,7 @@ def _filter_tables_by_time(self, tables, start_time, end_time):
             A list of table names that are inside the time range.
         """
 
-        return [table_name for (table_name, unix_seconds) in tables.iteritems()
+        return [table_name for (table_name, unix_seconds) in tables.items()
                 if self._in_range(start_time, end_time, unix_seconds)]
 
     def _in_range(self, start_time, end_time, time):
@@ -1167,7 +1168,7 @@ def _generate_hex_for_uris(self, uris):
         Returns:
             string of hexed uris
         """
-        return sha256(":".join(uris) + str(time())).hexdigest()
+        return sha256((":".join(uris) + str(time())).encode()).hexdigest()
 
     def _raise_insert_exception_if_error(self, job):
         error_http = job.get('error')

bigquery/query_builder.py

@@ -77,7 +77,7 @@ def _render_select(selections):
         return 'SELECT *'
 
     rendered_selections = []
-    for name, options in selections.iteritems():
+    for name, options in selections.items():
         if not isinstance(options, list):
             options = [options]
 
@@ -200,7 +200,8 @@ def _render_condition(field, field_type, comparators):
         if condition == "IN":
             if isinstance(value, (list, tuple, set)):
                 value = ', '.join(
-                    [_render_condition_value(v, field_type) for v in value]
+                    sorted([_render_condition_value(v, field_type)
+                            for v in value])
                 )
             else:
                 value = _render_condition_value(value, field_type)

bigquery/schema_builder.py

@@ -1,10 +1,12 @@
+from __future__ import absolute_import
 __author__ = 'Aneil Mallavarapu (http://github.com/aneilbaboo)'
 
 from datetime import datetime
 
+import six
 import dateutil.parser
 
-from errors import InvalidTypeException
+from .errors import InvalidTypeException
 
 
 def default_timestamp_parser(s):
@@ -30,7 +32,7 @@ def schema_from_record(record, timestamp_parser=default_timestamp_parser):
         schema: list
     """
     return [describe_field(k, v, timestamp_parser=timestamp_parser)
-            for k, v in record.items()]
+            for k, v in list(record.items())]
 
 
 def describe_field(k, v, timestamp_parser=default_timestamp_parser):
@@ -76,7 +78,7 @@ def bq_schema_field(name, bq_type, mode):
     if bq_type == "record":
         try:
             field['fields'] = schema_from_record(v, timestamp_parser)
-        except InvalidTypeException, e:
+        except InvalidTypeException as e:
             # recursively construct the key causing the error
             raise InvalidTypeException("%s.%s" % (k, e.key), e.value)
 
@@ -100,7 +102,7 @@ def bigquery_type(o, timestamp_parser=default_timestamp_parser):
     t = type(o)
     if t == int:
         return "integer"
-    elif t == str or t == unicode:
+    elif (t == six.binary_type and six.PY2) or t == six.text_type:
         if timestamp_parser and timestamp_parser(o):
             return "timestamp"
         else: