TDL-19835: Adding new streams and discovery mode

setup.py

@@ -12,7 +12,7 @@
       install_requires=[
           'singer-python==5.12.2',
           'requests==2.20.0',
-          'braintree==3.53.0',
+          'braintree==4.16.0',
       ],
       extras_require={
           'dev': [

tap_braintree/__init__.py

@@ -1,14 +1,8 @@
 #!/usr/bin/env python3
-
-from datetime import datetime, timedelta
-import os
-import pytz
-import json
 import sys
-
+import json
 import braintree
 import singer
-
 from singer import utils
 from tap_braintree.discover import discover
 from tap_braintree.sync import sync as _sync
@@ -23,6 +17,10 @@
 LOGGER = singer.get_logger()
 
 def do_discover():
+    """
+    Run discovery mode
+    """
+
     LOGGER.info("Starting discovery")
     catalog = discover()
     json.dump(catalog.to_dict(), sys.stdout, indent=2)
@@ -61,10 +59,7 @@ def main():
                 state
             )    
     except braintree.exceptions.authentication_error.AuthenticationError:
-        LOGGER.critical('Authentication error occured. '
-                        'Please check your merchant_id, public_key, and '
-                        'private_key for errors', exc_info=True)
-
+        LOGGER.critical('Authentication error occured. Please check your merchant_id, public_key, and private_key for errors', exc_info=True)
 
 if __name__ == '__main__':
     main()

tap_braintree/discover.py

@@ -1,56 +1,19 @@
-from asyncio.log import logger
-from singer.catalog import Catalog, CatalogEntry, Schema
-import os
-import json
 import singer
-from singer import metadata
+from singer.catalog import Catalog, CatalogEntry, Schema
 from tap_braintree.streams import STREAMS
+from tap_braintree.schema import get_schemas
 
 LOGGER = singer.get_logger()
 
-def get_abs_path(path):
-    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
-
-def get_schemas():
-    schemas = {}
-    field_metadata = {}
-
-    for stream_name, stream_metadata in STREAMS.items():
-
-        schema_path = get_abs_path("schemas/{}.json".format(stream_name))
-        with open(schema_path,"r") as file:
-            schema = json.load(file)
-        schemas[stream_name] = schema
-
-        mdata = metadata.new()
-
-        mdata = metadata.get_standard_metadata(
-            schema=schema,
-            key_properties=stream_metadata.key_properties,
-            valid_replication_keys=stream_metadata.replication_keys,
-            replication_method=stream_metadata.replication_method,
-        )
-
-        mdata = metadata.to_map(mdata)
-        # Loop through all keys and make replication keys of automatic inclusion
-        for field_name in schema["properties"].keys():
-
-            if stream_metadata.replication_keys and field_name in stream_metadata.replication_keys:
-                mdata = metadata.write(
-                    mdata, ("properties", field_name), "inclusion", "automatic",
-                )
-
-        mdata = metadata.to_list(mdata)
-        field_metadata[stream_name] = mdata
-
-    return schemas, field_metadata
-
 def discover():
+    """
+    Generate catalog for call the streams
+    """
+
     schemas, field_metadata = get_schemas()
     catalog = Catalog([])
 
     for stream_name, schema_dict in schemas.items():
-
         schema = Schema.from_dict(schema_dict)
         mdata = field_metadata[stream_name]
 
@@ -64,4 +27,4 @@ def discover():
             )
         )
 
-    return catalog
+    return catalog

tap_braintree/schema.py

@@ -0,0 +1,49 @@
+import os
+import json
+from singer import metadata
+from tap_braintree.streams import STREAMS
+
+def get_abs_path(path):
+    """
+    Return full path for given argument relative path
+    """
+
+    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
+
+def get_schemas():
+    """
+    Return metadata and schema for all the streams
+    """
+
+    schemas = {}
+    field_metadata = {}
+
+    for stream_name, stream_metadata in STREAMS.items():
+
+        schema_path = get_abs_path("schemas/{}.json".format(stream_name))
+        with open(schema_path,"r") as file:
+            schema = json.load(file)
+        schemas[stream_name] = schema
+
+        mdata = metadata.new()
+
+        mdata = metadata.get_standard_metadata(
+            schema=schema,
+            key_properties=stream_metadata.key_properties,
+            valid_replication_keys=stream_metadata.replication_keys,
+            replication_method=stream_metadata.replication_method,
+        )
+
+        mdata = metadata.to_map(mdata)
+        # Loop through all keys and make replication keys of automatic inclusion
+        for field_name in schema["properties"].keys():
+
+            if stream_metadata.replication_keys and field_name in stream_metadata.replication_keys:
+                mdata = metadata.write(
+                    mdata, ("properties", field_name), "inclusion", "automatic",
+                )
+
+        mdata = metadata.to_list(mdata)
+        field_metadata[stream_name] = mdata
+
+    return schemas, field_metadata

tap_braintree/schemas/add_ons.json

@@ -10,11 +10,12 @@
                 "string"
             ]
         },
-        "current_billing_cycle": {
+        "created_at": {
             "type": [
                 "null",
-                "integer"
-            ]
+                "string"
+            ],
+            "format": "date-time"
         },
         "description": {
             "type": [
@@ -34,6 +35,12 @@
                 "string"
             ]
         },
+        "merchant_id": {
+            "type": [
+                "null",
+                "string"
+            ]
+        },
         "name": {
             "type": [
                 "null",
@@ -52,11 +59,12 @@
                 "integer"
             ]
         },
-        "quantity": {
+        "updated_at": {
             "type": [
                 "null",
-                "integer"
-            ]
+                "string"
+            ],
+            "format": "date-time"
         }
     }
 }