Add sort_by[asc] param to API request to prevent records overwrite…

tap_chargebee/streams/addons.py

@@ -12,6 +12,7 @@ class AddonsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/addons'.format(self.config.get('site'))

tap_chargebee/streams/base.py

@@ -145,6 +145,10 @@ def sync_data(self):
             params = {"updated_at[after]": bookmark_date_posix}
             bookmark_key = 'updated_at'
 
+        # Add sort_by[asc] to prevent data overwrite by oldest deleted records
+        if self.SORT_BY is not None:
+            params['sort_by[asc]'] = self.SORT_BY
+
         LOGGER.info("Querying {} starting at {}".format(table, bookmark_date))
 
         while not done:

tap_chargebee/streams/coupons.py

@@ -12,6 +12,7 @@ class CouponsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'created_at' # https://apidocs.chargebee.com/docs/api/coupons#list_coupons_sort_by
 
 
     def get_url(self):

tap_chargebee/streams/credit_notes.py

@@ -12,6 +12,7 @@ class CreditNotesStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'date' # https://apidocs.chargebee.com/docs/api/credit_notes#list_credit_notes_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/credit_notes'.format(self.config.get('site'))

tap_chargebee/streams/customers.py

@@ -12,6 +12,7 @@ class CustomersStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'updated_at' # https://apidocs.chargebee.com/docs/api/customers#list_customers_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/customers'.format(self.config.get('site'))

tap_chargebee/streams/events.py

@@ -12,6 +12,7 @@ class EventsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['occurred_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'occurred_at' # https://apidocs.chargebee.com/docs/api/events#list_events_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/events'.format(self.config.get('site'))

tap_chargebee/streams/gifts.py

@@ -12,6 +12,7 @@ class GiftsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/gifts'.format(self.config.get('site'))

tap_chargebee/streams/invoices.py

@@ -12,6 +12,7 @@ class InvoicesStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'updated_at' # https://apidocs.chargebee.com/docs/api/invoices#list_invoices_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/invoices'.format(self.config.get('site'))

tap_chargebee/streams/orders.py

@@ -12,6 +12,7 @@ class OrdersStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'updated_at' # https://apidocs.chargebee.com/docs/api/orders#list_orders_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/orders'.format(self.config.get('site'))

tap_chargebee/streams/payment_sources.py

@@ -12,6 +12,7 @@ class PaymentSourcesStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/payment_sources'.format(self.config.get('site'))

tap_chargebee/streams/plans.py

@@ -12,6 +12,7 @@ class PlansStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/plans'.format(self.config.get('site'))

tap_chargebee/streams/promotional_credits.py

@@ -12,6 +12,7 @@ class PromotionalCreditsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['created_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/promotional_credits'.format(self.config.get('site'))

tap_chargebee/streams/subscriptions.py

@@ -12,6 +12,7 @@ class SubscriptionsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'updated_at' # https://apidocs.chargebee.com/docs/api/subscriptions#list_subscriptions_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/subscriptions'.format(self.config.get('site'))

tap_chargebee/streams/transactions.py

@@ -12,6 +12,7 @@ class TransactionsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = 'updated_at' # https://apidocs.chargebee.com/docs/api/transactions#list_transactions_sort_by
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/transactions'.format(self.config.get('site'))

tap_chargebee/streams/virtual_bank_accounts.py

@@ -12,6 +12,7 @@ class VirtualBankAccountsStream(BaseChargebeeStream):
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
     API_METHOD = 'GET'
+    SORT_BY = None
 
     def get_url(self):
         return 'https://{}.chargebee.com/api/v2/virtual_bank_accounts'.format(self.config.get('site'))