Merge pull request #138 from Kinto/136-add-pages-to-paginated-methods

leplatrem · web-flow · commit 35eea3dd2cee · 2017-05-10T12:52:31.000+02:00
Add pages options for pagination (fixes #136)
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -12,6 +12,8 @@ This document describes changes between each past release.
 - Keep tracks of Backoff headers and raise an ``BackoffException`` if
   we are not waiting enough between two calls. (#53)
 - Add ``--retry`` and ``--retry-after`` to CLI utils helpers (fixes #126)
+- Fetch only one page when ``_limit`` is specified and allow to override this
+  with a ``pages`` argument (fixes #136)
 
 **Bug fixes**
 
diff --git a/README.rst b/README.rst
@@ -367,6 +367,26 @@ It is possible (but not recommended) to force this value in the clients:
                   retry=10,
                   retry_after=5)
 
+Pagination
+----------
+
+When the server responses are paginated, the client will download every pages and
+merge them transparently.
+
+However, it is possible to specify a limit for the number of items to be retrieved
+in one page:
+
+.. code-block:: python
+
+    records = client.get_records(_limit=10)
+
+In order to retrieve every available pages with a limited number of items in each
+of them, you can specify the number of pages:
+
+.. code-block:: python
+
+    records = client.get_records(_limit=10, pages=float('inf'))  # Infinity
+
 
 Generating endpoint paths
 -------------------------
diff --git a/kinto_http/__init__.py b/kinto_http/__init__.py
@@ -121,13 +121,16 @@ def get_endpoint(self, name, bucket=None, group=None, collection=None, id=None):
         }
         return self.endpoints.get(name, **kwargs)
 
-    def _paginated(self, endpoint, records=None, if_none_match=None, **kwargs):
+    def _paginated(self, endpoint, records=None, if_none_match=None, pages=None, **kwargs):
         if records is None:
             records = collections.OrderedDict()
         headers = {}
         if if_none_match is not None:
             headers['If-None-Match'] = utils.quote(if_none_match)
 
+        if pages is None:
+            pages = 1 if '_limit' in kwargs else float('inf')
+
         record_resp, headers = self.session.request(
             'get', endpoint, headers=headers, params=kwargs)
 
@@ -139,12 +142,13 @@ def _paginated(self, endpoint, records=None, if_none_match=None, **kwargs):
             records_tuples = [(r['id'], r) for r in record_resp['data']]
             records.update(collections.OrderedDict(records_tuples))
 
-            if 'next-page' in map(str.lower, headers.keys()):
+            if pages > 1 and 'next-page' in map(str.lower, headers.keys()):
                 # Paginated wants a relative URL, but the returned one is
                 # absolute.
                 next_page = headers['Next-Page']
                 return self._paginated(next_page, records,
-                                       if_none_match=if_none_match)
+                                       if_none_match=if_none_match,
+                                       pages=pages - 1)
         return list(records.values())
 
     def _get_cache_headers(self, safe, data=None, if_match=None):
@@ -239,9 +243,9 @@ def patch_bucket(self, *args, **kwargs):
         kwargs['method'] = 'patch'
         return self.update_bucket(*args, **kwargs)
 
-    def get_buckets(self):
+    def get_buckets(self, **kwargs):
         endpoint = self.get_endpoint('buckets')
-        return self._paginated(endpoint)
+        return self._paginated(endpoint, **kwargs)
 
     def get_bucket(self, bucket=None):
         endpoint = self.get_endpoint('bucket', bucket=bucket)
@@ -279,9 +283,9 @@ def delete_buckets(self, safe=True, if_match=None):
 
     # Groups
 
-    def get_groups(self, bucket=None):
+    def get_groups(self, bucket=None, **kwargs):
         endpoint = self.get_endpoint('groups', bucket=bucket)
-        return self._paginated(endpoint)
+        return self._paginated(endpoint, **kwargs)
 
     def create_group(self, group, bucket=None,
                      data=None, permissions=None,
@@ -373,9 +377,9 @@ def delete_groups(self, bucket=None, safe=True, if_match=None):
 
     # Collections
 
-    def get_collections(self, bucket=None):
+    def get_collections(self, bucket=None, **kwargs):
         endpoint = self.get_endpoint('collections', bucket=bucket)
-        return self._paginated(endpoint)
+        return self._paginated(endpoint, **kwargs)
 
     def create_collection(self, collection=None, bucket=None,
                           data=None, permissions=None, safe=True,
diff --git a/kinto_http/tests/test_client.py b/kinto_http/tests/test_client.py
@@ -674,10 +674,14 @@ def test_pagination_is_followed(self):
                 [{'id': '1', 'value': 'item1'},
                  {'id': '2', 'value': 'item2'}, ],
                 {'Next-Page': link}),
-            # Second one returns a list of items without a pagination token.
             build_response(
                 [{'id': '3', 'value': 'item3'},
                  {'id': '4', 'value': 'item4'}, ],
+                {'Next-Page': link}),
+            # Second one returns a list of items without a pagination token.
+            build_response(
+                [{'id': '5', 'value': 'item5'},
+                 {'id': '6', 'value': 'item6'}, ],
             ),
         ]
         records = self.client.get_records('bucket', 'collection')
@@ -687,6 +691,60 @@ def test_pagination_is_followed(self):
             {'id': '2', 'value': 'item2'},
             {'id': '3', 'value': 'item3'},
             {'id': '4', 'value': 'item4'},
+            {'id': '5', 'value': 'item5'},
+            {'id': '6', 'value': 'item6'},
+        ]
+
+    def test_pagination_is_followed_for_number_of_pages(self):
+        # Mock the calls to request.
+        link = ('http://example.org/buckets/buck/collections/coll/records/'
+                '?token=1234')
+
+        self.session.request.side_effect = [
+            # First one returns a list of items with a pagination token.
+            build_response(
+                [{'id': '1', 'value': 'item1'},
+                 {'id': '2', 'value': 'item2'}, ],
+                {'Next-Page': link}),
+            build_response(
+                [{'id': '3', 'value': 'item3'},
+                 {'id': '4', 'value': 'item4'}, ],
+                {'Next-Page': link}),
+            # Second one returns a list of items without a pagination token.
+            build_response(
+                [{'id': '5', 'value': 'item5'},
+                 {'id': '6', 'value': 'item6'}, ],
+            ),
+        ]
+        records = self.client.get_records('bucket', 'collection', pages=2)
+
+        assert list(records) == [
+            {'id': '1', 'value': 'item1'},
+            {'id': '2', 'value': 'item2'},
+            {'id': '3', 'value': 'item3'},
+            {'id': '4', 'value': 'item4'},
+        ]
+
+    def test_pagination_is_not_followed_if_limit_is_specified(self):
+        # Mock the calls to request.
+        link = ('http://example.org/buckets/buck/collections/coll/records/'
+                '?token=1234')
+
+        self.session.request.side_effect = [
+            build_response(
+                [{'id': '1', 'value': 'item1'},
+                 {'id': '2', 'value': 'item2'}, ],
+                {'Next-Page': link}),
+            build_response(
+                [{'id': '3', 'value': 'item3'},
+                 {'id': '4', 'value': 'item4'}, ],
+            ),
+        ]
+        records = self.client.get_records('bucket', 'collection', _limit=2)
+
+        assert list(records) == [
+            {'id': '1', 'value': 'item1'},
+            {'id': '2', 'value': 'item2'}
         ]
 
     def test_pagination_supports_if_none_match(self):
