Add option to 'ignore_access_denied'

README.md

@@ -39,6 +39,7 @@ Built with the [Meltano SDK](https://sdk.meltano.com) for Singer Taps and Target
 | stream_map_config          | False    | None    | User-defined config values to be used within map expressions. |
 | flattening_enabled         | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
 | flattening_max_depth       | False    | None    | The max depth to flatten schemas. |
+| ignore_access_denied       | False    | False   | True to ignore access denied (HTTP 401) errors. This option is provided for compatibility with prior versions of this tap, which would silently move on to the next stream after receiving an access denied error. If 'False' (the default), the tap will fail if access denied errors are received for any selected streams. |
 
 A full list of supported settings and capabilities is available by running: `tap-gitlab --about`
 
@@ -83,6 +84,18 @@ Notes on group and project options:
 - If using the `--test=schema` option, some value (which is ignored) must be provided for the `projects` or `groups`
   settings or none of the corresponding streams will be included in the output.
 
+## Migration guide
+
+Coming from a prior version of this tap, here are some coniderations.
+
+1. Silent failures on access denied (401) errors.
+   - Prior versions of this tap would silently continue whenever a 401 error was received. To replicated this behavior, use the new `ignore_access_denied` config key.
+   - The long-term and proper fix is to deselect any streams you don't have access to, or else give elevated permissions (Maintaner, for instance) to the access token.
+2. Group and project streams when only `projects` or `groups` are specified.
+   - Prior versions of this tap would send schema messages for _both_ projects and schemas, regardless of whether both `groups` and `projects` config values were set. This would result in empty tables being created for project and groups, even if one of these had no data to sync.
+   - To replicate this behavior, provide both `groups` and `projects`. If you access token does not have full access to groups (for instance), refer to the guidance above to ignore access denied messages.
+   - The long-term and proper fix is to remove dependencies on `group` tables if group is not specified, and vice versa for `projects` if `projects` is not set.
+
 ## Installation
 
 ```bash

tap_gitlab/client.py

@@ -6,7 +6,7 @@
 import urllib
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union, cast
+from typing import Any, Dict, Iterable, List, Optional, Union, cast
 from urllib.parse import urlparse
 
 import requests
@@ -21,6 +21,16 @@
 SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
 
 
+def _truthy(val: Any) -> bool:
+    """Convert strings from env vars and settings to booleans."""
+    if isinstance(val, str):
+        if val.lower() in ["false", "0"]:
+            return False
+
+    # Convert val to bool
+    return not not val  # pylint: disable=C0113
+
+
 class GitLabStream(RESTStream):
     """GitLab stream class."""
 
@@ -84,7 +94,6 @@ def get_url_params(
     ) -> Dict[str, Any]:
         """Return a dictionary of values to be used in URL parameterization."""
         # If the class has extra default params, start with those:
-        # TODO: SDK Bug: without copy(), this will leak params across classes/objects.
         params: dict = copy.copy(self.extra_url_params or {})
 
         if next_page_token:
@@ -140,6 +149,57 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[di
 
         return result
 
+    def validate_response(self, response: requests.Response) -> None:
+        """Validate HTTP response.
+
+        Overrides the base class in order to ignore 401 access denied errors if the
+        config value 'ignore_access_denied' is True.
+
+        Args:
+            response: A `requests.Response`_ object.
+
+        Raises:
+            FatalAPIError: If the request is not retriable.
+            RetriableAPIError: If the request is retriable.
+
+        .. _requests.Response:
+            https://docs.python-requests.org/en/latest/api/#requests.Response
+        """
+        if (
+            _truthy(self.config.get("ignore_access_denied", False))
+            and response.status_code == 401
+        ):
+            self.logger.info(
+                "Ignoring 401 access denied error "
+                "('ignore_access_denied' setting is True)."
+            )
+            return
+
+        super().validate_response(response)
+
+    def parse_response(self, response: requests.Response) -> Iterable[dict]:
+        """Parse the response and return an iterator of result rows.
+
+        We override this method in order to skip over any 'access_denied' (401) errors
+        and avoid parsing those responses as records.
+
+        Args:
+            response: A raw `requests.Response`_ object.
+
+        Yields:
+            One item for every item found in the response.
+
+        .. _requests.Response:
+            https://docs.python-requests.org/en/latest/api/#requests.Response
+        """
+        if (
+            _truthy(self.config.get("ignore_access_denied", False))
+            and response.status_code == 401
+        ):
+            yield from ()
+
+        yield from super().parse_response(response)
+
 
 class ProjectBasedStream(GitLabStream):
     """Base class for streams that are keys based on project ID."""

tap_gitlab/tap.py

@@ -138,6 +138,19 @@ class TapGitLab(Tap):
                 "recorded to this path as it is received."
             ),
         ),
+        th.Property(
+            "ignore_access_denied",
+            th.BooleanType,
+            required=False,
+            default=False,
+            description=(
+                "True to ignore access denied (HTTP 401) errors. This option is "
+                "provided for compatibility with prior versions of this tap, which "
+                "would silently move on to the next stream after receiving an access "
+                "denied error. If 'False', the tap will fail if access denied errors "
+                "are received for any selected streams."
+            ),
+        ),
     ).to_dict()
 
     def discover_streams(self) -> List[Stream]:

tox.ini

@@ -45,9 +45,10 @@ commands =
     poetry run mypy tap_gitlab --exclude='tap_gitlab/tests'
 
 [flake8]
-ignore = W503
+ignore = W503, C901, ANN101
 max-line-length = 88
 max-complexity = 10
+docstring-convention = google
 
 [pydocstyle]
 ignore = D105,D203,D213