diff --git a/README.md b/README.md index a663d2d..3f9b579 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ pip install git+https://gitlab.com/meltano/tap-gitlab.git "fetch_pipelines_extended": false, "fetch_retried_jobs": false, "fetch_group_variables": false, - "fetch_project_variables": false + "fetch_project_variables": false, + "fetch_bridges": false } ``` @@ -99,6 +100,8 @@ pip install git+https://gitlab.com/meltano/tap-gitlab.git If `fetch_project_variables` is true (defaults to false), then Project-level CI/CD variables will be retrieved for each available / specified project. This feature is treated as an opt-in to prevent users from accidentally extracting any potential secrets stored as Project-level CI/CD variables. + If `fetch_bridges` is true (defaults to false), then Pipelines triggered from other pipelines will be retrieved. + 4. [Optional] Create the initial state file You can provide JSON file that contains a date for the API endpoints diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 8303361..5cb5d70 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -27,6 +27,8 @@ 'fetch_retried_jobs': False, 'fetch_group_variables': False, 'fetch_project_variables': False, + # NOTE: bridges are the name gitlab gives to triggered pipelines + 'fetch_bridges': False, } STATE = {} CATALOG = None @@ -205,7 +207,13 @@ def load_schema(entity): 'schema': load_schema('group_variables'), 'key_properties': ['group_id', 'key'], 'replication_method': 'FULL_TABLE', - } + }, + "bridges": { + "url": "/projects/{id}/pipelines/{secondary_id}/bridges", + "schema": load_schema("bridges"), + "key_properties": ["id"], + "replication_method": "FULL_TABLE", + }, } ULTIMATE_RESOURCES = ("epics", "epic_issues") @@ -214,6 +222,7 @@ def load_schema(entity): 'pipelines_extended', 'group_variables', 'project_variables', + 'bridges', ) LOGGER = singer.get_logger() @@ -733,8 +742,39 @@ def sync_pipelines(project): # it's pipeline's updated_at is changed. sync_jobs(project, transformed_row) + sync_bridges(project, transformed_row) + singer.write_state(STATE) + +def sync_bridges(project, pipeline): + entity = "bridges" + stream = CATALOG.get_stream(entity) + if not stream.is_selected(): + return + + mdata = metadata.to_map(stream.metadata) + url = get_url(entity=entity, id=project["id"], secondary_id=pipeline["id"]) + + with Transformer(pre_hook=format_timestamp) as transformer: + for row in gen_request(url): + row["project_id"] = project["id"] + transformed_row = transformer.transform( + row, RESOURCES[entity]["schema"], mdata + ) + + singer.write_record(entity, transformed_row, time_extracted=utils.now()) + + if transformed_row["downstream_pipeline"]: + # Sync additional details of a pipeline using get-a-single-pipeline endpoint + # https://docs.gitlab.com/ee/api/pipelines.html#get-a-single-pipeline + sync_pipelines_extended(project, transformed_row["downstream_pipeline"]) + + # Sync all jobs attached to the pipeline. + # Although jobs cannot be queried by updated_at, if a job changes + # it's pipeline's updated_at is changed. + sync_jobs(project, transformed_row["downstream_pipeline"]) + def sync_pipelines_extended(project, pipeline): entity = "pipelines_extended" stream = CATALOG.get_stream(entity) @@ -938,6 +978,7 @@ def main_impl(): CONFIG['fetch_retried_jobs'] = truthy(CONFIG['fetch_retried_jobs']) CONFIG['fetch_group_variables'] = truthy(CONFIG['fetch_group_variables']) CONFIG['fetch_project_variables'] = truthy(CONFIG['fetch_project_variables']) + CONFIG['fetch_bridges'] = truthy(CONFIG['fetch_bridges']) if '/api/' not in CONFIG['api_url']: CONFIG['api_url'] += '/api/v4' diff --git a/tap_gitlab/schemas/bridges.json b/tap_gitlab/schemas/bridges.json new file mode 100644 index 0000000..0e59c28 --- /dev/null +++ b/tap_gitlab/schemas/bridges.json @@ -0,0 +1,345 @@ +{ + "type": "object", + "properties": { + "commit": { + "type": "object", + "properties": { + "author_email": { + "type": [ + "null", + "string" + ] + }, + "author_name": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + }, + "short_id": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + } + } + }, + "allow_failure": { + "type": [ + "null", + "boolean" + ] + }, + "archived": { + "type": [ + "null", + "boolean" + ] + }, + "queued_duration": { + "type": [ + "null", + "number" + ], + "format": "float" + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "pipeline": { + "type": "object", + "properties": { + "sha": { + "type": [ + "null", + "string" + ] + }, + "project_id": { + "type": [ + "null", + "integer" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "ref": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "web_url": { + "type": [ + "null", + "string" + ] + } + } + }, + "stage": { + "type": [ + "null", + "string" + ] + }, + "project": { + "type": "object", + "properties": { + "ci_job_token_scope_enabled": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "ref": { + "type": [ + "null", + "string" + ] + }, + "tag": { + "type": [ + "null", + "boolean" + ] + }, + "user": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "state": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "web_url": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "bio": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "string" + ] + }, + "public_email": { + "type": [ + "null", + "string" + ] + }, + "skype": { + "type": [ + "null", + "string" + ] + }, + "linkedin": { + "type": [ + "null", + "string" + ] + }, + "twitter": { + "type": [ + "null", + "string" + ] + }, + "website_url": { + "type": [ + "null", + "string" + ] + }, + "organization": { + "type": [ + "null", + "string" + ] + } + } + }, + "downstream_pipeline": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" + }, + "sha": { + "type": [ + "null", + "string" + ] + }, + "ref": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "web_url": { + "type": [ + "null", + "string" + ] + } + } + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "started_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "erased_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "finished_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "duration": { + "type": [ + "null", + "integer" + ] + }, + "coverage": { + "type": [ + "null", + "number" + ], + "format": "float" + }, + "web_url": { + "type": [ + "null", + "string" + ] + } + } +} \ No newline at end of file