-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* Create API importer for Bugcrowd * Fix linting * Documentation update * Implement URI extraction via regex, pagination fetch loop, switch to unique id from tool deduplication alg * Update api_client.py * Various fixes * Fix dateutil parse and auth header * Fix linting * Switch to session * Implement unit testing - WIP * Bugcrowd api importer unit tests * Fix flake8 * Simplify parameterization for bugcrowd JSONAPI format * Fix urlencoding and loop for pagination * Implement generator api client * v3 of fetcher with multithreading * Linting with Black, test data changed for generator function, fix tests * fix pep8 and add ignore W503 in flake8 * remove json from test * Use logger for endpoint parsing errors, without breaking parser * Strip bug url to improve endpoint parsing * Remove regex usage * Handle endpoint uri a bit better * use logger error for endpoint converting * Improve requests exception handling * Remove regexes, convert_endpoint function * Raise exeptions for responses and connection tests * Do not save broken endpoints, add cleaning in tests * Align to dev branch * Named ValidationError exceptions * Fix conflicts * Fix conflicts * Add response text in error message * Fix liniting * Update __init__.py Co-authored-by: Damien Carol <[email protected]>
- Loading branch information
1 parent
83051b4
commit 9bb46ba
Showing
15 changed files
with
1,469 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| import requests | ||
| from urllib.parse import urlencode | ||
| from dojo.models import Tool_Type | ||
|
|
||
|
|
||
| class BugcrowdAPI: | ||
| """ | ||
| A simple client for the bugcrowd.io API | ||
| """ | ||
|
|
||
| bugcrowd_api_url = "https://api.bugcrowd.com" | ||
| default_headers = { | ||
| "Accept": "application/vnd.bugcrowd+json", | ||
| "User-Agent": "DefectDojo", | ||
| "Bugcrowd-Version": "2021-10-28", | ||
| } | ||
|
|
||
| def __init__(self, tool_config): | ||
| Tool_Type.objects.get_or_create(name="Bugcrowd API") | ||
|
|
||
| self.session = requests.Session() | ||
| if tool_config.authentication_type == "API": | ||
| self.api_token = tool_config.api_key | ||
| self.session.headers.update( | ||
| {"Authorization": "Token {}".format(self.api_token)} | ||
| ) | ||
| self.session.headers.update(self.default_headers) | ||
| else: | ||
| raise Exception( | ||
| "bugcrowd Authentication type {} not supported".format( | ||
| tool_config.authentication_type | ||
| ) | ||
| ) | ||
|
|
||
| def get_findings(self, program, target): | ||
| """ | ||
| Returns the findings in a paginated iterator for a given bugcrowd program and target, if target is *, everything is returned | ||
| :param program: | ||
| :param target: | ||
| :return: | ||
| """ | ||
| params_default = { | ||
| "filter[duplicate]": "false", | ||
| "filter[program]": program, | ||
| "page[limit]": 100, | ||
| "page[offset]": 0, | ||
| "include": "monetary_rewards,target", | ||
| "sort": "submitted-desc", | ||
| } | ||
|
|
||
| if target: | ||
| params = params_default | ||
| params["filter[target]"] = target | ||
| params_encoded = urlencode(params) | ||
| else: | ||
| params_encoded = urlencode(params_default) | ||
|
|
||
| next = "{}/submissions?{}".format(self.bugcrowd_api_url, params_encoded) | ||
| while next != "": | ||
| response = self.session.get(url=next) | ||
| response.raise_for_status() | ||
| if response.ok: | ||
| data = response.json() | ||
| if len(data["data"]) != 0: | ||
| yield data["data"] | ||
|
|
||
| # When we hit the end of the submissions, break out | ||
| if len(data["data"]) == 0: | ||
| next = "" | ||
| break | ||
|
|
||
| # Otherwise, keep updating next link | ||
| next = "{}{}".format(self.bugcrowd_api_url, data["links"]["next"]) | ||
| else: | ||
| next = "over" | ||
|
|
||
| def test_connection(self): | ||
| # Request programs | ||
| response_programs = self.session.get( | ||
| url="{}/programs".format(self.bugcrowd_api_url) | ||
| ) | ||
| response_programs.raise_for_status() | ||
|
|
||
| # Request submissions to validate the org token | ||
| response_subs = self.session.get( | ||
| url="{}/submissions".format(self.bugcrowd_api_url) | ||
| ) | ||
| response_subs.raise_for_status() | ||
| if response_programs.ok and response_subs.ok: | ||
| data = response_programs.json().get("data") | ||
| data_subs = response_subs.json().get("meta") | ||
| total_subs = str(data_subs["total_hits"]) | ||
|
|
||
| progs = list(filter(lambda prog: prog["type"] == "program", data)) | ||
| program_names = ", ".join( | ||
| list(map(lambda p: p["attributes"]["code"], progs)) | ||
| ) | ||
| # Request targets to validate the org token | ||
| response_targets = self.session.get( | ||
| url="{}/targets".format(self.bugcrowd_api_url) | ||
| ) | ||
| response_targets.raise_for_status() | ||
| if response_targets.ok: | ||
| data_targets = response_targets.json().get("data") | ||
| targets = list( | ||
| filter(lambda prog: prog["type"] == "target", data_targets) | ||
| ) | ||
| target_names = ", ".join( | ||
| list(map(lambda p: p["attributes"]["name"], targets)) | ||
| ) | ||
| return f'With {total_subs} submissions, you have access to the "{ program_names }" programs, \ | ||
| you can use these as Service key 1 for filtering submissions \ | ||
| You also have targets "{ target_names }" that can be used in Service key 2' | ||
| else: | ||
| raise Exception( | ||
| "Bugcrowd API test not successful, no targets were defined in Bugcrowd which is used for filtering, check your configuration, HTTP response was: {}".format( | ||
| response_targets.text | ||
| ) | ||
| ) | ||
| else: | ||
| raise Exception( | ||
| "Bugcrowd API test not successful, could not retrieve the programs or submissions, check your configuration, HTTP response for programs was: {}, HTTP response for submissions was: {}".format( | ||
| response_programs.text, response_subs.text | ||
| ) | ||
| ) | ||
|
|
||
| def test_product_connection(self, api_scan_configuration): | ||
| submissions = [] | ||
| submission_gen = self.get_findings( | ||
| api_scan_configuration.service_key_1, api_scan_configuration.service_key_2 | ||
| ) | ||
| for page in submission_gen: | ||
| submissions = submissions + page | ||
| submission_number = len(submissions) | ||
| return f'You have access to "{submission_number}" submissions (no duplicates)\ | ||
| in Bugcrowd in the Program code "{api_scan_configuration.service_key_1}" \ | ||
| and Target "{api_scan_configuration.service_key_2}" (leave service key 2 empty to get all submissions in program)' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| import logging | ||
| from django.core.exceptions import ValidationError | ||
| from dojo.models import Product_API_Scan_Configuration | ||
| from dojo.tools.bugcrowd_api.api_client import BugcrowdAPI | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class BugcrowdApiImporter(object): | ||
| """ | ||
| Import from Bugcrowd API | ||
| """ | ||
|
|
||
| def get_findings(self, test): | ||
| client, config = self.prepare_client(test) | ||
| logger.debug( | ||
| "Fetching submissions program {} and target {}".format( | ||
| str(config.service_key_1), str(config.service_key_2) | ||
| ) | ||
| ) | ||
|
|
||
| submissions_paged = client.get_findings( | ||
| config.service_key_1, | ||
| config.service_key_2, | ||
| ) | ||
|
|
||
| submissions = [] | ||
| counter = 0 | ||
| for page in submissions_paged: | ||
| submissions += page | ||
| counter += 1 | ||
| logger.debug("{} Bugcrowd submissions pages fetched".format(counter)) | ||
|
|
||
| return submissions | ||
|
|
||
| def prepare_client(self, test): | ||
| product = test.engagement.product | ||
| if test.api_scan_configuration: | ||
| config = test.api_scan_configuration | ||
| # Double check of config | ||
| if config.product != product: | ||
| raise ValidationError( | ||
| "API Scan Configuration for Bugcrowd API and Product do not match." | ||
| ) | ||
| else: | ||
| configs = Product_API_Scan_Configuration.objects.filter( | ||
| product=product, tool_configuration__tool_type__name="Bugcrowd API" | ||
| ) | ||
| if configs.count() == 1: | ||
| config = configs.first() | ||
| elif configs.count() > 1: | ||
| raise ValidationError( | ||
| "More than one Product API Scan Configuration has been configured, but none of them has been chosen.\ | ||
| Please specify at Test which one should be used." | ||
| ) | ||
| else: | ||
| raise ValidationError( | ||
| "There are no API Scan Configurations for this Product. \ | ||
| Please add at least one API Scan Configuration for bugcrowd to this Product." | ||
| ) | ||
|
|
||
| tool_config = config.tool_configuration | ||
| return BugcrowdAPI(tool_config), config |
Oops, something went wrong.