diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..55ec8d7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 120 diff --git a/src/secret_scanning.py b/src/secret_scanning.py index 9400eb8..476c2ac 100644 --- a/src/secret_scanning.py +++ b/src/secret_scanning.py @@ -18,12 +18,12 @@ def get_repo_ss_alerts(api_endpoint, github_pat, repo_name): - List of _all_ secret scanning alerts on the repository (both default and generic secret types) """ # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based - url_default = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=" + url_default = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&hide_secret=true" ss_alerts_default = api_helpers.make_api_call(url_default, github_pat) # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key" - url_generic = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}" + url_generic = f"{api_endpoint}/repos/{repo_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true" ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat) # Combine results and deduplicate @@ -73,6 +73,7 @@ def write_repo_ss_list(secrets_list): [ "number", "created_at", + "updated_at", "html_url", "state", "resolution", @@ -80,24 +81,71 @@ def write_repo_ss_list(secrets_list): "resolved_by_username", "resolved_by_type", "resolved_by_isadmin", + "resolution_comment", "secret_type", "secret_type_display_name", + "validity", + "publicly_leaked", + "multi_repo", + "is_base64_encoded", + "first_location_path", + "first_location_start_line", + "first_location_commit_sha", + "push_protection_bypassed", + "push_protection_bypassed_by", + "push_protection_bypassed_at", + "push_protection_bypass_request_reviewer", + "push_protection_bypass_request_reviewer_comment", + "push_protection_bypass_request_comment", + "push_protection_bypass_request_html_url", + "assigned_to", ] ) for alert in secrets_list: + first_location = alert.get("first_location_detected") or {} + writer.writerow( [ alert["number"], alert["created_at"], + alert["updated_at"], alert["html_url"], alert["state"], alert["resolution"], alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]), + alert.get("resolution_comment", ""), alert["secret_type"], alert["secret_type_display_name"], + alert["validity"], + str(alert["publicly_leaked"]), + str(alert["multi_repo"]), + str(alert["is_base64_encoded"]), + first_location.get("path") + or first_location.get("pull_request_body_url") + or first_location.get("issue_body_url") + or first_location.get("discussion_body_url") + or "", + ("" if first_location is None else first_location.get("start_line", "")), + ("" if first_location is None else first_location.get("commit_sha", "")), + str(alert["push_protection_bypassed"]), + ( + "" + if alert.get("push_protection_bypassed_by") is None + else alert["push_protection_bypassed_by"].get("login", "") + ), + alert.get("push_protection_bypassed_at", ""), + ( + "" + if alert.get("push_protection_bypass_request_reviewer") is None + else alert["push_protection_bypass_request_reviewer"].get("login", "") + ), + alert.get("push_protection_bypass_request_reviewer_comment", ""), + alert.get("push_protection_bypass_request_comment", ""), + alert.get("push_protection_bypass_request_html_url", ""), + ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")), ] ) @@ -115,32 +163,32 @@ def get_org_ss_alerts(api_endpoint, github_pat, org_name): - List of _all_ secret scanning alerts on the organization (both default and generic secret types) """ # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based - url_default = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=" + url_default = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&hide_secret=true" ss_alerts_default = api_helpers.make_api_call(url_default, github_pat) # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key" - url_generic = ( - f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}" - ) + url_generic = f"{api_endpoint}/orgs/{org_name}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true" ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat) - # Combine results and deduplicate + # Combine results and deduplicate using composite key (repo + alert number) combined_alerts = [] - alert_numbers_seen = set() + alert_keys_seen = set() # Composite key: (repo, alert_number) duplicates_found = False # Add default alerts for alert in ss_alerts_default: - alert_numbers_seen.add(alert["number"]) + alert_key = (alert["repository"]["full_name"], alert["number"]) + alert_keys_seen.add(alert_key) combined_alerts.append(alert) # Add generic alerts, checking for duplicates for alert in ss_alerts_generic: - if alert["number"] in alert_numbers_seen: + alert_key = (alert["repository"]["full_name"], alert["number"]) + if alert_key in alert_keys_seen: duplicates_found = True else: - alert_numbers_seen.add(alert["number"]) + alert_keys_seen.add(alert_key) combined_alerts.append(alert) # Warn if duplicates were found @@ -172,6 +220,7 @@ def write_org_ss_list(secrets_list): [ "number", "created_at", + "updated_at", "html_url", "state", "resolution", @@ -179,8 +228,24 @@ def write_org_ss_list(secrets_list): "resolved_by_username", "resolved_by_type", "resolved_by_isadmin", + "resolution_comment", "secret_type", "secret_type_display_name", + "validity", + "publicly_leaked", + "multi_repo", + "is_base64_encoded", + "first_location_path", + "first_location_start_line", + "first_location_commit_sha", + "push_protection_bypassed", + "push_protection_bypassed_by", + "push_protection_bypassed_at", + "push_protection_bypass_request_reviewer", + "push_protection_bypass_request_reviewer_comment", + "push_protection_bypass_request_comment", + "push_protection_bypass_request_html_url", + "assigned_to", "repo_name", "repo_owner", "repo_owner_type", @@ -191,19 +256,50 @@ def write_org_ss_list(secrets_list): ] ) for alert in secrets_list: + first_location = alert.get("first_location_detected") or {} + writer.writerow( [ alert["number"], alert["created_at"], + alert["updated_at"], alert["html_url"], alert["state"], alert["resolution"], alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]), + alert.get("resolution_comment", ""), alert["secret_type"], alert["secret_type_display_name"], + alert["validity"], + str(alert["publicly_leaked"]), + str(alert["multi_repo"]), + str(alert["is_base64_encoded"]), + first_location.get("path") + or first_location.get("pull_request_body_url") + or first_location.get("issue_body_url") + or first_location.get("discussion_body_url") + or "", + ("" if first_location is None else first_location.get("start_line", "")), + ("" if first_location is None else first_location.get("commit_sha", "")), + str(alert["push_protection_bypassed"]), + ( + "" + if alert.get("push_protection_bypassed_by") is None + else alert["push_protection_bypassed_by"].get("login", "") + ), + alert.get("push_protection_bypassed_at", ""), + ( + "" + if alert.get("push_protection_bypass_request_reviewer") is None + else alert["push_protection_bypass_request_reviewer"].get("login", "") + ), + alert.get("push_protection_bypass_request_reviewer_comment", ""), + alert.get("push_protection_bypass_request_comment", ""), + alert.get("push_protection_bypass_request_html_url", ""), + ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")), alert["repository"]["full_name"], alert["repository"]["owner"]["login"], alert["repository"]["owner"]["type"], @@ -229,30 +325,42 @@ def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug): - List of _all_ secret scanning alerts on the enterprise (both default and generic secret types) """ # First call: get default secret types (without any filters), use after= to force object based cursor instead of page based - url_default = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=" + url_default = ( + f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&hide_secret=true" + ) ss_alerts_default = api_helpers.make_api_call(url_default, github_pat) # Second call: get generic secret types with hardcoded list, use after= to force object based cursor instead of page based generic_secret_types = "password,http_basic_authentication_header,http_bearer_authentication_header,mongodb_connection_string,mysql_connection_string,openssh_private_key,pgp_private_key,postgres_connection_string,rsa_private_key" - url_generic = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}" + url_generic = f"{api_endpoint}/enterprises/{enterprise_slug}/secret-scanning/alerts?per_page=100&after=&secret_type={generic_secret_types}&hide_secret=true" ss_alerts_generic = api_helpers.make_api_call(url_generic, github_pat) - # Combine results and deduplicate + # Combine results and deduplicate using composite key (org + repo + alert number) combined_alerts = [] - alert_numbers_seen = set() + alert_keys_seen = set() # Composite key: (org, repo, alert_number) duplicates_found = False # Add default alerts for alert in ss_alerts_default: - alert_numbers_seen.add(alert["number"]) + alert_key = ( + alert["repository"]["owner"]["login"], + alert["repository"]["name"], + alert["number"], + ) + alert_keys_seen.add(alert_key) combined_alerts.append(alert) # Add generic alerts, checking for duplicates for alert in ss_alerts_generic: - if alert["number"] in alert_numbers_seen: + alert_key = ( + alert["repository"]["owner"]["login"], + alert["repository"]["name"], + alert["number"], + ) + if alert_key in alert_keys_seen: duplicates_found = True else: - alert_numbers_seen.add(alert["number"]) + alert_keys_seen.add(alert_key) combined_alerts.append(alert) # Warn if duplicates were found @@ -284,6 +392,7 @@ def write_enterprise_ss_list(secrets_list): [ "number", "created_at", + "updated_at", "html_url", "state", "resolution", @@ -291,8 +400,24 @@ def write_enterprise_ss_list(secrets_list): "resolved_by_username", "resolved_by_type", "resolved_by_isadmin", + "resolution_comment", "secret_type", "secret_type_display_name", + "validity", + "publicly_leaked", + "multi_repo", + "is_base64_encoded", + "first_location_path", + "first_location_start_line", + "first_location_commit_sha", + "push_protection_bypassed", + "push_protection_bypassed_by", + "push_protection_bypassed_at", + "push_protection_bypass_request_reviewer", + "push_protection_bypass_request_reviewer_comment", + "push_protection_bypass_request_comment", + "push_protection_bypass_request_html_url", + "assigned_to", "repo_name", "repo_owner", "repo_owner_type", @@ -303,19 +428,50 @@ def write_enterprise_ss_list(secrets_list): ] ) for alert in secrets_list: + first_location = alert.get("first_location_detected") or {} + writer.writerow( [ alert["number"], alert["created_at"], + alert["updated_at"], alert["html_url"], alert["state"], alert["resolution"], alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + ("" if alert["resolved_by"] is None else alert["resolved_by"]["login"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["type"]), + ("" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"]), + alert.get("resolution_comment", ""), alert["secret_type"], alert["secret_type_display_name"], + alert["validity"], + str(alert["publicly_leaked"]), + str(alert["multi_repo"]), + str(alert["is_base64_encoded"]), + first_location.get("path") + or first_location.get("pull_request_body_url") + or first_location.get("issue_body_url") + or first_location.get("discussion_body_url") + or "", + ("" if first_location is None else first_location.get("start_line", "")), + ("" if first_location is None else first_location.get("commit_sha", "")), + str(alert["push_protection_bypassed"]), + ( + "" + if alert.get("push_protection_bypassed_by") is None + else alert["push_protection_bypassed_by"].get("login", "") + ), + alert.get("push_protection_bypassed_at", ""), + ( + "" + if alert.get("push_protection_bypass_request_reviewer") is None + else alert["push_protection_bypass_request_reviewer"].get("login", "") + ), + alert.get("push_protection_bypass_request_reviewer_comment", ""), + alert.get("push_protection_bypass_request_comment", ""), + alert.get("push_protection_bypass_request_html_url", ""), + ("" if alert.get("assigned_to") is None else alert["assigned_to"].get("login", "")), alert["repository"]["full_name"], alert["repository"]["owner"]["login"], alert["repository"]["owner"]["type"],