diff --git a/augur/application/config.py b/augur/application/config.py index f46f6dc278..48c16aa6e4 100644 --- a/augur/application/config.py +++ b/augur/application/config.py @@ -40,7 +40,8 @@ def redact_setting_value(section_name, setting_name, value): }, "Keys": { "github": "", - "gitlab": "" + "gitlab": "", + "github_api_base_url": "https://api.github.com" }, "Facade": { "check_updates": 1, diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 896ccd61d1..d09a33fb5d 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -8,6 +8,7 @@ from augur.application.db.models import ContributorRepo from augur.application.db.lib import bulk_insert_dicts from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth +from augur.tasks.github.util.github_api_url import get_github_api_base_url ### This worker scans all the platform users in Augur, and pulls their platform activity ### logs. Those are then used to analyze what repos each is working in (which will include repos not @@ -92,7 +93,7 @@ def contributor_breadth_model(self) -> None: print(f"Processing cntrb {index} of {total}") index += 1 - repo_cntrb_url = f"https://api.github.com/users/{cntrb['gh_login']}/events" + repo_cntrb_url = f"{get_github_api_base_url()}/users/{cntrb['gh_login']}/events" newest_event_in_db = datetime(1970, 1, 1) if cntrb["gh_login"] in cntrb_newest_events_map: diff --git a/augur/tasks/github/contributors.py b/augur/tasks/github/contributors.py index 20f796647e..0876b08e21 100644 --- a/augur/tasks/github/contributors.py +++ b/augur/tasks/github/contributors.py @@ -10,6 +10,7 @@ from augur.application.db.util import execute_session_query from augur.application.db.lib import bulk_insert_dicts, get_session, batch_insert_contributors from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth +from augur.tasks.github.util.github_api_url import get_github_api_base_url @@ -45,7 +46,7 @@ def process_contributors(): del contributor_dict["_sa_instance_state"] - url = f"https://api.github.com/users/{contributor_dict['cntrb_login']}" + url = f"{get_github_api_base_url()}/users/{contributor_dict['cntrb_login']}" data = retrieve_dict_data(url, key_auth, logger) @@ -91,12 +92,12 @@ def retrieve_dict_data(url: str, key_auth, logger): ) break - elif "You have exceeded a secondary rate limit. Please wait a few minutes before you try again" in page_data['message']: + if "You have exceeded a secondary rate limit. Please wait a few minutes before you try again" in page_data['message']: logger.info('\n\n\n\nSleeping for 100 seconds due to secondary rate limit issue.\n\n\n\n') time.sleep(100) continue - elif "You have triggered an abuse detection mechanism." in page_data['message']: + elif "You have triggered an abuse detection mechanism." in page_data['message']: # pylint: disable=no-else-continue #self.update_rate_limit(response, temporarily_disable=True,platform=platform) continue else: diff --git a/augur/tasks/github/detect_move/core.py b/augur/tasks/github/detect_move/core.py index a3eb3803d6..761527272d 100644 --- a/augur/tasks/github/detect_move/core.py +++ b/augur/tasks/github/detect_move/core.py @@ -69,7 +69,7 @@ def extract_owner_and_repo_from_endpoint(key_auth, url, logger): def ping_github_for_repo_move(session, key_auth, repo, logger,collection_hook='core'): owner, name = get_owner_repo(repo.repo_git) - url = f"https://api.github.com/repos/{owner}/{name}" + url = f"{get_github_api_base_url()}/repos/{owner}/{name}" attempts = 0 while attempts < 10: diff --git a/augur/tasks/github/events.py b/augur/tasks/github/events.py index 38a5e9e9c6..7d7bb0bf51 100644 --- a/augur/tasks/github/events.py +++ b/augur/tasks/github/events.py @@ -1,6 +1,4 @@ import logging -import traceback -import sqlalchemy as s from sqlalchemy.sql import text from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone @@ -10,11 +8,12 @@ from augur.application.db.data_parse import * from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth -from augur.tasks.github.util.github_task_session import GithubTaskManifest + from augur.tasks.github.util.util import get_owner_repo from augur.tasks.util.worker_util import remove_duplicate_dicts -from augur.application.db.models import PullRequestEvent, IssueEvent, Contributor, Repo -from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_issues_by_repo_id, get_pull_requests_by_repo_id, update_issue_closed_cntrbs_by_repo_id, get_session, get_engine, get_core_data_last_collected, batch_insert_contributors +from augur.application.db.models import PullRequestEvent, IssueEvent +from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_issues_by_repo_id, get_pull_requests_by_repo_id, update_issue_closed_cntrbs_by_repo_id, get_engine, get_core_data_last_collected, batch_insert_contributors +from augur.tasks.github.util.github_api_url import get_github_api_base_url platform_id = 1 @@ -47,7 +46,7 @@ def collect_events(repo_git: str, full_collection: bool): def bulk_events_collection_endpoint_contains_all_data(key_auth, logger, owner, repo): - url = f"https://api.github.com/repos/{owner}/{repo}/issues/events?per_page=100" + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues/events?per_page=100" github_data_access = GithubDataAccess(key_auth, logger) @@ -131,7 +130,7 @@ def _collect_events(self, repo_git: str, key_auth, since): owner, repo = get_owner_repo(repo_git) - url = f"https://api.github.com/repos/{owner}/{repo}/issues/events" + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues/events" github_data_access = GithubDataAccess(key_auth, self._logger) @@ -309,7 +308,7 @@ def _collect_and_process_issue_events(self, owner, repo, repo_id, key_auth, sinc issue_number = issue["issue_number"] - event_url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/events" + event_url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues/{issue_number}/events" try: @@ -370,7 +369,7 @@ def _collect_and_process_pr_events(self, owner, repo, repo_id, key_auth, since): pr_number = pr["gh_pr_number"] - event_url = f"https://api.github.com/repos/{owner}/{repo}/issues/{pr_number}/events" + event_url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues/{pr_number}/events" try: diff --git a/augur/tasks/github/facade_github/contributor_interfaceable/contributor_interface.py b/augur/tasks/github/facade_github/contributor_interfaceable/contributor_interface.py index 33f5ca83d7..9910ed4c0b 100644 --- a/augur/tasks/github/facade_github/contributor_interfaceable/contributor_interface.py +++ b/augur/tasks/github/facade_github/contributor_interfaceable/contributor_interface.py @@ -8,6 +8,7 @@ # Debugger from augur.tasks.github.util.github_paginator import GithubApiResult from augur.application.db.lib import get_repo_by_repo_id, bulk_insert_dicts, execute_sql, get_contributors_by_github_user_id +from augur.tasks.github.util.github_api_url import get_github_api_base_url ##TODO: maybe have a TaskSession class that holds information about the database, logger, config, etc. @@ -71,11 +72,11 @@ def request_dict_from_endpoint(logger, session, url, timeout_wait=10): success = True break - elif type(response_data) == list: + elif type(response_data) == list: # pylint: disable=no-else-break logger.warning("Wrong type returned, trying again...") logger.debug(f"Returned list: {response_data}") - elif type(response_data) == str: + elif type(response_data) == str: # pylint: disable=no-else-break logger.warning(f"Warning! page_data was string: {response_data}") if "" in response_data: logger.warning("HTML was returned, trying again...\n") @@ -92,7 +93,7 @@ def request_dict_from_endpoint(logger, session, url, timeout_wait=10): success = True break - except: + except: # pylint: disable=bare-except pass attempts += 1 @@ -106,8 +107,7 @@ def request_dict_from_endpoint(logger, session, url, timeout_wait=10): def create_endpoint_from_email(email): # Note: I added "+type:user" to avoid having user owned organizations be returned # Also stopped splitting per note above. - url = 'https://api.github.com/search/users?q={}+in:email+type:user'.format( - email) + url = f"{get_github_api_base_url()}/search/users?q={email}+in:email+type:user" return url @@ -131,7 +131,7 @@ def create_endpoint_from_commit_sha(logger, commit_sha, repo_id): split_git = result.repo_git.split('/') repo_name_and_org = split_git[-2] + "/" + result.repo_name - url = "https://api.github.com/repos/" + repo_name_and_org + "/commits/" + commit_sha + url = f"{get_github_api_base_url()}/repos/{split_git[-2]}/{result.repo_name}/commits/{commit_sha}" logger.debug(f"Commit Hash URL: {url}") @@ -151,8 +151,7 @@ def create_endpoint_from_name(contributor): # Pythonic way to get the end of a list so that we truely get the last name. 'lname': contributor[name_field].split()[-1] } - url = 'https://api.github.com/search/users?q=fullname:{}+{}'.format( - cmt_cntrb['fname'], cmt_cntrb['lname']) + url = f"{get_github_api_base_url()}/search/users?q=fullname:{cmt_cntrb['fname']}+{cmt_cntrb['lname']}" return url @@ -388,7 +387,7 @@ def get_login_with_commit_hash(logger, auth, commit_data, repo_id): try: match = login_json['author']['login'] - except: + except: # pylint: disable=bare-except match = None return match diff --git a/augur/tasks/github/facade_github/core.py b/augur/tasks/github/facade_github/core.py index 55b2281ad8..2dd66d2a95 100644 --- a/augur/tasks/github/facade_github/core.py +++ b/augur/tasks/github/facade_github/core.py @@ -5,6 +5,7 @@ from augur.tasks.util.AugurUUID import GithubUUID from augur.application.db.lib import bulk_insert_dicts, batch_insert_contributors from augur.tasks.github.util.github_data_access import GithubDataAccess +from augur.tasks.github.util.github_api_url import get_github_api_base_url @@ -26,10 +27,7 @@ def query_github_contributors(logger, key_auth, github_url): raise e # Set the base of the url and place to hold contributors to insert - contributors_url = ( - f"https://api.github.com/repos/{owner}/{name}/" + - "contributors?state=all" - ) + contributors_url = f"{get_github_api_base_url()}/repos/{owner}/{name}/contributors?state=all" # Get contributors that we already have stored # Set our duplicate and update column map keys (something other than PK) to @@ -53,7 +51,7 @@ def query_github_contributors(logger, key_auth, github_url): # Need to hit this single contributor endpoint to get extra data including... # `created at` # i think that's it - cntrb_url = ("https://api.github.com/users/" + repo_contributor['login']) + cntrb_url = f"{get_github_api_base_url()}/users/{repo_contributor['login']}" logger.info("Hitting endpoint: " + cntrb_url + " ...\n") diff --git a/augur/tasks/github/facade_github/tasks.py b/augur/tasks/github/facade_github/tasks.py index 53a3d6648a..6490f1b0e3 100644 --- a/augur/tasks/github/facade_github/tasks.py +++ b/augur/tasks/github/facade_github/tasks.py @@ -1,10 +1,12 @@ import logging +import traceback from augur.tasks.init.celery_app import celery_app as celery from augur.tasks.init.celery_app import AugurFacadeRepoCollectionTask from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth +from augur.tasks.github.util.github_api_url import get_github_api_base_url from augur.tasks.github.facade_github.core import * from augur.application.db.lib import execute_sql, get_contributor_aliases_by_email, get_unresolved_commit_emails_by_name, get_contributors_by_full_name, get_repo_by_repo_git, batch_insert_contributors from augur.application.db.lib import get_session, execute_session_query @@ -67,7 +69,7 @@ def process_commit_metadata(logger, auth, contributorQueue, repo_id, platform_id logger.error("Failed to get login from supplemental data!") continue - url = ("https://api.github.com/users/" + login) + url = f"{get_github_api_base_url()}/users/{login}" try: user_data = github_data_access.get_resource(url) diff --git a/augur/tasks/github/issues.py b/augur/tasks/github/issues.py index 91e56deaf7..f961f1190f 100644 --- a/augur/tasks/github/issues.py +++ b/augur/tasks/github/issues.py @@ -15,6 +15,7 @@ from augur.application.db.models import Issue, IssueLabel, IssueAssignee from augur.application.config import get_development_flag from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_core_data_last_collected, batch_insert_contributors +from augur.tasks.github.util.github_api_url import get_github_api_base_url development = get_development_flag() @@ -101,7 +102,7 @@ def retrieve_all_issue_data(repo_git: str, logger: logging.Logger, key_auth: Git logger.info(f"Collecting issues for {owner}/{repo}") - url = f"https://api.github.com/repos/{owner}/{repo}/issues?state=all" + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues?state=all" if since: url += f"&since={since.isoformat()}" diff --git a/augur/tasks/github/messages.py b/augur/tasks/github/messages.py index 6a888be5e7..0d81ee682a 100644 --- a/augur/tasks/github/messages.py +++ b/augur/tasks/github/messages.py @@ -11,6 +11,7 @@ from augur.application.db.models import PullRequest, Message, Issue, PullRequestMessageRef, IssueMessageRef, Contributor, Repo, CollectionStatus from augur.application.db import get_engine, get_session from augur.application.db.lib import get_core_data_last_collected +from augur.tasks.github.util.github_api_url import get_github_api_base_url from sqlalchemy.sql import text platform_id = 1 @@ -63,7 +64,7 @@ def fast_retrieve_all_pr_and_issue_messages(repo_git: str, logger, key_auth, tas owner, repo = get_owner_repo(repo_git) # url to get issue and pull request comments - url = f"https://api.github.com/repos/{owner}/{repo}/issues/comments" + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/issues/comments" if since: url += f"?since={since.isoformat()}" diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index 3d9f0a4a2d..3ed44de664 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -15,6 +15,7 @@ from augur.application.db.util import execute_session_query from ..messages import process_github_comment_contributors from augur.application.db.lib import get_secondary_data_last_collected, get_updated_prs, get_core_data_last_collected +from augur.tasks.github.util.github_api_url import get_github_api_base_url from typing import List @@ -72,7 +73,7 @@ def retrieve_all_pr_data(repo_git: str, logger, key_auth, since): #-> Generator[ logger.debug(f"Collecting pull requests for {owner}/{repo}") - url = f"https://api.github.com/repos/{owner}/{repo}/pulls?state=all&direction=desc&sort=updated" + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/pulls?state=all&direction=desc&sort=updated" github_data_access = GithubDataAccess(key_auth, logger) @@ -219,7 +220,7 @@ def collect_pull_request_review_comments(repo_git: str, full_collection: bool) - """ owner, repo = get_owner_repo(repo_git) - review_msg_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/comments" + review_msg_url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/pulls/comments" logger = logging.getLogger(collect_pull_request_review_comments.__name__) logger.debug(f"Collecting pull request review comments for {owner}/{repo}") @@ -498,7 +499,7 @@ def collect_pull_request_reviews(repo_git: str, full_collection: bool) -> None: if index % 100 == 0: logger.debug(f"{owner}/{repo} Processing PR {index + 1} of {pr_count}") - pr_review_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/reviews" + pr_review_url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/pulls/{pr_number}/reviews" try: pr_reviews = list(github_data_access.paginate_resource(pr_review_url)) diff --git a/augur/tasks/github/releases/core.py b/augur/tasks/github/releases/core.py index f6b2f5e56b..ca06e427e7 100644 --- a/augur/tasks/github/releases/core.py +++ b/augur/tasks/github/releases/core.py @@ -5,6 +5,7 @@ from augur.tasks.github.util.gh_graphql_entities import request_graphql_dict from augur.application.db.util import execute_session_query from augur.application.db.lib import bulk_insert_dicts +from augur.tasks.github.util.github_api_url import get_github_api_base_url def get_release_inf(repo_id, release, tag_only): @@ -159,7 +160,7 @@ def fetch_data(key_auth, logger, github_url, repo_id, tag_only = False): owner, repo = get_owner_repo(github_url) - url = 'https://api.github.com/graphql' + url = f"{get_github_api_base_url()}/graphql" query = get_query(logger, owner, repo, tag_only) diff --git a/augur/tasks/github/repo_info/core.py b/augur/tasks/github/repo_info/core.py index 57cd970bc0..7407a3f560 100644 --- a/augur/tasks/github/repo_info/core.py +++ b/augur/tasks/github/repo_info/core.py @@ -11,12 +11,13 @@ from augur.tasks.github.util.github_task_session import * from augur.application.db.models.augur_data import RepoBadging from urllib.parse import quote +from augur.tasks.github.util.github_api_url import get_github_api_base_url def query_committers_count(key_auth, logger, owner, repo): data = {} logger.info('Querying committers count\n') - url = f'https://api.github.com/repos/{owner}/{repo}/contributors?per_page=100' + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}/contributors?per_page=100" ## If the repository is empty there are zero committers, and the API returns nothing at all. Response ## header of 200 along with an empty JSON. try: @@ -58,7 +59,7 @@ def get_repo_data(logger, url, response): def get_repo_data(logger, owner, repo): try: - url = f'https://api.github.com/repos/{owner}/{repo}' + url = f"{get_github_api_base_url()}/repos/{owner}/{repo}" github_data_access = GithubDataAccess(None, logger) result = github_data_access.get_resource(url) return result @@ -93,7 +94,7 @@ def is_archived(logger, repo_data): return False def grab_repo_info_from_graphql_endpoint(key_auth, logger, query): - url = 'https://api.github.com/graphql' + url = f"{get_github_api_base_url()}/graphql" # Hit the graphql endpoint and retry 3 times in case of failure logger.info("Hitting endpoint: {} ...\n".format(url)) data = request_graphql_dict(key_auth, logger, url, query) @@ -284,7 +285,7 @@ def badges_model(logger,repo_git,repo_id,db): try: response_data = response.json() - except: + except (json.JSONDecodeError, AttributeError): response_data = json.loads(json.dumps(response.text)) #Insert any data that was returned diff --git a/augur/tasks/github/util/gh_graphql_entities.py b/augur/tasks/github/util/gh_graphql_entities.py index 0667ab3315..d809de3c05 100644 --- a/augur/tasks/github/util/gh_graphql_entities.py +++ b/augur/tasks/github/util/gh_graphql_entities.py @@ -7,6 +7,8 @@ import time import traceback from augur.tasks.github.util.github_paginator import GithubApiResult, process_dict_response +from augur.tasks.github.util.github_api_url import get_github_api_base_url +process_graphql_dict_response = process_dict_response """ Should be designed on a per entity basis that has attributes that call @@ -86,7 +88,7 @@ def request_graphql_dict(key_auth, logger, url,query,variables={},timeout_wait=1 try: response_data = result.json() - except: + except (json.JSONDecodeError, AttributeError): response_data = json.loads(json.dumps(result.text)) #self.logger.info(f"api return: {response_data}") @@ -101,10 +103,10 @@ def request_graphql_dict(key_auth, logger, url,query,variables={},timeout_wait=1 success = True break - elif type(response_data) == list: + if type(response_data) == list: logger.warning("Wrong type returned, trying again...") logger.info(f"Returned list: {response_data}") - elif type(response_data) == str: + if type(response_data) == str: logger.info( f"Warning! page_data was string: {response_data}") if "" in response_data: @@ -116,7 +118,7 @@ def request_graphql_dict(key_auth, logger, url,query,variables={},timeout_wait=1 # Sometimes raw text can be converted to a dict response_data = json.loads(response_data) logger.info(f"{response_data}") - err = process_graphql_dict_response(logger,result,response_data) + err = process_dict_response(logger, result, response_data) #If we get an error message that's not None if err and err != GithubApiResult.SUCCESS: @@ -124,7 +126,7 @@ def request_graphql_dict(key_auth, logger, url,query,variables={},timeout_wait=1 success = True break - except: + except (json.JSONDecodeError, KeyError, TypeError): pass attempts += 1 @@ -140,13 +142,13 @@ def request_graphql_dict(key_auth, logger, url,query,variables={},timeout_wait=1 #Should keep track of embedded data that is incomplete. class GraphQlPageCollection(collections.abc.Sequence): #Bind is needed for things like query by repo. Contains bind variables for the graphql query - def __init__(self,query,keyAuth,logger,bind={},numPerPage=100,url="https://api.github.com/graphql",repaginateIfIncomplete=[]): + def __init__(self,query,keyAuth,logger,bind={},numPerPage=100,url=None,repaginateIfIncomplete=[]): self.per_page = numPerPage self.query = query self.keyAuth = keyAuth self.logger = logger - self.url = url + self.url = url if url else f"{get_github_api_base_url()}/graphql" self.page_cache = [] @@ -191,7 +193,7 @@ def request_graphql_dict(self,variables={},timeout_wait=10): try: response_data = result.json() - except: + except: # pylint: disable=bare-except response_data = json.loads(json.dumps(result.text)) #self.logger.info(f"api return: {response_data}") @@ -234,7 +236,7 @@ def request_graphql_dict(self,variables={},timeout_wait=10): success = True break - except: + except: # pylint: disable=bare-except pass attempts += 1 @@ -275,7 +277,7 @@ def __getitem__(self, index):# -> dict: #first try cache try: return self.page_cache[index] - except: + except: # pylint: disable=bare-except pass @@ -405,7 +407,7 @@ class GitHubRepo(): def __init__(self, logger, key_auth, owner, repo): self.keyAuth = key_auth - self.url = "https://api.github.com/graphql" + self.url = f"{get_github_api_base_url()}/graphql" self.logger = logger @@ -538,7 +540,7 @@ class PullRequest(): def __init__(self, logger, key_auth, owner, repo, number): self.keyAuth = key_auth - self.url = "https://api.github.com/graphql" + self.url = f"{get_github_api_base_url()}/graphql" self.logger = logger diff --git a/augur/tasks/github/util/github_api_key_handler.py b/augur/tasks/github/util/github_api_key_handler.py index 47933e67dc..d1fac4ee56 100644 --- a/augur/tasks/github/util/github_api_key_handler.py +++ b/augur/tasks/github/util/github_api_key_handler.py @@ -3,13 +3,13 @@ import random from typing import List -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session # pylint: disable=unused-import from augur.tasks.util.redis_list import RedisList from augur.application.db.lib import get_value, get_worker_oauth_keys -from sqlalchemy import func +from augur.tasks.github.util.github_api_url import get_github_api_base_url +from sqlalchemy import func # pylint: disable=unused-import -RATE_LIMIT_URL = "https://api.github.com/rate_limit" class NoValidKeysError(Exception): @@ -98,7 +98,7 @@ def get_api_keys(self) -> List[str]: try: keys = self.get_api_keys_from_database() break - except: + except: # pylint: disable=bare-except time.sleep(5) attempts += 1 @@ -158,7 +158,7 @@ def is_bad_api_key(self, client: httpx.Client, oauth_key: str) -> bool: headers = {'Authorization': f'token {oauth_key}'} - data = client.request(method="GET", url=RATE_LIMIT_URL, headers=headers, timeout=180).json() + data = client.request(method="GET", url=f"{get_github_api_base_url()}/rate_limit", headers=headers, timeout=180).json() try: if data["message"] == "Bad credentials": @@ -173,7 +173,7 @@ def get_key_rate_limit(client, github_key): headers = {'Authorization': f'token {github_key}'} - data = client.request(method="GET", url=RATE_LIMIT_URL, headers=headers, timeout=180).json() + data = client.request(method="GET", url=f"{get_github_api_base_url()}/rate_limit", headers=headers, timeout=180).json() if "message" in data: return None, None diff --git a/augur/tasks/github/util/github_api_url.py b/augur/tasks/github/util/github_api_url.py new file mode 100644 index 0000000000..a7c4cc539b --- /dev/null +++ b/augur/tasks/github/util/github_api_url.py @@ -0,0 +1,28 @@ +""" +Centralized GitHub API base URL management for GitHub Enterprise support. + +This module provides functions to get the GitHub API base URL that works with +both the public GitHub API (api.github.com) and GitHub Enterprise instances. + +The base URL can be configured via the 'github_api_base_url' setting in the +'Keys' section of the Augur config. + +Example Enterprise URL: https://api.mycompany.ghe.com +""" +from augur.application.db.lib import get_value + +# Default GitHub API URL +DEFAULT_GITHUB_API_URL = "https://api.github.com" + + +def get_github_api_base_url() -> str: + """Get the configured GitHub API base URL. + + Returns: + The GitHub API base URL from config, or the default public API URL. + """ + base_url = get_value("Keys", "github_api_base_url") + if base_url: + # Remove trailing slash if present + return base_url.rstrip("/") + return DEFAULT_GITHUB_API_URL diff --git a/augur/tasks/github/util/github_graphql_data_access.py b/augur/tasks/github/util/github_graphql_data_access.py index 96b0c6ab76..92580adb79 100644 --- a/augur/tasks/github/util/github_graphql_data_access.py +++ b/augur/tasks/github/util/github_graphql_data_access.py @@ -2,8 +2,7 @@ import time import httpx from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception, RetryError - -URL = "https://api.github.com/graphql" +from augur.tasks.github.util.github_api_url import get_github_api_base_url class RatelimitException(Exception): @@ -74,7 +73,7 @@ def make_request(self, query, variables, timeout=40): if variables: json_dict['variables'] = variables - response = client.post(url=URL,auth=self.key_manager,json=json_dict, timeout=timeout) + response = client.post(url=f"{get_github_api_base_url()}/graphql",auth=self.key_manager,json=json_dict, timeout=timeout) response.raise_for_status()