feat: add GitHub Enterprise API support (#3277)

Signed-off-by: Suyash Jain <[email protected]>

Author: SuyashJain17

augur/application/config.py

@@ -40,7 +40,8 @@ def redact_setting_value(section_name, setting_name, value):
             },
             "Keys": {
                 "github": "<gh_api_key>",
-                "gitlab": "<gl_api_key>"
+                "gitlab": "<gl_api_key>",
+                "github_api_base_url": "https://api.github.com"
             },
             "Facade": {
                 "check_updates": 1,

augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py

@@ -8,6 +8,7 @@
 from augur.application.db.models import ContributorRepo
 from augur.application.db.lib import bulk_insert_dicts
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
+from augur.tasks.github.util.github_api_url import get_user_events_url
 
 ### This worker scans all the platform users in Augur, and pulls their platform activity 
 ### logs. Those are then used to analyze what repos each is working in (which will include repos not
@@ -92,7 +93,7 @@ def contributor_breadth_model(self) -> None:
         print(f"Processing cntrb {index} of {total}")
         index += 1
 
-        repo_cntrb_url = f"https://api.github.com/users/{cntrb['gh_login']}/events"
+        repo_cntrb_url = get_user_events_url(cntrb['gh_login'])
 
         newest_event_in_db = datetime(1970, 1, 1)
         if cntrb["gh_login"] in cntrb_newest_events_map:

augur/tasks/github/contributors.py

@@ -10,6 +10,7 @@
 from augur.application.db.util import execute_session_query
 from augur.application.db.lib import bulk_insert_dicts, get_session, batch_insert_contributors
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
+from augur.tasks.github.util.github_api_url import get_user_url
 
 
 
@@ -45,7 +46,7 @@ def process_contributors():
 
         del contributor_dict["_sa_instance_state"]
 
-        url = f"https://api.github.com/users/{contributor_dict['cntrb_login']}" 
+        url = get_user_url(contributor_dict['cntrb_login']) 
 
         data = retrieve_dict_data(url, key_auth, logger)
 

augur/tasks/github/detect_move/core.py

@@ -9,6 +9,7 @@
 from augur.application.db.lib import bulk_insert_dicts
 from augur.application.db.models import HistoricalRepoURLs
 from sqlalchemy.exc import IntegrityError
+from augur.tasks.github.util.github_api_url import get_repo_url
 
 
 class RepoMovedException(Exception):
@@ -69,7 +70,7 @@ def extract_owner_and_repo_from_endpoint(key_auth, url, logger):
 def ping_github_for_repo_move(session, key_auth, repo, logger,collection_hook='core'):
 
     owner, name = get_owner_repo(repo.repo_git)
-    url = f"https://api.github.com/repos/{owner}/{name}"
+    url = get_repo_url(owner, name)
 
     attempts = 0
     while attempts < 10:

augur/tasks/github/events.py

@@ -15,6 +15,7 @@
 from augur.tasks.util.worker_util import remove_duplicate_dicts
 from augur.application.db.models import PullRequestEvent, IssueEvent, Contributor, Repo
 from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_issues_by_repo_id, get_pull_requests_by_repo_id, update_issue_closed_cntrbs_by_repo_id, get_session, get_engine, get_core_data_last_collected, batch_insert_contributors
+from augur.tasks.github.util.github_api_url import get_repo_url
 
 
 platform_id = 1
@@ -47,7 +48,7 @@ def collect_events(repo_git: str, full_collection: bool):
 
 def bulk_events_collection_endpoint_contains_all_data(key_auth, logger, owner, repo):
 
-    url = f"https://api.github.com/repos/{owner}/{repo}/issues/events?per_page=100"
+    url = get_repo_url(owner, repo, "/issues/events?per_page=100")
 
     github_data_access = GithubDataAccess(key_auth, logger)
 
@@ -131,7 +132,7 @@ def _collect_events(self, repo_git: str, key_auth, since):
 
         owner, repo = get_owner_repo(repo_git)
 
-        url = f"https://api.github.com/repos/{owner}/{repo}/issues/events"
+        url = get_repo_url(owner, repo, "/issues/events")
 
         github_data_access = GithubDataAccess(key_auth, self._logger)
 
@@ -309,7 +310,7 @@ def _collect_and_process_issue_events(self, owner, repo, repo_id, key_auth, sinc
 
             issue_number = issue["issue_number"]
 
-            event_url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/events"
+            event_url = get_repo_url(owner, repo, f"/issues/{issue_number}/events")
 
             try:
 
@@ -370,7 +371,7 @@ def _collect_and_process_pr_events(self, owner, repo, repo_id, key_auth, since):
 
             pr_number = pr["gh_pr_number"]
 
-            event_url = f"https://api.github.com/repos/{owner}/{repo}/issues/{pr_number}/events"
+            event_url = get_repo_url(owner, repo, f"/issues/{pr_number}/events")
 
             try:
 

augur/tasks/github/facade_github/contributor_interfaceable/contributor_interface.py

@@ -8,6 +8,7 @@
 # Debugger
 from augur.tasks.github.util.github_paginator import GithubApiResult
 from augur.application.db.lib import get_repo_by_repo_id, bulk_insert_dicts, execute_sql, get_contributors_by_github_user_id
+from augur.tasks.github.util.github_api_url import get_search_users_url, get_repo_url
 
 
 ##TODO: maybe have a TaskSession class that holds information about the database, logger, config, etc.
@@ -106,8 +107,7 @@ def request_dict_from_endpoint(logger, session, url, timeout_wait=10):
 def create_endpoint_from_email(email):
     # Note: I added "+type:user" to avoid having user owned organizations be returned
     # Also stopped splitting per note above.
-    url = 'https://api.github.com/search/users?q={}+in:email+type:user'.format(
-        email)
+    url = get_search_users_url(f"{email}+in:email+type:user")
 
 
     return url
@@ -131,7 +131,7 @@ def create_endpoint_from_commit_sha(logger, commit_sha, repo_id):
     split_git = result.repo_git.split('/')
     repo_name_and_org = split_git[-2] + "/" + result.repo_name
 
-    url = "https://api.github.com/repos/" + repo_name_and_org + "/commits/" + commit_sha
+    url = get_repo_url(split_git[-2], result.repo_name, f"/commits/{commit_sha}")
 
     logger.debug(f"Commit Hash URL: {url}")
 
@@ -151,8 +151,7 @@ def create_endpoint_from_name(contributor):
         # Pythonic way to get the end of a list so that we truely get the last name.
         'lname': contributor[name_field].split()[-1]
     }
-    url = 'https://api.github.com/search/users?q=fullname:{}+{}'.format(
-        cmt_cntrb['fname'], cmt_cntrb['lname'])
+    url = get_search_users_url(f"fullname:{cmt_cntrb['fname']}+{cmt_cntrb['lname']}")
 
     return url
 

augur/tasks/github/facade_github/core.py

@@ -5,6 +5,7 @@
 from augur.tasks.util.AugurUUID import GithubUUID
 from augur.application.db.lib import bulk_insert_dicts, batch_insert_contributors
 from augur.tasks.github.util.github_data_access import GithubDataAccess
+from augur.tasks.github.util.github_api_url import get_repo_url, get_user_url
 
 
 
@@ -26,10 +27,7 @@ def query_github_contributors(logger, key_auth, github_url):
         raise e
 
     # Set the base of the url and place to hold contributors to insert
-    contributors_url = (
-        f"https://api.github.com/repos/{owner}/{name}/" +
-        "contributors?state=all"
-    )
+    contributors_url = get_repo_url(owner, name, "/contributors?state=all")
 
     # Get contributors that we already have stored
     #   Set our duplicate and update column map keys (something other than PK) to
@@ -53,7 +51,7 @@ def query_github_contributors(logger, key_auth, github_url):
             # Need to hit this single contributor endpoint to get extra data including...
             #   `created at`
             #   i think that's it
-            cntrb_url = ("https://api.github.com/users/" + repo_contributor['login'])
+            cntrb_url = get_user_url(repo_contributor['login'])
 
 
             logger.info("Hitting endpoint: " + cntrb_url + " ...\n")

augur/tasks/github/facade_github/tasks.py

@@ -5,6 +5,7 @@
 from augur.tasks.init.celery_app import AugurFacadeRepoCollectionTask
 from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
+from augur.tasks.github.util.github_api_url import get_user_url
 from augur.tasks.github.facade_github.core import *
 from augur.application.db.lib import execute_sql, get_contributor_aliases_by_email, get_unresolved_commit_emails_by_name, get_contributors_by_full_name, get_repo_by_repo_git, batch_insert_contributors
 from augur.application.db.lib import get_session, execute_session_query
@@ -67,7 +68,7 @@ def process_commit_metadata(logger, auth, contributorQueue, repo_id, platform_id
             logger.error("Failed to get login from supplemental data!")
             continue
 
-        url = ("https://api.github.com/users/" + login)
+        url = get_user_url(login)
 
         try:
             user_data = github_data_access.get_resource(url)

augur/tasks/github/issues.py

@@ -15,6 +15,7 @@
 from augur.application.db.models import Issue, IssueLabel, IssueAssignee
 from augur.application.config import get_development_flag
 from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_core_data_last_collected, batch_insert_contributors
+from augur.tasks.github.util.github_api_url import get_repo_url
 
 
 development = get_development_flag()
@@ -101,7 +102,7 @@ def retrieve_all_issue_data(repo_git: str, logger: logging.Logger, key_auth: Git
 
     logger.info(f"Collecting issues for {owner}/{repo}")
 
-    url = f"https://api.github.com/repos/{owner}/{repo}/issues?state=all"
+    url = get_repo_url(owner, repo, "/issues?state=all")
 
     if since:
         url += f"&since={since.isoformat()}"

augur/tasks/github/messages.py

@@ -11,6 +11,7 @@
 from augur.application.db.models import PullRequest, Message, Issue, PullRequestMessageRef, IssueMessageRef, Contributor, Repo, CollectionStatus
 from augur.application.db import get_engine, get_session
 from augur.application.db.lib import get_core_data_last_collected
+from augur.tasks.github.util.github_api_url import get_repo_url
 from sqlalchemy.sql import text
 
 platform_id = 1
@@ -63,7 +64,7 @@ def fast_retrieve_all_pr_and_issue_messages(repo_git: str, logger, key_auth, tas
     owner, repo = get_owner_repo(repo_git)
 
     # url to get issue and pull request comments
-    url = f"https://api.github.com/repos/{owner}/{repo}/issues/comments"
+    url = get_repo_url(owner, repo, "/issues/comments")
 
     if since:
         url += f"?since={since.isoformat()}"