Merge pull request #226 from Joyakis/refactor/shared-get-session

 Centralize get_session()

Author: TimidRobot

scripts/1-fetch/arxiv_fetch.py

@@ -22,8 +22,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -335,19 +333,6 @@ def initialize_all_data_files(args):
     initialize_data_file(FILE_ARXIV_AUTHOR_BUCKET, HEADER_AUTHOR_BUCKET)
 
 
-def get_requests_session():
-    """Create request session with retry logic"""
-    retry_strategy = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
-    return session
-
-
 def normalize_license_text(raw_text):
     """
     Convert raw license text to standardized CC license identifiers.
@@ -533,7 +518,7 @@ def query_arxiv(args):
     """
 
     LOGGER.info("Beginning to fetch results from ArXiv API")
-    session = get_requests_session()
+    session = shared.get_session()
 
     results_per_iteration = 50
 

scripts/1-fetch/europeana_fetch.py

@@ -23,7 +23,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter, Retry
 
 # Add parent directory for shared imports
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -103,19 +102,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    """Create a requests session with retry."""
-    max_retries = Retry(
-        total=5, backoff_factor=10, status_forcelist=shared.STATUS_FORCELIST
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
-
-
 def simplify_legal_tool(legal_tool):
     """Simplify license URLs into human-readable labels
 
@@ -433,7 +419,7 @@ def main():
             "EUROPEANA_API_KEY not found in environment variables", 1
         )
 
-    session = get_requests_session()
+    session = shared.get_session(accept_header="application/json")
 
     # Fetch facet lists once, including counts
     providers_full = get_facet_list(session, "DATA_PROVIDER")

scripts/1-fetch/github_fetch.py

@@ -17,8 +17,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -80,25 +78,6 @@ def check_for_completion():
         pass  # File may not be found without --enable-save, etc.
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    headers = {
-        "accept": "application/vnd.github+json",
-        "User-Agent": shared.USER_AGENT,
-    }
-    if GH_TOKEN:
-        headers["authorization"] = f"Bearer {GH_TOKEN}"
-    session.headers.update(headers)
-
-    return session
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args
@@ -162,7 +141,12 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     check_for_completion()
-    session = get_requests_session()
+    session = shared.get_session(
+        accept_header="application/vnd.github+json",
+    )
+    if GH_TOKEN:
+        session.headers.update({"authorization": f"Bearer {GH_TOKEN}"})
+
     tool_data = query_github(args, session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(

scripts/1-fetch/openverse_fetch.py

@@ -25,8 +25,6 @@
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -83,20 +81,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update(
-        {"accept": "application/json", "User-Agent": shared.USER_AGENT}
-    )
-    return session
-
-
 def get_all_sources_and_licenses(session, media_type):
     """
     Fetch all available sources for a given media_type.
@@ -225,8 +209,8 @@ def write_data(args, data):
 
 def main():
     args = parse_arguments()
-    session = get_requests_session()
     LOGGER.info("Starting Openverse Fetch Script...")
+    session = shared.get_session(accept_header="application/json")
     records = query_openverse(session)
     write_data(args, records)
     LOGGER.info(f"Fetched {len(records)} unique Openverse records.")

scripts/1-fetch/wikipedia_fetch.py

@@ -13,12 +13,9 @@
 from operator import itemgetter
 
 # Third-party
-import requests
 from pygments import highlight
 from pygments.formatters import TerminalFormatter
 from pygments.lexers import PythonTracebackLexer
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -66,18 +63,6 @@ def parse_arguments():
     return args
 
 
-def get_requests_session():
-    max_retries = Retry(
-        total=5,
-        backoff_factor=10,
-        status_forcelist=shared.STATUS_FORCELIST,
-    )
-    session = requests.Session()
-    session.mount("https://", HTTPAdapter(max_retries=max_retries))
-    session.headers.update({"User-Agent": shared.USER_AGENT})
-    return session
-
-
 def write_data(args, tool_data):
     if not args.enable_save:
         return args
@@ -173,7 +158,8 @@ def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
-    tool_data = query_wikipedia_languages(get_requests_session())
+    session = shared.get_session()
+    tool_data = query_wikipedia_languages(session)
     args = write_data(args, tool_data)
     args = shared.git_add_and_commit(
         args,

scripts/shared.py

@@ -7,6 +7,8 @@
 # Third-party
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
+from requests import Session
+from requests.adapters import HTTPAdapter, Retry
 
 # Constants
 STATUS_FORCELIST = [
@@ -31,6 +33,25 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
+def get_session(accept_header=None):
+    """Create a reusable HTTP session with retry logic."""
+    session = Session()
+
+    retry_strategy = Retry(
+        total=5,
+        backoff_factor=10,
+        status_forcelist=STATUS_FORCELIST,
+    )
+    session.mount("https://", HTTPAdapter(max_retries=retry_strategy))
+
+    headers = {"User-Agent": USER_AGENT}
+    if accept_header:
+        headers["accept"] = accept_header
+    session.headers.update(headers)
+
+    return session
+
+
 def git_fetch_and_merge(args, repo_path, branch=None):
     if not args.enable_git:
         return