PD-90 PD-92 Python to 3.13 and replace Gino with sqlalchemy

.github/workflows/docs.yaml

@@ -13,10 +13,10 @@ jobs:
     - uses: actions/checkout@v2
       with:
         ref: ${{ github.head_ref }}
-    - name: Set up Python 3.9
+    - name: Set up Python 3.13
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: 3.13
     - uses: actions/cache@preview
       with:
         path: ~/.cache/pypoetry/virtualenvs

.github/workflows/tests.yaml

@@ -18,10 +18,10 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.9
+    - name: Set up Python 3.13
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: 3.13
     - uses: actions/cache@preview
       with:
         path: ~/.cache/pypoetry/virtualenvs

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: git@github.com:Yelp/detect-secrets
-    rev: v1.2.0
+    rev: v1.5.0
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.2.0",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -118,24 +118,24 @@
         "line_number": 26
       }
     ],
-    "tests/test-requestor-config.yaml": [
+    "tests/migrations/test_migration_42cbae986650.py": [
       {
-        "type": "Secret Keyword",
-        "filename": "tests/test-requestor-config.yaml",
-        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "type": "Hex High Entropy String",
+        "filename": "tests/migrations/test_migration_42cbae986650.py",
+        "hashed_secret": "4fad492cfee1c1b0f14750167ec26646f69c0c6f",
         "is_verified": false,
-        "line_number": 18
+        "line_number": 57
       }
     ],
-    "tests/test_migrations.py": [
+    "tests/test-requestor-config.yaml": [
       {
-        "type": "Hex High Entropy String",
-        "filename": "tests/test_migrations.py",
-        "hashed_secret": "4fad492cfee1c1b0f14750167ec26646f69c0c6f",
+        "type": "Secret Keyword",
+        "filename": "tests/test-requestor-config.yaml",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 43
+        "line_number": 17
       }
     ]
   },
-  "generated_at": "2024-09-04T17:19:26Z"
+  "generated_at": "2025-12-19T22:34:07Z"
 }

Dockerfile

@@ -1,6 +1,6 @@
-ARG AZLINUX_BASE_VERSION=master
+ARG AZLINUX_BASE_VERSION=3.13-pythonnginx
 
-FROM quay.io/cdis/python-nginx-al:feat_python-nginx AS base
+FROM quay.io/cdis/amazonlinux-base:${AZLINUX_BASE_VERSION} AS base
 
 ENV appname=requestor
 
@@ -30,6 +30,7 @@ RUN poetry install --no-interaction --without dev
 FROM base
 
 COPY --from=builder /${appname} /${appname}
+COPY --from=builder /venv /venv
 
 # Switch to non-root user 'gen3' for the serving process
 USER gen3

docs/local_installation.md

@@ -5,7 +5,7 @@
 Install required software:
 
 *   [PostgreSQL](PostgreSQL) 9.6 or above
-*   [Python](https://www.python.org/downloads/) 3.9.x
+*   [Python](https://www.python.org/downloads/) 3.13.x
 *   [Poetry](https://poetry.eustace.io/docs/#installation)
 
 Then use `poetry install` to install the dependencies. Before that,

docs/openapi.yaml

@@ -92,7 +92,7 @@ components:
       type: http
 info:
   title: Requestor
-  version: 1.7.1
+  version: 2.0.0
 openapi: 3.1.0
 paths:
   /_status:

migrations/env.py

@@ -1,19 +1,25 @@
-from alembic import context
 from logging.config import fileConfig
-from sqlalchemy import engine_from_config, pool
 
-from requestor.config import config
-from requestor.app import db, load_modules
+from alembic import context
+import asyncio
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from requestor.app import load_modules
+from requestor.config import config as requestor_config
+from requestor.db import Base
 
 
-conf = context.config
-fileConfig(conf.config_file_name)
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+config.set_main_option("sqlalchemy.url", requestor_config["DB_URL"])
 load_modules()
-target_metadata = db
-conf.set_main_option("sqlalchemy.url", str(config["DB_URL"]))
+target_metadata = Base.metadata
 
 
-def run_migrations_offline():
+def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
     This configures the context with just a URL
@@ -25,7 +31,7 @@ def run_migrations_offline():
     script output.
 
     """
-    url = conf.get_main_option("sqlalchemy.url")
+    url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
         target_metadata=target_metadata,
@@ -37,24 +43,36 @@ def run_migrations_offline():
         context.run_migrations()
 
 
-def run_migrations_online():
-    """Run migrations in 'online' mode.
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
 
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
 
+async def run_async_migrations() -> None:
+    """In this scenario we need to create an Engine
+    and associate a connection with the context.
     """
-    connectable = engine_from_config(
-        conf.get_section(conf.config_ini_section),
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )
 
-    with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
 
-        with context.begin_transaction():
-            context.run_migrations()
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    connectable = context.config.attributes.get("connection", None)
+    if connectable is None:
+        asyncio.run(run_async_migrations())
+    else:
+        do_run_migrations(connectable)  # to support running migrations in unit tests
 
 
 if context.is_offline_mode():

migrations/versions/42cbae986650_policy_id_and_revoke.py

@@ -61,7 +61,7 @@ def upgrade():
     offset = 0
     limit = 500
     query = f"SELECT resource_path FROM requests ORDER by resource_path LIMIT {limit} OFFSET {offset}"
-    results = connection.execute(query).fetchall()
+    results = connection.execute(sa.text(query)).fetchall()
 
     # add the `policy_id` corresponding to each row's `resource_path`
     # and default `revoke` to False
@@ -78,13 +78,15 @@ def upgrade():
                 )
                 existing_policies["policies"].append(created_policy_id)
             connection.execute(
-                f"UPDATE requests SET policy_id='{escape(policy_id)}', revoke=False WHERE resource_path='{escape(resource_path)}'"
+                sa.text(
+                    f"UPDATE requests SET policy_id='{escape(policy_id)}', revoke=False WHERE resource_path='{escape(resource_path)}'"
+                )
             )
 
         # Grab another batch of rows
         offset += limit
         query = f"SELECT resource_path FROM requests ORDER by resource_path LIMIT {limit} OFFSET {offset}"
-        results = connection.execute(query).fetchall()
+        results = connection.execute(sa.text(query)).fetchall()
 
     # now that there are no null values, make the columns non-nullable
     op.alter_column("requests", "policy_id", nullable=False)
@@ -107,7 +109,7 @@ def downgrade():
     offset = 0
     limit = 500
     query = f"SELECT policy_id FROM requests ORDER by policy_id LIMIT {limit} OFFSET {offset}"
-    results = connection.execute(query).fetchall()
+    results = connection.execute(sa.text(query)).fetchall()
 
     while results:
         for policy_id in set(r[0] for r in results):
@@ -124,13 +126,15 @@ def downgrade():
             # use the first item in the policy’s list of resources, because this
             # schema only allows 1 resource_path
             connection.execute(
-                f"UPDATE requests SET resource_path='{escape(resource_paths[0])}' WHERE policy_id='{escape(policy_id)}'"
+                sa.text(
+                    f"UPDATE requests SET resource_path='{escape(resource_paths[0])}' WHERE policy_id='{escape(policy_id)}'"
+                )
             )
 
         # get another batch of rows
         offset += limit
         query = f"SELECT policy_id FROM requests ORDER by policy_id LIMIT {limit} OFFSET {offset}"
-        results = connection.execute(query).fetchall()
+        results = connection.execute(sa.text(query)).fetchall()
 
     # now that there are no null values, make the column non-nullable
     op.alter_column("requests", "resource_path", nullable=False)