Merge pull request #87 from cal-itp/various-fixes

Update dependencies and add shared secret loading code

.github/workflows/ci.yml

@@ -67,36 +67,3 @@ jobs:
           repository_username: __token__
           repository_password: ${{ secrets.PYPI_API_TOKEN }}
           build_format: "sdist"
-
-# This is disabled for now; I don't hate the image definition living here, but the CI/CD is overly complicated I think
-#  build_push:
-#    name: Package docker image
-#    runs-on: ubuntu-18.04
-#    needs:
-#      - lint
-#      - test
-#      - release
-#    if: "success('lint') && success('test') && !failure('release')"
-#    steps:
-#      - name: Login to GitHub Container Registry
-#        uses: docker/login-action@v1
-#        with:
-#          registry: ghcr.io
-#          username: ${{ github.actor }}
-#          password: ${{ secrets.GITHUB_TOKEN }}
-#
-#      # build release image ----
-#      - name: "Release: Build and push"
-#        uses: docker/build-push-action@v2
-#        if: ${{ github.event_name == 'release' &&  startsWith(github.event.release.tag_name, 'hub') }}
-#        with:
-#          push: true
-#          tags: ghcr.io/${{github.repository}}:${{ github.event.release.tag_name }}
-#
-#      # build any image pushed on a branch starting with development ----
-#      - name: "Development: Build and push"
-#        uses: docker/build-push-action@v2
-#        if: ${{ github.event_name != 'release' && startsWith(github.ref_name, 'development') }}
-#        with:
-#          push: true
-#          tags: ghcr.io/${{github.repository}}:${{github.ref_name}}

calitp/auth.py

@@ -0,0 +1,34 @@
+import os
+from typing import Sequence
+
+import google_crc32c
+from google.cloud import secretmanager
+
+AUTH_KEYS_ENV_VAR = "CALITP_AUTH_KEYS"
+DEFAULT_AUTH_KEYS = tuple(os.environ[AUTH_KEYS_ENV_VAR].split(",")) if AUTH_KEYS_ENV_VAR in os.environ else tuple()
+
+
+def load_secrets(keys: Sequence[str] = DEFAULT_AUTH_KEYS, secret_client=secretmanager.SecretManagerServiceClient()):
+    if not keys:
+        print("no secrets to load")
+        return
+
+    for key in keys:
+        if key in os.environ:
+            print(f"found {key} already in os.environ, skipping")
+        else:
+            print(f"fetching secret {key}")
+            name = f"projects/cal-itp-data-infra/secrets/{key}/versions/latest"
+            response = secret_client.access_secret_version(request={"name": name})
+
+            crc32c = google_crc32c.Checksum()
+            crc32c.update(response.payload.data)
+            if response.payload.data_crc32c != int(crc32c.hexdigest(), 16):
+                raise ValueError(f"Data corruption detected for secret {name}.")
+
+            os.environ[key] = response.payload.data.decode("UTF-8").strip()
+
+
+if __name__ == "__main__":
+    print("loading secrets...")
+    load_secrets()

calitp/storage.py

@@ -10,7 +10,17 @@
 from abc import ABC
 from datetime import datetime
 from enum import Enum
-from typing import ClassVar, Dict, List, Optional, Tuple, Type, Union, get_type_hints
+from typing import (
+    ClassVar,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    get_type_hints,
+)
 
 import backoff
 import gcsfs
@@ -556,7 +566,7 @@ def convert_feed_type(cls, v):
 
         return v
 
-    def build_request(self, auth_dict: dict) -> Request:
+    def build_request(self, auth_dict: Mapping[str, str]) -> Request:
         params = {k: auth_dict[v] for k, v in self.auth_query_params.items()}
         headers = {k: auth_dict[v] for k, v in self.auth_headers.items()}
 
@@ -668,7 +678,7 @@ def timestamped_filename(self):
 
 def download_feed(
     config: GTFSDownloadConfig,
-    auth_dict: Dict,
+    auth_dict: Mapping[str, str],
     ts: pendulum.DateTime,
     default_filename="feed",
 ) -> Tuple[GTFSFeedExtract, bytes]:

calitp/tables.py

@@ -111,10 +111,10 @@ def _repr_html_(self):
             """
 
 
-tbl = AutoTable(
+tbls = AutoTable(
     get_engine(),
     lambda s: s,  # s.replace(".", "_"),
     lambda s: "zzz_test_" not in s if not is_development() else True,
 )
 
-tbl._init()
+tbls._init()

docs/index.md

@@ -1,8 +1,8 @@
 # calitp
 
 ```python
-from calitp.tables import tbl
-from calitp.import query_sql
+from calitp.tables import tbls
+from calitp. import query_sql
 
 
 ```
@@ -16,7 +16,7 @@ from calitp.import query_sql
 ### siuba
 
 ```python
-tbl.gtfs_schedule.agency()
+tbls.gtfs_schedule.agency()
 ```
 
 ### sql