diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 833ac34..758bd9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,36 +67,3 @@ jobs: repository_username: __token__ repository_password: ${{ secrets.PYPI_API_TOKEN }} build_format: "sdist" - -# This is disabled for now; I don't hate the image definition living here, but the CI/CD is overly complicated I think -# build_push: -# name: Package docker image -# runs-on: ubuntu-18.04 -# needs: -# - lint -# - test -# - release -# if: "success('lint') && success('test') && !failure('release')" -# steps: -# - name: Login to GitHub Container Registry -# uses: docker/login-action@v1 -# with: -# registry: ghcr.io -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} -# -# # build release image ---- -# - name: "Release: Build and push" -# uses: docker/build-push-action@v2 -# if: ${{ github.event_name == 'release' && startsWith(github.event.release.tag_name, 'hub') }} -# with: -# push: true -# tags: ghcr.io/${{github.repository}}:${{ github.event.release.tag_name }} -# -# # build any image pushed on a branch starting with development ---- -# - name: "Development: Build and push" -# uses: docker/build-push-action@v2 -# if: ${{ github.event_name != 'release' && startsWith(github.ref_name, 'development') }} -# with: -# push: true -# tags: ghcr.io/${{github.repository}}:${{github.ref_name}} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0cc8e4d..0000000 --- a/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM jupyter/datascience-notebook - -LABEL org.opencontainers.image.source https://github.com/cal-itp/calitp-py - -USER root -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - -# GitHub CLI https://github.com/cli/cli/blob/trunk/docs/install_linux.md -RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null -RUN apt update \ - && apt install -y keychain nodejs git-lfs gh libspatialindex-dev -USER $NB_UID -RUN npm install -g --unsafe-perm=true --allow-root netlify-cli - -# gcloud CLI https://cloud.google.com/sdk/docs/install#deb -RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-377.0.0-linux-x86_64.tar.gz \ - && tar -zxvf google-cloud-sdk-377.0.0-linux-x86_64.tar.gz \ - && ./google-cloud-sdk/install.sh - -ADD _jupyterhub/requirements.txt /app/requirements.txt - -RUN pip install -r /app/requirements.txt - -RUN mkdir /opt/conda/share/jupyter/lab/settings/ -COPY _jupyterhub/overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json - -COPY _jupyterhub/custom.sh /tmp/custom.sh diff --git a/_jupyterhub/custom.sh b/_jupyterhub/custom.sh deleted file mode 100644 index 9033190..0000000 --- a/_jupyterhub/custom.sh +++ /dev/null @@ -1,6 +0,0 @@ -parse_git_branch() { - git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' -} -export PS1="\u@\h \[\033[32m\]\w\[\033[33m\]\$(parse_git_branch)\[\033[00m\] $ " -chmod 600 ~/.ssh/id_ed25519 -eval `keychain --eval --agents ssh id_ed25519` diff --git a/_jupyterhub/examples/example_ipyleaflet.py b/_jupyterhub/examples/example_ipyleaflet.py deleted file mode 100644 index 6075262..0000000 --- a/_jupyterhub/examples/example_ipyleaflet.py +++ /dev/null @@ -1,10 +0,0 @@ -# flake8: noqa - -# + -from ipyleaflet import Map, Marker - -center = (52.204793, 360.121558) -m = Map(center=center, zoom=15) -marker = Marker(location=center, draggable=True) -m.add_layer(marker) -display(m) diff --git a/_jupyterhub/overrides.json b/_jupyterhub/overrides.json deleted file mode 100644 index b250ab6..0000000 --- a/_jupyterhub/overrides.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "@ryantam626/jupyterlab_code_formatter:settings": { - "preferences": { - "default_formatter": { - "python": [ - "isort", - "black" - ] - } - }, - "formatOnSave": false - }, - "@jupyterlab/shortcuts-extension:shortcuts": { - "shortcuts": [ - { - "command": "jupyterlab_code_formatter:black", - "keys": [ - "Ctrl Shift K" - ], - "selector": ".jp-Notebook.jp-mod-editMode" - } - ] - } -} diff --git a/_jupyterhub/requirements.txt b/_jupyterhub/requirements.txt deleted file mode 100644 index e8269ad..0000000 --- a/_jupyterhub/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -black==22.3.0 -calitp==0.0.16 -intake==0.6.4 -intake-dcat==0.4.0 -intake-geopandas==0.3.0 -intake-parquet==0.2.3 -ipyleaflet==0.15.0 -jupytext==1.13.5 -jupyterlab-code-formatter==1.4.10 -siuba==1.0.0a2 -voila==0.3.6 -plotnine==0.8.0 -plotly==5.5.0 -folium==0.12.1.post1 -branca==0.4.2 -altair_saver==0.5.0 -vega==3.5.0 -pygeos==0.12.0 -rtree==0.9.7 -openpyxl==3.0.9 -python-dotenv==0.19.2 -isort==5.10.1 -seaborn==0.11.2 -csvkit==1.0.7 -pandas-profiling==3.1.0 -pre-commit==2.18.1 -jupyter-resource-usage==0.6.1 diff --git a/calitp/auth.py b/calitp/auth.py new file mode 100644 index 0000000..d087269 --- /dev/null +++ b/calitp/auth.py @@ -0,0 +1,34 @@ +import os +from typing import Sequence + +import google_crc32c +from google.cloud import secretmanager + +AUTH_KEYS_ENV_VAR = "CALITP_AUTH_KEYS" +DEFAULT_AUTH_KEYS = tuple(os.environ[AUTH_KEYS_ENV_VAR].split(",")) if AUTH_KEYS_ENV_VAR in os.environ else tuple() + + +def load_secrets(keys: Sequence[str] = DEFAULT_AUTH_KEYS, secret_client=secretmanager.SecretManagerServiceClient()): + if not keys: + print("no secrets to load") + return + + for key in keys: + if key in os.environ: + print(f"found {key} already in os.environ, skipping") + else: + print(f"fetching secret {key}") + name = f"projects/cal-itp-data-infra/secrets/{key}/versions/latest" + response = secret_client.access_secret_version(request={"name": name}) + + crc32c = google_crc32c.Checksum() + crc32c.update(response.payload.data) + if response.payload.data_crc32c != int(crc32c.hexdigest(), 16): + raise ValueError(f"Data corruption detected for secret {name}.") + + os.environ[key] = response.payload.data.decode("UTF-8").strip() + + +if __name__ == "__main__": + print("loading secrets...") + load_secrets() diff --git a/calitp/storage.py b/calitp/storage.py index 15db413..76ab718 100644 --- a/calitp/storage.py +++ b/calitp/storage.py @@ -10,7 +10,17 @@ from abc import ABC from datetime import datetime from enum import Enum -from typing import ClassVar, Dict, List, Optional, Tuple, Type, Union, get_type_hints +from typing import ( + ClassVar, + Dict, + List, + Mapping, + Optional, + Tuple, + Type, + Union, + get_type_hints, +) import backoff import gcsfs @@ -556,7 +566,7 @@ def convert_feed_type(cls, v): return v - def build_request(self, auth_dict: dict) -> Request: + def build_request(self, auth_dict: Mapping[str, str]) -> Request: params = {k: auth_dict[v] for k, v in self.auth_query_params.items()} headers = {k: auth_dict[v] for k, v in self.auth_headers.items()} @@ -668,7 +678,7 @@ def timestamped_filename(self): def download_feed( config: GTFSDownloadConfig, - auth_dict: Dict, + auth_dict: Mapping[str, str], ts: pendulum.DateTime, default_filename="feed", ) -> Tuple[GTFSFeedExtract, bytes]: diff --git a/calitp/tables.py b/calitp/tables.py index 0638773..81a1d96 100644 --- a/calitp/tables.py +++ b/calitp/tables.py @@ -111,10 +111,10 @@ def _repr_html_(self): """ -tbl = AutoTable( +tbls = AutoTable( get_engine(), lambda s: s, # s.replace(".", "_"), lambda s: "zzz_test_" not in s if not is_development() else True, ) -tbl._init() +tbls._init() diff --git a/docs/index.md b/docs/index.md index 938a9da..2b04003 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # calitp ```python -from calitp.tables import tbl -from calitp.import query_sql +from calitp.tables import tbls +from calitp. import query_sql ``` @@ -16,7 +16,7 @@ from calitp.import query_sql ### siuba ```python -tbl.gtfs_schedule.agency() +tbls.gtfs_schedule.agency() ``` ### sql diff --git a/poetry.lock b/poetry.lock index 0912a74..acd1531 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "aiohttp" -version = "3.8.1" +version = "3.8.3" description = "Async http client/server framework (asyncio)" category = "main" optional = false @@ -37,14 +37,6 @@ category = "main" optional = false python-versions = ">=3.6" -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "attrs" version = "22.1.0" @@ -69,7 +61,7 @@ python-versions = ">=3.7,<4.0" [[package]] name = "black" -version = "22.6.0" +version = "22.8.0" description = "The uncompromising code formatter." category = "dev" optional = false @@ -91,15 +83,15 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "cachetools" -version = "4.2.4" +version = "5.2.0" description = "Extensible memoizing collections and decorators" category = "main" optional = false -python-versions = "~=3.5" +python-versions = "~=3.7" [[package]] name = "certifi" -version = "2022.6.15" +version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -107,7 +99,7 @@ python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.1.0" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false @@ -135,6 +127,20 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "db-dtypes" +version = "1.0.4" +description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.16.6,<2.0dev" +packaging = ">=17.0" +pandas = ">=0.24.2,<2.0dev" +pyarrow = ">=3.0.0,<10.0dev" + [[package]] name = "decorator" version = "5.1.1" @@ -145,7 +151,7 @@ python-versions = ">=3.5" [[package]] name = "exceptiongroup" -version = "1.0.0rc8" +version = "1.0.0rc9" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -164,7 +170,7 @@ python-versions = ">=3.7" [[package]] name = "fsspec" -version = "2022.5.0" +version = "2022.8.2" description = "File-system specification" category = "main" optional = false @@ -184,7 +190,7 @@ github = ["requests"] gs = ["gcsfs"] gui = ["panel"] hdfs = ["pyarrow (>=1)"] -http = ["requests", "aiohttp"] +http = ["requests", "aiohttp (!=4.0.0a0,!=4.0.0a1)"] libarchive = ["libarchive-c"] oci = ["ocifs"] s3 = ["s3fs"] @@ -203,16 +209,16 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "gcsfs" -version = "2022.5.0" +version = "2022.8.2" description = "Convenient Filesystem interface over GCS" category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -aiohttp = "<4" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" decorator = ">4.1.2" -fsspec = "2022.5.0" +fsspec = "2022.8.2" google-auth = ">=1.2" google-auth-oauthlib = "*" google-cloud-storage = "*" @@ -224,49 +230,48 @@ gcsfuse = ["fusepy"] [[package]] name = "google-api-core" -version = "1.32.0" +version = "1.33.1" description = "Google API client core library" category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +python-versions = ">=3.7" [package.dependencies] -google-auth = ">=1.25.0,<2.0dev" -googleapis-common-protos = ">=1.6.0,<2.0dev" -grpcio = {version = ">=1.29.0,<2.0dev", optional = true, markers = "extra == \"grpc\""} -packaging = ">=14.3" -protobuf = {version = ">=3.12.0,<4.0.0dev", markers = "python_version > \"3\""} -pytz = "*" +google-auth = ">=1.25.0,<3.0dev" +googleapis-common-protos = ">=1.56.2,<2.0dev" +grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""} +grpcio-status = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=3.20.1,<4.0.0dev" requests = ">=2.18.0,<3.0.0dev" -six = ">=1.13.0" [package.extras] -grpc = ["grpcio (>=1.29.0,<2.0dev)"] -grpcgcp = ["grpcio-gcp (>=0.2.2)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2)"] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] [[package]] name = "google-auth" -version = "1.35.0" +version = "2.12.0" description = "Google Authentication Library" category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" [package.dependencies] -cachetools = ">=2.0.0,<5.0" +cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" rsa = {version = ">=3.1.4,<5", markers = "python_version >= \"3.6\""} six = ">=1.9.0" [package.extras] aiohttp = ["requests (>=2.20.0,<3.0.0dev)", "aiohttp (>=3.6.2,<4.0.0dev)"] +enterprise_cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] pyopenssl = ["pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] [[package]] name = "google-auth-oauthlib" -version = "0.5.2" +version = "0.5.3" description = "Google Authentication Library" category = "main" optional = false @@ -289,15 +294,12 @@ python-versions = ">=3.6, <3.11" [package.dependencies] google-api-core = {version = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} -google-cloud-bigquery-storage = {version = ">=2.0.0,<3.0.0dev", optional = true, markers = "extra == \"bqstorage\""} google-cloud-core = ">=1.4.1,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" grpcio = ">=1.38.1,<2.0dev" packaging = ">=14.3,<22.0dev" -pandas = {version = ">=0.24.2", optional = true, markers = "extra == \"pandas\""} proto-plus = ">=1.15.0,<2.0.0dev" protobuf = ">=3.12.0,<4.0.0dev" -pyarrow = {version = ">=3.0.0,<8.0dev", optional = true, markers = "extra == \"bqstorage\""} python-dateutil = ">=2.7.2,<3.0dev" requests = ">=2.18.0,<3.0.0dev" @@ -345,6 +347,23 @@ google-auth = ">=1.25.0,<3.0dev" [package.extras] grpc = ["grpcio (>=1.38.0,<2.0dev)"] +[[package]] +name = "google-cloud-secret-manager" +version = "2.12.4" +description = "Secret Manager API API client library" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +google-api-core = {version = ">=1.32.0,<2.0.0 || >=2.8.0,<3.0.0dev", extras = ["grpc"]} +grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" +proto-plus = ">=1.22.0,<2.0.0dev" +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +libcst = ["libcst (>=0.2.5)"] + [[package]] name = "google-cloud-storage" version = "2.5.0" @@ -365,11 +384,11 @@ protobuf = ["protobuf (<5.0.0dev)"] [[package]] name = "google-crc32c" -version = "1.3.0" +version = "1.5.0" description = "A python wrapper of the C library 'Google CRC32C'" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] testing = ["pytest"] @@ -398,6 +417,7 @@ optional = false python-versions = ">=3.7" [package.dependencies] +grpcio = {version = ">=1.0.0,<2.0.0dev", optional = true, markers = "extra == \"grpc\""} protobuf = ">=3.15.0,<5.0.0dev" [package.extras] @@ -405,7 +425,7 @@ grpc = ["grpcio (>=1.0.0,<2.0.0dev)"] [[package]] name = "greenlet" -version = "1.1.2" +version = "1.1.3" description = "Lightweight in-process concurrent programming" category = "main" optional = false @@ -414,19 +434,44 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.extras] docs = ["sphinx"] +[[package]] +name = "grpc-google-iam-v1" +version = "0.12.4" +description = "IAM API client library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} +grpcio = ">=1.0.0,<2.0.0dev" + [[package]] name = "grpcio" -version = "1.48.0" +version = "1.49.1" description = "HTTP/2-based RPC framework" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] six = ">=1.5.2" [package.extras] -protobuf = ["grpcio-tools (>=1.48.0)"] +protobuf = ["grpcio-tools (>=1.49.1)"] + +[[package]] +name = "grpcio-status" +version = "1.48.2" +description = "Status proto mapping for gRPC" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.48.2" +protobuf = ">=3.12.0" [[package]] name = "gtfs-realtime-bindings" @@ -441,7 +486,7 @@ protobuf = "*" [[package]] name = "humanize" -version = "4.3.0" +version = "4.4.0" description = "Python humanize utilities" category = "main" optional = false @@ -452,7 +497,7 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "hypothesis" -version = "6.54.3" +version = "6.54.6" description = "A library for property-based testing" category = "dev" optional = false @@ -464,7 +509,7 @@ exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} sortedcontainers = ">=2.1.0,<3.0.0" [package.extras] -all = ["black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark-parser (>=0.6.5)", "libcst (>=0.3.16)", "numpy (>=1.9.0)", "pandas (>=1.0)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "importlib-metadata (>=3.6)", "backports.zoneinfo (>=0.2.1)", "tzdata (>=2022.1)"] +all = ["black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark-parser (>=0.6.5)", "libcst (>=0.3.16)", "numpy (>=1.9.0)", "pandas (>=1.0)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "importlib-metadata (>=3.6)", "backports.zoneinfo (>=0.2.1)", "tzdata (>=2022.2)"] cli = ["click (>=7.0)", "black (>=19.10b0)", "rich (>=9.0.0)"] codemods = ["libcst (>=0.3.16)"] dateutil = ["python-dateutil (>=1.4)"] @@ -477,11 +522,11 @@ pandas = ["pandas (>=1.0)"] pytest = ["pytest (>=4.6)"] pytz = ["pytz (>=2014.1)"] redis = ["redis (>=3.0.0)"] -zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2022.1)"] +zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2022.2)"] [[package]] name = "idna" -version = "3.3" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -535,7 +580,7 @@ python-versions = "*" [[package]] name = "numpy" -version = "1.23.2" +version = "1.23.3" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -543,7 +588,7 @@ python-versions = ">=3.8" [[package]] name = "oauthlib" -version = "3.2.0" +version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" category = "main" optional = false @@ -567,7 +612,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pandas" -version = "1.4.3" +version = "1.5.0" description = "Powerful data structures for data analysis, time series, and statistics" category = "main" optional = false @@ -575,30 +620,33 @@ python-versions = ">=3.8" [package.dependencies] numpy = [ - {version = ">=1.18.5", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, - {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, ] python-dateutil = ">=2.8.1" pytz = ">=2020.1" [package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +test = ["pytest-xdist (>=1.31)", "pytest (>=6.0)", "hypothesis (>=5.5.3)"] [[package]] name = "pandas-gbq" -version = "0.14.1" -description = "Pandas interface to Google BigQuery" +version = "0.17.9" +description = "Google BigQuery connector for pandas" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7, <3.11" [package.dependencies] -google-auth = "*" -google-auth-oauthlib = "*" -google-cloud-bigquery = {version = ">=1.11.1,<3.0.0dev", extras = ["bqstorage", "pandas"]} -pandas = ">=0.20.1" +db-dtypes = ">=0.3.1,<2.0.0" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0" +google-auth-oauthlib = ">=0.0.1" +google-cloud-bigquery = ">=1.27.2,<2.4.0 || >=2.5.0,<4.0.0dev" +google-cloud-bigquery-storage = ">=1.1.0,<3.0.0dev" +numpy = ">=1.16.6" +pandas = ">=0.24.2" +pyarrow = ">=3.0.0,<10.0dev" pydata-google-auth = "*" [package.extras] @@ -606,11 +654,11 @@ tqdm = ["tqdm (>=4.23.0)"] [[package]] name = "pathspec" -version = "0.9.0" +version = "0.10.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.7" [[package]] name = "pendulum" @@ -650,7 +698,7 @@ dev = ["tox", "pre-commit"] [[package]] name = "proto-plus" -version = "1.22.0" +version = "1.22.1" description = "Beautiful, Pythonic protocol buffers." category = "main" optional = false @@ -664,7 +712,7 @@ testing = ["google-api-core[grpc] (>=1.31.5)"] [[package]] name = "protobuf" -version = "3.20.1" +version = "3.20.3" description = "Protocol Buffers" category = "main" optional = false @@ -710,14 +758,14 @@ pyasn1 = ">=0.4.6,<0.5.0" [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.2" description = "Data validation and settings management using python type hints" category = "main" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.7" [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.1.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -748,14 +796,13 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" -version = "7.1.2" +version = "7.1.3" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" @@ -848,7 +895,7 @@ pyasn1 = ">=0.1.3" [[package]] name = "siuba" -version = "0.3.0" +version = "0.4.0" description = "A package for quick, scrappy analyses with pandas and SQL" category = "main" optional = false @@ -945,7 +992,7 @@ python-versions = ">=3.7" [[package]] name = "tqdm" -version = "4.64.0" +version = "4.64.1" description = "Fast, Extensible Progress Meter" category = "main" optional = false @@ -970,7 +1017,7 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.11" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -978,7 +1025,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -996,13 +1043,12 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "ed740146ace27830227f7ca6acc8c1598b3357aee96ecf003f8d684e6e37e521" +content-hash = "fb1600d3864591a40ac992513ba967b8fd6b6ec37386ac90fb2b15bd6f02a89d" [metadata.files] aiohttp = [] aiosignal = [] async-timeout = [] -atomicwrites = [] attrs = [] backoff = [] black = [] @@ -1011,6 +1057,7 @@ certifi = [] charset-normalizer = [] click = [] colorama = [] +db-dtypes = [] decorator = [] exceptiongroup = [] frozenlist = [] @@ -1023,12 +1070,15 @@ google-auth-oauthlib = [] google-cloud-bigquery = [] google-cloud-bigquery-storage = [] google-cloud-core = [] +google-cloud-secret-manager = [] google-cloud-storage = [] google-crc32c = [] google-resumable-media = [] googleapis-common-protos = [] greenlet = [] +grpc-google-iam-v1 = [] grpcio = [] +grpcio-status = [] gtfs-realtime-bindings = [] humanize = [] hypothesis = [] diff --git a/pyproject.toml b/pyproject.toml index d16e2dd..87e3d8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "calitp" -version = "2022.9.21" +version = "2022.10.26" description = "Shared code for the Cal-ITP data codebases" authors = ["Andrew Vaccaro "] license = "GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007" @@ -10,14 +10,13 @@ python = ">=3.8,<3.11" fsspec = "^2022.5.0,!=2022.7.1" # https://github.com/fsspec/gcsfs/issues/486 gcsfs = "^2022.5.0,!=2022.7.1" # https://github.com/fsspec/gcsfs/issues/486 pandas = "^1.3.3" -pandas-gbq = "<0.15.0" sqlalchemy-bigquery = "^1.4.4" google-cloud-bigquery = "<3.0.0dev,>=1.15.0" gtfs-realtime-bindings = "^0.0.7" humanize = "^4.2.3" pendulum = "^2.1.2" pydantic = "^1.9.1" -siuba = "^0.3.0" +siuba = "^0.4.0" Jinja2 = "<3.1.0" # These are just to play nicely with composer google-cloud-bigquery-storage = "2.14.1" @@ -25,6 +24,8 @@ google-api-core = "<2.0.0dev,>=1.32.0" protobuf = ">=3.19.0,<4.0.0dev" tqdm = "^4.64.0" backoff = "^2.1.2" +google-cloud-secret-manager = "^2.12.4" +pandas-gbq = "^0.17.9" [tool.poetry.dev-dependencies] black = "^22.6.0"