diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100755 index 00000000..f3f97fa5 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,14 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3/.devcontainer/base.Dockerfile + + +# [Choice] Ubuntu version (use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon): ubuntu-22.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT=ubuntu-24.04 +FROM mcr.microsoft.com/vscode/devcontainers/base:${VARIANT} + +# Postgres & our packages. Currently not customizable via VERSION param. +# RUN apt-get update \ +# && apt-get -y install --no-install-recommends curl ca-certificates gnupg +RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN apt-get update \ + && apt-get -y install --no-install-recommends postgresql-plpython3-14 postgresql-14-postgis-3 libpq-dev \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100755 index 00000000..c493b693 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,67 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3 +{ + "name": "Python3 & Poetry & Postgres", + "build": { + "dockerfile": "Dockerfile", + "args": { + // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local on arm64/Apple Silicon. + "VARIANT": "ubuntu-24.04" + } + }, + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python", + "python.linting.enabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + }, + // VSCODE ONLY: Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ] + } + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + //"forwardPorts": [48423], + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "bash ./.devcontainer/post-install.sh", + "postStartCommand": "bash ./.devcontainer/post-start.sh", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": "latest", + "ghcr.io/devcontainers/features/git:1": "latest", + // add python to container + "ghcr.io/devcontainers/features/python:1": { + "version": "3.13" + }, + // add poetry to container + "ghcr.io/devcontainers-extra/features/poetry:2": { + "version": "2.1.3" + } + }, + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + "remoteEnv": { + // "PATH": "${containerEnv:PATH}:${containerEnv:HOME}/.local/bin" + }, + "runArgs": [ + // allow container to be treated with no network isolation + "--network=host", + // give a nicer name to the container + "--name", + "${localEnv:USER}_crmprtd_devcontainer" + ] +} \ No newline at end of file diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh new file mode 100755 index 00000000..4fa1dd38 --- /dev/null +++ b/.devcontainer/post-install.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -ex + +## +## Create some aliases +## +echo 'alias ll="ls -alF"' >> $HOME/.bashrc +echo 'alias la="ls -A"' >> $HOME/.bashrc +echo 'alias l="ls -CF"' >> $HOME/.bashrc + +# Convenience workspace directory for later use +WORKSPACE_DIR=$(pwd) + +# Change some Poetry settings to better deal with working in a container +poetry config cache-dir ${WORKSPACE_DIR}/.cache +poetry config virtualenvs.in-project true + +# Now install all dependencies +poetry install --all-extras + +echo "Done!" \ No newline at end of file diff --git a/.devcontainer/post-start.sh b/.devcontainer/post-start.sh new file mode 100644 index 00000000..8ce61968 --- /dev/null +++ b/.devcontainer/post-start.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +# Convenience workspace directory for later use +WORKSPACE_DIR=$(pwd) + +# # Set current workspace as safe for git +# git config --global --add safe.directory ${WORKSPACE_DIR} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 265dfa22..5ff5caa9 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,8 @@ docs/_build/ # PyBuilder target/ + +#temp data directories +infill*/ +.pgpass + diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..cee935aa --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,40 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: crmprtd_process", + "type": "debugpy", + "module": "crmprtd.process:main", + "request": "launch", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}", + "justMyCode": false, + "env": {}, + "args": [ + "-N", "ec", + "-c", "postgresql://crmp@dbtest01.pcic.uvic.ca:5432/crmp?passfile=/workspaces/crmprtd/.pgpass", + "-L", "logging.yaml", + "-l", "infill-cache/manual-infill.log", + "-o", "DEBUG", + "-D", + "<", "infill-cache/crmprtd_download_2025-05-12.xml" + ] + }, + { + "name": "Pytest: Current File", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}", + "justMyCode": false, + "env": {}, + "args": [ + "${file}" + ] + } + ], +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..f69ea046 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,16 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "init-poetry-shell", + "type": "shell", + "command": "eval $(saml2aws script)", + "presentation": { + "reveal": "always", + "panel": "shared", + } + } + ] +} \ No newline at end of file diff --git a/crmprtd/download.py b/crmprtd/download.py index a2a3741c..07120215 100644 --- a/crmprtd/download.py +++ b/crmprtd/download.py @@ -19,6 +19,7 @@ - Cons: Significant changes to existing code. Greater complexity. Significant changes to scripts that use it. """ + from typing import List from importlib import import_module from argparse import ArgumentParser diff --git a/crmprtd/more_itertools.py b/crmprtd/more_itertools.py index a49ec21b..30a62315 100644 --- a/crmprtd/more_itertools.py +++ b/crmprtd/more_itertools.py @@ -1,6 +1,7 @@ """ Some additional iteration tools """ + from itertools import islice, cycle diff --git a/crmprtd/networks/_test/download.py b/crmprtd/networks/_test/download.py index 8c278476..35993981 100644 --- a/crmprtd/networks/_test/download.py +++ b/crmprtd/networks/_test/download.py @@ -1,6 +1,7 @@ """ A test downloader that does nothing. """ + import logging import os from argparse import ArgumentParser diff --git a/crmprtd/networks/bc_hydro/download.py b/crmprtd/networks/bc_hydro/download.py index 76d370f4..d66febdd 100644 --- a/crmprtd/networks/bc_hydro/download.py +++ b/crmprtd/networks/bc_hydro/download.py @@ -1,4 +1,4 @@ -""" Downloads data from BC hyrdo +"""Downloads data from BC hyrdo BC Hydro posts a rolling window (3 months) observing hourly data once a week. Data is in txt files. @@ -8,6 +8,7 @@ for errors. If the script is run less than once every 3 months you will miss data. """ + from typing import List import pysftp import logging diff --git a/crmprtd/networks/bc_hydro/normalize.py b/crmprtd/networks/bc_hydro/normalize.py index f305c4fd..573107c2 100644 --- a/crmprtd/networks/bc_hydro/normalize.py +++ b/crmprtd/networks/bc_hydro/normalize.py @@ -12,6 +12,7 @@ from crmprtd import Row from crmprtd import setup_logging +from crmprtd.swob_ml import get_substitutions log = logging.getLogger(__name__) @@ -28,15 +29,7 @@ def normalize(file_stream): num_pattern = re.compile(r"-?\d+(\.\d+)?$") variable_substitutions_path = "networks/bc_hydro/variable_substitutions.yaml" - try: - with (files("crmprtd") / variable_substitutions_path).open("rb") as f: - variable_substitutions = yaml.safe_load(f) - except FileNotFoundError: - log.warning( - f"Cannot open resource file '{variable_substitutions_path}'. " - f"Proceeding with normalization, but there's a risk that variable names will not be recognized." - ) - return + variable_substitutions = get_substitutions(variable_substitutions_path) for line in file_stream: line = line.decode("utf-8") @@ -79,7 +72,10 @@ def normalize(file_stream): elif num_pattern.match(value): value = float(value) - if varname in variable_substitutions: + if ( + variable_substitutions is not None + and varname in variable_substitutions + ): varname = variable_substitutions[varname] yield Row( diff --git a/crmprtd/networks/crd/download.py b/crmprtd/networks/crd/download.py index fbf392dd..910b5c75 100644 --- a/crmprtd/networks/crd/download.py +++ b/crmprtd/networks/crd/download.py @@ -15,6 +15,7 @@ can be supplied as the username in the authentication file or via the --username paramenter. No password is necessary. """ + import logging import sys from argparse import ArgumentParser diff --git a/crmprtd/networks/ec/__init__.py b/crmprtd/networks/ec/__init__.py index b7f57550..ed046c4b 100644 --- a/crmprtd/networks/ec/__init__.py +++ b/crmprtd/networks/ec/__init__.py @@ -24,8 +24,8 @@ def no_ns_element(name): def makeurl( - freq="daily", - province="BC", + freq = "daily", + province = "BC", language="e", time=None, baseurl="https://dd.weather.gc.ca", diff --git a/crmprtd/networks/ec/normalize.py b/crmprtd/networks/ec/normalize.py index d7a0e228..189dc57d 100644 --- a/crmprtd/networks/ec/normalize.py +++ b/crmprtd/networks/ec/normalize.py @@ -1,7 +1,22 @@ -from crmprtd.swob_ml import normalize as swob_ml_normalize +import logging + +from importlib.resources import files +from crmprtd import Row +from crmprtd.swob_ml import ( + normalize as swob_ml_normalize, + get_substitutions, + apply_substitutions, +) + +log = logging.getLogger(__name__) def normalize(file_stream): - return swob_ml_normalize( + variable_substitutions_path = "networks/ec/variable_substitutions.yaml" + variable_substitutions = get_substitutions(variable_substitutions_path) + + rows = swob_ml_normalize( file_stream, "EC_raw", station_id_attr="climate_station_number" ) + + return apply_substitutions(variable_substitutions, rows) diff --git a/crmprtd/networks/ec/variable_substitutions.yaml b/crmprtd/networks/ec/variable_substitutions.yaml new file mode 100644 index 00000000..a9524c92 --- /dev/null +++ b/crmprtd/networks/ec/variable_substitutions.yaml @@ -0,0 +1,10 @@ +# Defines a mapping between variable names given to us by EC +# c.a. 2022 and what the variables were named in the PCDS (a.k.a. their +# "historic" name) +# Values should be of the form: "name_in_near_real_time_feed": "net_var_name-in-pcds" + +'air_temperature_yesterday_high': 'air_temperature' +'air_temperature_yesterday_low': 'air_temperature' +'total_precipitation': 'total_precipitation' +'wind_direction': 'wind_from_direction' +'wind_gust_speed': 'wind_speed' diff --git a/crmprtd/networks/wamr/download.py b/crmprtd/networks/wamr/download.py index 06c2dce8..d11f6ce6 100644 --- a/crmprtd/networks/wamr/download.py +++ b/crmprtd/networks/wamr/download.py @@ -9,6 +9,7 @@ the last run). If the script is run less than once per month, you will miss data. """ + from typing import List import ftplib import logging diff --git a/crmprtd/process.py b/crmprtd/process.py index 2293916c..d3e2e258 100644 --- a/crmprtd/process.py +++ b/crmprtd/process.py @@ -229,8 +229,9 @@ def gulpy_plus_plus(): help="The network from which the data is coming from. " "Since gulpy input already identifies the network by way of the provided history_ids, the name will only be used for logging.", ) - parser.add_argument('filenames', metavar='filename', nargs='+', - help='CSV files to process') + parser.add_argument( + "filenames", metavar="filename", nargs="+", help="CSV files to process" + ) args = parser.parse_args() setup_logging( diff --git a/crmprtd/swob_ml.py b/crmprtd/swob_ml.py index c4551a0a..b88937e3 100644 --- a/crmprtd/swob_ml.py +++ b/crmprtd/swob_ml.py @@ -1,6 +1,8 @@ # Standard module +from typing import Generator, Optional import pytz import logging +import yaml from importlib.resources import files # Installed libraries @@ -127,3 +129,41 @@ def normalize_xml( lat=lat, lon=lon, ) + + +def get_substitutions(variable_substitutions_path) -> Optional[dict[str, str]]: + try: + with (files("crmprtd") / variable_substitutions_path).open("rb") as f: + return yaml.safe_load(f) + except FileNotFoundError: + log.warning( + f"Cannot open resource file '{variable_substitutions_path}'. " + f"Proceeding with normalization, but there's a risk that variable names will not be recognized." + ) + return + + +def apply_substitutions( + variable_substitutions: Optional[dict[str, str]], rows: Generator[Row, None, None] +): + match variable_substitutions: + case None: + log.warning( + "No variable substitutions provided. Skipping substitution step." + ) + return + case _: + for row in rows: + if row.variable_name in variable_substitutions: + yield Row( + time=row.time, + val=row.val, + variable_name=variable_substitutions[row.variable_name], + unit=row.unit, + network_name=row.network_name, + station_id=row.station_id, + lat=row.lat, + lon=row.lon, + ) + else: + yield row diff --git a/crmprtd/tests/ec_data.py b/crmprtd/tests/ec_data.py index a19c32ee..26f7fdb6 100644 --- a/crmprtd/tests/ec_data.py +++ b/crmprtd/tests/ec_data.py @@ -1,14 +1,7199 @@ from lxml.etree import fromstring hourly_bc_2016061115 = fromstring( - b"""2016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.025278 -122.362016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.243056 -121.7602782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.708335 -121.2813892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.350278 -124.1602782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.185 -128.1566672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.388611 -126.5869442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.3875 -126.5958332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.102472 -122.9363612016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.129028 -119.2895282016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.124722 -119.2927782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.49285 -130.6392016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.125848 -123.0022462016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.383167 -125.9586672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.143905 -123.1105582016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.951944 -125.2730562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.935833 -131.0158332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.296111 -117.63252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.187453 -127.4711612016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z55.687222 -121.6266672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.652583 -120.0823332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.266389 -121.6847222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.716667 -124.92016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.612222 -115.7819442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.081689 -116.500682016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.078889 -121.9786112016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.03041 -131.6015552016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z55.742222 -120.1830562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.422222 -130.0313892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.4261 -130.0252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.424608 -123.22572016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.208665 -123.8105562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.431972 -123.4393332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.383309 -126.5431092016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.453517 -125.9928962016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.841391 -122.5741672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.836389 -122.5969442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.455294 -124.2855572016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z56.238333 -120.7402782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.300222 -116.9843332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.299167 -116.9822222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.58032 -130.69782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.804611 -124.5252222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.939761 -127.632052016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.172234 -130.360852016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.369833 -121.4934722016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.368333 -121.4980562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.275 -121.2363892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.48778 -123.2994532016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.702222 -120.4419442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.7025 -120.4486112016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.94075 -119.4002112016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.957222 -119.3777782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.547694 -123.2370332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.31558 -132.7722016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.255388 -133.0584882016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.683728 -121.9341512016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.29592 -130.6092016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.224444 -121.5819442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.224444 -121.5819442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z55.305289 -123.1378022016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z55.299444 -123.1333332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.574917 -123.5299172016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.027222 -132.1252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.112502 -120.7780562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.93 -125.7666672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.269425 -117.8170942016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.054444 -123.872016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.491389 -117.3052782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.824169 -123.7188912016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.771889 -125.9967192016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.028292 -119.4409922016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.305646 -122.7340892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.302222 -122.7380562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.4625 -119.6022222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.208323 -122.6900212016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.330361 -123.2647222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.316583 -124.9268332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.680556 -127.3666672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.526306 -123.496252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.834556 -124.4968062016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.834167 -124.5002782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.888889 -122.6719452016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.884167 -122.67752016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.286111 -130.4447222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.465 -120.512016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.467778 -120.51252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.114444 -124.1355562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.337222 -124.3938892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.026669 -122.5063912016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.026111 -122.5102782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.297984 -123.5314412016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.958222 -118.1762782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.966667 -118.1833332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.15914 -131.6613262016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.703 -119.2906782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.105896 -123.3033672016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.249445 -131.8131272016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z53.254167 -131.8138892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.821111 -128.9080562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.775022 -123.1280782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.783907 -123.0447452016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.457997 -123.7152622016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.376694 -123.9210282016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z57.250278 -122.7180562016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.486611 -124.4349442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.824167 -127.1894442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.825278 -127.1827782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.11151 -127.942016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.745 -114.88392016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.745278 -114.8827782016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.783057 -123.1608352016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z55.933333 -129.9833332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.562556 -119.6486942016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.674556 -124.4031392016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z54.468611 -128.5783332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z58.653056 -124.2358332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.082222 -125.77252016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.003911 -123.1333442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.457 -123.3046112016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.1825 -123.1872362016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.295353 -123.1218692016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.194722 -123.1838892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.223306 -119.1935282016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.413304 -123.3247762016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.422778 -123.38752016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z48.647222 -123.4258332016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.111944 -117.7388892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.347042 -123.1933082016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.128889 -122.9547222016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z50.128944 -122.9546112016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z49.018056 -122.7838892016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z52.183333 -122.0544442016-06-11T15:00:00.000Z2016-06-11T15:00:00.000Z51.442889 -116.344556""" -) -hourly_bc_2016061116 = fromstring( - b"""2016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.025278 -122.362016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.243056 -121.7602782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.708335 -121.2813892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.350278 -124.1602782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.185 -128.1566672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.388611 -126.5869442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.3875 -126.5958332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.102472 -122.9363612016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.129028 -119.2895282016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.124722 -119.2927782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.49285 -130.6392016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.125848 -123.0022462016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.383167 -125.9586672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.143905 -123.1105582016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.951944 -125.2730562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.935833 -131.0158332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.296111 -117.63252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.187453 -127.4711612016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z55.687222 -121.6266672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.652583 -120.0823332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.266389 -121.6847222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.716667 -124.92016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.612222 -115.7819442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.081689 -116.500682016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.078889 -121.9786112016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.03041 -131.6015552016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z55.742222 -120.1830562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.422222 -130.0313892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.4261 -130.0252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.424608 -123.22572016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.208665 -123.8105562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.431972 -123.4393332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.383309 -126.5431092016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.453517 -125.9928962016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.841391 -122.5741672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.836389 -122.5969442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.455294 -124.2855572016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z56.238333 -120.7402782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.300222 -116.9843332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.299167 -116.9822222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.58032 -130.69782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.804611 -124.5252222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.939761 -127.632052016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.172234 -130.360852016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.369833 -121.4934722016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.368333 -121.4980562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.275 -121.2363892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.48778 -123.2994532016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.702222 -120.4419442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.7025 -120.4486112016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.94075 -119.4002112016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.957222 -119.3777782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.547694 -123.2370332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.31558 -132.7722016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.255388 -133.0584882016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.683728 -121.9341512016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.29592 -130.6092016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.224444 -121.5819442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.224444 -121.5819442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z55.305289 -123.1378022016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z55.299444 -123.1333332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.574917 -123.5299172016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.027222 -132.1252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.112502 -120.7780562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.93 -125.7666672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.269425 -117.8170942016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.054444 -123.872016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.491389 -117.3052782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.824169 -123.7188912016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.771889 -125.9967192016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.028292 -119.4409922016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.305646 -122.7340892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.302222 -122.7380562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.4625 -119.6022222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.208323 -122.6900212016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.330361 -123.2647222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.316583 -124.9268332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.680556 -127.3666672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.526306 -123.496252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.834556 -124.4968062016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.834167 -124.5002782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.888889 -122.6719452016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.884167 -122.67752016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.286111 -130.4447222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.465 -120.512016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.467778 -120.51252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.114444 -124.1355562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.337222 -124.3938892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.026669 -122.5063912016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.026111 -122.5102782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.297984 -123.5314412016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.958222 -118.1762782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.966667 -118.1833332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.15914 -131.6613262016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.703 -119.2906782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.105896 -123.3033672016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.249445 -131.8131272016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z53.254167 -131.8138892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.821111 -128.9080562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.775022 -123.1280782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.783907 -123.0447452016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.457997 -123.7152622016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.376694 -123.9210282016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z57.250278 -122.7180562016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.486611 -124.4349442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.824167 -127.1894442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.825278 -127.1827782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.11151 -127.942016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.745 -114.88392016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.745278 -114.8827782016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.783057 -123.1608352016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z55.933333 -129.9833332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.562556 -119.6486942016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.674556 -124.4031392016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z54.468611 -128.5783332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z58.653056 -124.2358332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.082222 -125.77252016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.003911 -123.1333442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.457 -123.3046112016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.1825 -123.1872362016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.295353 -123.1218692016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.194722 -123.1838892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.223306 -119.1935282016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.413304 -123.3247762016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.422778 -123.38752016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z48.647222 -123.4258332016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.111944 -117.7388892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.347042 -123.1933082016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.128889 -122.9547222016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z50.128944 -122.9546112016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z49.018056 -122.7838892016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z52.183333 -122.0544442016-06-11T16:00:00.000Z2016-06-11T16:00:00.000Z51.442889 -116.344556""" -) -yesterday_bc_20160616 = fromstring( - b"""2016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.025278 -122.362016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.243056 -121.7602782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.708335 -121.2813892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.350278 -124.1602782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.185 -128.1566672016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.388611 -126.5869442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.102472 -122.9363612016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.129028 -119.2895282016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.49285 -130.6392016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.125848 -123.0022462016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.383167 -125.9586672016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.143905 -123.1105582016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.935833 -131.0158332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.187453 -127.4711612016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.652583 -120.0823332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.266389 -121.6847222016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.716667 -124.92016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.612222 -115.7819442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.081689 -116.500682016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.03041 -131.6015552016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z55.742222 -120.1830562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z58.422222 -130.0313892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z58.4261 -130.0252016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.424608 -123.22572016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.208665 -123.8105562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.431972 -123.4393332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.383309 -126.5431092016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.453517 -125.9928962016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z58.841391 -122.5741672016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z58.836389 -122.5969442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.455294 -124.2855572016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z56.238333 -120.7402782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.300222 -116.9843332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.58032 -130.69782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.939761 -127.632052016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.172234 -130.360852016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.369833 -121.4934722016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.368333 -121.4980562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.48778 -123.2994532016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.702222 -120.4419442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.7025 -120.4486112016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.94075 -119.4002112016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.957222 -119.3777782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.31558 -132.7722016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.255388 -133.0584882016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.683728 -121.9341512016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.29592 -130.6092016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.224444 -121.5819442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.224444 -121.5819442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z55.305289 -123.1378022016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z55.299444 -123.1333332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.574917 -123.5299172016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.112502 -120.7780562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.269425 -117.8170942016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.491389 -117.3052782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.824169 -123.7188912016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.771889 -125.9967192016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.028292 -119.4409922016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.305646 -122.7340892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.4625 -119.6022222016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.208323 -122.6900212016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.330361 -123.2647222016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.316583 -124.9268332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.680556 -127.3666672016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.526306 -123.496252016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.834556 -124.4968062016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.888889 -122.6719452016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.884167 -122.67752016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.286111 -130.4447222016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.465 -120.512016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.114444 -124.1355562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.337222 -124.3938892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.026669 -122.5063912016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.026111 -122.5102782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.297984 -123.5314412016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.958222 -118.1762782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.966667 -118.1833332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.15914 -131.6613262016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.703 -119.2906782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.105896 -123.3033672016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.249445 -131.8131272016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z53.254167 -131.8138892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.821111 -128.9080562016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.775022 -123.1280782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.783907 -123.0447452016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.457997 -123.7152622016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.376694 -123.9210282016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.486611 -124.4349442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.824167 -127.1894442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.825278 -127.1827782016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.11151 -127.942016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.745 -114.88392016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.783057 -123.1608352016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.562556 -119.6486942016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.674556 -124.4031392016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z54.468611 -128.5783332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.457 -123.3046112016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.1825 -123.1872362016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.295353 -123.1218692016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.194722 -123.1838892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.223306 -119.1935282016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.413304 -123.3247762016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z48.647222 -123.4258332016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.111944 -117.7388892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.347042 -123.1933082016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z50.128944 -122.9546112016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z49.018056 -122.7838892016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z52.183333 -122.0544442016-06-16T23:00:00.000Z2016-06-16T23:00:00.000Z51.442889 -116.344556""" -) -yesterday_bc_20160617 = fromstring( - b"""2016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.025278 -122.362016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.243056 -121.7602782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.708335 -121.2813892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.350278 -124.1602782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.185 -128.1566672016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.388611 -126.5869442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.102472 -122.9363612016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.129028 -119.2895282016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.49285 -130.6392016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.125848 -123.0022462016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.383167 -125.9586672016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.143905 -123.1105582016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.935833 -131.0158332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.187453 -127.4711612016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.652583 -120.0823332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.266389 -121.6847222016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.716667 -124.92016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.612222 -115.7819442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.081689 -116.500682016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.03041 -131.6015552016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z55.742222 -120.1830562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z58.422222 -130.0313892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z58.4261 -130.0252016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.424608 -123.22572016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.208665 -123.8105562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.431972 -123.4393332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.383309 -126.5431092016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.453517 -125.9928962016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z58.841391 -122.5741672016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z58.836389 -122.5969442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.455294 -124.2855572016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z56.238333 -120.7402782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.300222 -116.9843332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.58032 -130.69782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.939761 -127.632052016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.172234 -130.360852016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.369833 -121.4934722016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.368333 -121.4980562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.48778 -123.2994532016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.702222 -120.4419442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.7025 -120.4486112016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.94075 -119.4002112016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.957222 -119.3777782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.31558 -132.7722016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.255388 -133.0584882016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.683728 -121.9341512016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.29592 -130.6092016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.224444 -121.5819442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.224444 -121.5819442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z55.305289 -123.1378022016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z55.299444 -123.1333332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.574917 -123.5299172016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.112502 -120.7780562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.269425 -117.8170942016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.491389 -117.3052782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.824169 -123.7188912016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.028292 -119.4409922016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.305646 -122.7340892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.4625 -119.6022222016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.208323 -122.6900212016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.330361 -123.2647222016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.316583 -124.9268332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.680556 -127.3666672016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.526306 -123.496252016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.834556 -124.4968062016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.888889 -122.6719452016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.884167 -122.67752016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.286111 -130.4447222016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.465 -120.512016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.114444 -124.1355562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.337222 -124.3938892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.026669 -122.5063912016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.026111 -122.5102782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.297984 -123.5314412016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.958222 -118.1762782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.966667 -118.1833332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.15914 -131.6613262016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.703 -119.2906782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.105896 -123.3033672016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.249445 -131.8131272016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z53.254167 -131.8138892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.821111 -128.9080562016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.775022 -123.1280782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.783907 -123.0447452016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.457997 -123.7152622016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.376694 -123.9210282016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.486611 -124.4349442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.824167 -127.1894442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.825278 -127.1827782016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.11151 -127.942016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.745 -114.88392016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.783057 -123.1608352016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.562556 -119.6486942016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.674556 -124.4031392016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z54.468611 -128.5783332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.457 -123.3046112016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.1825 -123.1872362016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.295353 -123.1218692016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.194722 -123.1838892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.223306 -119.1935282016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.413304 -123.3247762016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z48.647222 -123.4258332016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.111944 -117.7388892016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z49.347042 -123.1933082016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z50.128944 -122.9546112016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z52.183333 -122.0544442016-06-17T23:00:00.000Z2016-06-17T23:00:00.000Z51.442889 -116.344556""" + b""" + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.025278 -122.36 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.243056 -121.760278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.708335 -121.281389 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.350278 -124.160278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.185 -128.156667 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.388611 -126.586944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.129 -119.289917 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.49285 -130.639 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.383167 -125.958667 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.143905 -123.110558 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.935833 -131.015833 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.187453 -127.471161 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.652583 -120.082333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.266389 -121.684722 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.716667 -124.9 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.612222 -115.781944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.081689 -116.50068 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.03041 -131.601555 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 55.742222 -120.183056 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 58.422222 -130.031389 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 58.4261 -130.025 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.125848 -123.002246 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.424608 -123.2257 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.208665 -123.810556 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.431972 -123.439333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.383309 -126.543109 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.453517 -125.992896 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 58.841391 -122.574167 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 58.836389 -122.596944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.455294 -124.285557 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.459455 -124.292502 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 56.247502 -120.749724 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 56.238333 -120.740278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.300222 -116.984333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.58032 -130.6978 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.939761 -127.63205 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.172234 -130.36085 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.369833 -121.493472 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.368333 -121.498056 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.48778 -123.299453 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.702222 -120.441944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.7025 -120.448611 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.94075 -119.400211 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.957222 -119.377778 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.31558 -132.772 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.053618 -128.633635 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.255388 -133.058488 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.683728 -121.934151 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.29592 -130.609 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.224444 -121.581944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.224444 -121.581944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 55.305289 -123.137802 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 55.299444 -123.133333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.574917 -123.529917 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.112502 -120.778056 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.269425 -117.817094 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.491389 -117.305278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.824169 -123.718891 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.771889 -125.996719 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.028292 -119.440992 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.305646 -122.734089 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.4625 -119.602222 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.208323 -122.690021 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.330361 -123.264722 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.316583 -124.926833 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.684458 -127.376945 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.680556 -127.366667 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.526306 -123.49625 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.834556 -124.496806 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.888889 -122.671945 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.884167 -122.6775 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.286111 -130.444722 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.465 -120.51 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.114444 -124.135556 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.337222 -124.393889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.026669 -122.506391 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.026111 -122.510278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.297984 -123.531441 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.958222 -118.176278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.966667 -118.183333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.15914 -131.661326 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.621667 -123.418889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.703 -119.290678 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.105896 -123.303367 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.249445 -131.813127 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 53.254167 -131.813889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.821111 -128.908056 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.775022 -123.128078 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.783907 -123.044745 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.457997 -123.715262 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.376694 -123.921028 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.486611 -124.434944 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.824167 -127.189444 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.825278 -127.182778 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.11151 -127.94 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.745 -114.8839 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.783208 -123.161194 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.562556 -119.648694 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.674556 -124.403139 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 54.468611 -128.578333 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.457 -123.304611 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.1825 -123.187236 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.295353 -123.121869 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.194722 -123.183889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.223306 -119.193528 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.413304 -123.324776 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 48.647222 -123.425833 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.111944 -117.738889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.347042 -123.193308 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.128944 -122.954611 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.018056 -122.783889 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.183333 -122.054444 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 51.442889 -116.344556 + + + + + + + + + + + + + + + + + + +""" ) diff --git a/crmprtd/tests/test_ec_normalize.py b/crmprtd/tests/test_ec_normalize.py index 57e900e4..2afcfc70 100644 --- a/crmprtd/tests/test_ec_normalize.py +++ b/crmprtd/tests/test_ec_normalize.py @@ -1,77 +1,466 @@ from crmprtd.networks.ec.normalize import normalize from io import BytesIO +# this is a partial daily summary from the Environment Canada data service +# it should result in approx 24 rows of non-empty values +partial_daily = b""" + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.025278 -122.36 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.243056 -121.760278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 50.708335 -121.281389 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 49.350278 -124.160278 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.185 -128.156667 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2025-07-01T23:00:00.000Z + + + + + 2025-07-01T23:00:00.000Z + + + + + + + + + 52.388611 -126.586944 + + + + + + + + + + + + + + + + + + +""" # noqa + def test_normalize_good_data(): - lines = b""" - - - - - - - - - - - - - - - - - - - - - - - - - - - 2016-05-28T02:00:00.000Z - - - - - 2016-05-28T02:00:00.000Z - - - - - - - - - 49.025278 -122.36 - - - - - - - - - - - - - - - - - - - - - - - - -""" # noqa + lines = partial_daily rows = [row for row in normalize(BytesIO(lines))] - assert len(rows) == 10 + assert len(rows) == 24 for row in rows: assert row.station_id is not None assert row.time is not None @@ -80,6 +469,23 @@ def test_normalize_good_data(): assert row.network_name is not None +variable_names = [ + "air_temperature_yesterday_high", + "air_temperature_yesterday_low", + # 'total_precipitation', # this is present in the configuration but doesn't currently change the value + "wind_direction", + "wind_gust_speed", +] + + +def test_variable_replace(): + lines = partial_daily + rows = [row for row in normalize(BytesIO(lines))] + assert len(rows) == 24 + for row in rows: + assert row.variable_name not in variable_names + + def test_normalize_no_station_id(): lines = b""" diff --git a/logging.yaml b/logging.yaml new file mode 100644 index 00000000..23eb451b --- /dev/null +++ b/logging.yaml @@ -0,0 +1,64 @@ +# Log level standard usage +# -- debug -- +# Used to relay detailed information during testing. For the purposes +# of this application any postgresql UniquenessError should be logged +# using this level. +# +# -- info -- +# Used to output information that is useful while running application. +# +# --warning -- +# Used when handling exceptions. +# +# -- error -- +# Used for unhandled exceptions. +# +# -- exception -- +# Used like the error level, but will also output a stack trace. +# +# -- critical -- +# Used for outlying cases where program may be unable to continue. + +version: 1 +disable_existing_loggers: False +formatters: + simple: + format: '%(asctime)s:%(levelname)s:%(name)s - %(message)s' + json: + format: '%(asctime)s:%(levelname)s:%(name)s - %(message)s' + class: pythonjsonlogger.jsonlogger.JsonFormatter +handlers: + console: + class: logging.StreamHandler + level: INFO + formatter: json + stream: ext://sys.stderr + error: + class: logging.StreamHandler + level: ERROR + formatter: json + stream: ext://sys.stderr + mail: + class: logging.handlers.SMTPHandler + level: CRITICAL + formatter: json + mailhost: smtp.uvic.ca + fromaddr: noreply@pcic.uvic.ca + toaddrs: + - tkunkel@uvic.ca + subject: 'Errors with crmprtd daemon' + file: + class: logging.handlers.RotatingFileHandler + level: INFO + formatter: json + backupCount: 10 + maxBytes: 10000000 + filename: 'crmprtd_log.txt' +loggers: + crmprtd: + level: INFO + handlers: [console,mail] + propagate: yes +root: + level: DEBUG + handlers: [file] \ No newline at end of file diff --git a/scripts/BULK_PROCESS_README.md b/scripts/BULK_PROCESS_README.md new file mode 100644 index 00000000..bff6f20b --- /dev/null +++ b/scripts/BULK_PROCESS_README.md @@ -0,0 +1,117 @@ +# Bulk Process Script Usage + +This script (`bulk_process.py`) processes multiple downloaded files using the `crmprtd.process` pipeline. It's designed to work with files previously downloaded by `bulk_download.py` or other download scripts. + +## Basic Usage + +### Process all XML files in a directory +```bash +python scripts/bulk_process.py -d /path/to/directory -N ec -c "dbname=rtcrmp user=crmp" +``` + +### Process files matching a specific pattern +```bash +python scripts/bulk_process.py -d /path/to/directory -p "crmprtd_download_2025-06-*.xml" -N ec -c "dbname=rtcrmp user=crmp" +``` + +### Process a single file +```bash +python scripts/bulk_process.py -f /path/to/file.xml -N ec -c "dbname=rtcrmp user=crmp" +``` + +### Process files from a list +Create a text file with one filename per line, then: +```bash +python scripts/bulk_process.py --file-list file_list.txt -N ec -c "dbname=rtcrmp user=crmp" +``` + +## Common Options + +### Essential Parameters +- `-N, --network`: Network type (required) - one of: ec, bc_hydro, crd, moti, wamr, wmb, etc. +- `-c, --connection_string`: PostgreSQL connection string (required) + +### File Selection (choose one) +- `-d, --directory`: Directory containing files to process +- `-f, --filename`: Single file to process +- `--file-list`: Text file with list of files to process + +### Processing Options +- `-S, --start_date`: Start date filter (e.g., "2025-06-01") +- `-E, --end_date`: End date filter (e.g., "2025-06-30") +- `-D, --diag`: Diagnostic mode (no database commits) +- `-I, --infer`: Run inference stage to determine metadata +- `--continue-on-error`: Continue processing if one file fails +- `--move-processed`: Move successfully processed files to 'processed' subdirectory + +### Database Options +- `-R, --insert_strategy`: Strategy for inserting data (BULK, SINGLE, CHUNK_BISECT, ADAPTIVE) +- `-C, --bulk_chunk_size`: Chunk size for BULK strategy (default: 1000) +- `--sample_size`: Sample size for duplicate detection (default: 50) + +## Examples + +### Process Environment Canada (EC) files from infill-cache directory +```bash +python scripts/bulk_process.py \ + -d /workspaces/crmprtd/infill-cache \ + -N ec \ + -c "dbname=rtcrmp user=crmp" \ + --continue-on-error \ + --move-processed +``` + +### Process specific date range of files in diagnostic mode +```bash +python scripts/bulk_process.py \ + -d /workspaces/crmprtd/infill-cache \ + -p "crmprtd_download_2025-06-*.xml" \ + -N ec \ + -c "dbname=rtcrmp user=crmp" \ + -S "2025-06-01" \ + -E "2025-06-30" \ + -D +``` + +### Process files with custom logging +```bash +python scripts/bulk_process.py \ + -d /workspaces/crmprtd/infill-cache \ + -N ec \ + -c "dbname=rtcrmp user=crmp" \ + -l /tmp/bulk_process_custom.log \ + -e your_email@example.com \ + --log-level DEBUG +``` + +## Networks Available + +Available network types (`-N` option): +- `ec`: Environment and Climate Change Canada +- `bc_hydro`: BC Hydro +- `crd`: Capital Regional District +- `moti`: Ministry of Transportation and Infrastructure +- `wamr`: Weather stations +- `wmb`: Water Management Branch +- And others: `yt_avalanche`, `yt_water`, `yt_firewx`, `dfo_ccg_lighthouse`, etc. + +## File Management + +- Use `--move-processed` to automatically move successfully processed files to a 'processed' subdirectory +- This helps avoid reprocessing the same files +- Failed files remain in the original location for retry + + +## Integration with bulk_download.py + +This script is designed to work with files downloaded by `bulk_download.py`: + +1. First download files: + ```bash + python scripts/bulk_download.py --starttime "2025-06-01 00:00:00" --endtime "2025-06-30 23:59:59" -F daily + ``` + +2. Then process them: + ```bash + python scripts/bulk_process.py -d /path/to/cache/directory -N ec -c "dbname=rtcrmp user=crmp" + ``` diff --git a/scripts/bulk_download.py b/scripts/bulk_download.py new file mode 100755 index 00000000..c0eba2ad --- /dev/null +++ b/scripts/bulk_download.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +# Standard module +import pytz +import sys +from datetime import datetime, timedelta +from argparse import ArgumentParser +from importlib.resources import files +from time import sleep +from crmprtd.download import main as download_main +from contextlib import redirect_stdout + + +def main(opts, args): + if opts.stime and opts.etime: + # Range mode: start and end times are provided + # It is assumed that start and end times are correctly formatted on entry + # Exceptions are not handled by design. + stime = datetime.strptime(opts.stime, "%Y-%m-%d %H:%M:%S") + etime = datetime.strptime(opts.etime, "%Y-%m-%d %H:%M:%S") + + # Truncate timestamps to drop minute and second components (round down to nearest hour) + # as we'd usually be pretty close to the hour mark as part of the cron jobs + stime = stime.replace(minute=0, second=0, microsecond=0) + etime = etime.replace(minute=0, second=0, microsecond=0) + + if opts.frequency == "hourly": + timestep = timedelta(hours=1) + elif opts.frequency == "daily": + timestep = timedelta(days=1) + else: + raise Exception("Frequency must be 'hourly' or 'daily'") + + base_args = args.copy() + while stime <= etime: + iter_time = datetime.strftime(stime, "%Y-%m-%d %H:%M:%S") + # inject time, frequency, and network into the download function via the args + # argsparse for the bulk downloader steals these arguments, so we need to pass them back into the list + fun_args = [ + *base_args, + "--time", + iter_time, + "--frequency", + opts.frequency, + "--network", + opts.network, + ] + + # save this file to the cache directory, with a file name based on the network, timestamp, and frequency + with open( + f"{opts.cache_dir}/{opts.network}_{iter_time.replace(':', '-')}_{opts.frequency}.xml", + "w", + ) as f: + with redirect_stdout(f): + download_main(fun_args) + stime += timestep + sleep(3) # Avoid overwhelming the server with requests + + +if __name__ == "__main__": + sysargs = sys.argv[1:] + parser = ArgumentParser(conflict_handler="resolve") + parser.add_argument( + "-y", + "--log_conf", + dest="log_conf", + help=("YAML file to use to override the default logging " " configuration"), + ) + parser.add_argument("-l", "--log", dest="log", help="log filename") + parser.add_argument( + "-C", + "--cache_dir", + dest="cache_dir", + help=( + "directory in which to put the downloaded file in " + "the event of a post-download error" + ), + ) + parser.add_argument( + "-D", + "--diag", + dest="diag", + action="store_true", + help="Turn on diagnostic mode (no commits)", + ) + parser.add_argument( + "--start", + dest="stime", + help=( + "Start time of range to recover (interpreted with " + "strptime(format='%%Y-%%m-%%d %%H:%%M:%%S')" + ), + ) + parser.add_argument( + "--end", + dest="etime", + help=( + "End time of range to recover (interpreted with " + "strptime(format='%%Y-%%m-%%d %%H:%%M:%%S')" + ), + ) + parser.add_argument( + "-N", + "--network", + dest="network", + help="The network from which the data is coming from", + ) + parser.add_argument( + "-F", + "--frequency", + dest="frequency", + required=True, + choices=["daily", "hourly"], + help="daily|hourly - frequency for bulk download", + ) + + with (files("crmprtd") / "data/logging.yaml").open("rb") as f: + parser.set_defaults( + log_conf=f, + log="/tmp/crmp/download_main.txt", + error_email="pcic.devops@uvic.ca", + cache_dir="/home/data/projects/crmprtd/bulk_download", + diag=False, + time=None, + stime=None, + etime=datetime.now(pytz.timezone("UTC")).strftime("%Y-%m-%d %H:%M:%S"), + ) + opts, args = parser.parse_known_args(sysargs) + print(f"Parsed opts: {opts}") + main(opts, args) diff --git a/scripts/bulk_process.py b/scripts/bulk_process.py new file mode 100755 index 00000000..ca58eae4 --- /dev/null +++ b/scripts/bulk_process.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python + +# Standard modules +import os +import glob +import logging +from argparse import ArgumentParser +from importlib.resources import files +from datetime import datetime +import pytz + +# Import the process function directly instead of main +from crmprtd.process import process +from crmprtd.constants import InsertStrategy +from crmprtd.download_utils import verify_date +from crmprtd import setup_logging + + +def main(args): + """ + Main function to process multiple files in a directory using crmprtd.process. + + This script can operate in different modes: + 1. Process all files matching a pattern in a directory + 2. Process a specific file + 3. Process files from a list + """ + # Create log directory if it doesn't exist + if args.log_filename: + log_dir = os.path.dirname(args.log_filename) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + + # Setup logging first + setup_logging( + args.log_conf, + args.log_filename, + args.error_email, + args.log_level, + "crmprtd", + ) + + log = logging.getLogger("crmprtd") + + # Parse date arguments - provide defaults if not specified + utc = pytz.utc + start_date = utc.localize(verify_date(args.start_date, datetime.min, "start date")) + end_date = utc.localize(verify_date(args.end_date, datetime.max, "end date")) + + log.info(f"Processing files with date range: {start_date} to {end_date}") + + # Generate file list based on arguments + files_to_process = [] + + if args.directory and args.file_pattern: + # Process files matching pattern in directory + pattern_path = os.path.join(args.directory, args.file_pattern) + files_to_process = glob.glob(pattern_path) + log.info( + f"Found {len(files_to_process)} files matching pattern '{args.file_pattern}' in directory '{args.directory}'" + ) + + elif args.directory: + # Process all XML files in directory (default pattern) + default_pattern = "*.xml" + pattern_path = os.path.join(args.directory, default_pattern) + files_to_process = glob.glob(pattern_path) + log.info( + f"Found {len(files_to_process)} XML files in directory '{args.directory}'" + ) + + elif args.filename: + # Process single file + files_to_process = [args.filename] + log.info(f"Processing single file: {args.filename}") + + elif args.file_list: + # Process files from a list file + with open(args.file_list, "r") as f: + files_to_process = [line.strip() for line in f if line.strip()] + log.info( + f"Processing {len(files_to_process)} files from list: {args.file_list}" + ) + + else: + raise ValueError("Must specify either --directory, --filename, or --file-list") + + # Validate files exist + valid_files = [] + for file_path in files_to_process: + if os.path.exists(file_path): + valid_files.append(file_path) + else: + log.warning(f"File does not exist, skipping: {file_path}") + + if not valid_files: + log.error("No valid files found to process") + return + + log.info(f"Processing {len(valid_files)} files") + + # Process each file + successful_files = [] + failed_files = [] + + # Setup processed directory if needed + processed_dir = None + if args.move_processed and valid_files: + processed_dir = os.path.join(os.path.dirname(valid_files[0]), "processed") + os.makedirs(processed_dir, exist_ok=True) + log.info(f"Created processed directory: {processed_dir}") + + for i, file_path in enumerate(valid_files, 1): + log.info( + f"Processing file {i}/{len(valid_files)}: {os.path.basename(file_path)}" + ) + + try: + # Open file as binary stream + with open(file_path, "rb") as f: + # Call process function directly with file stream + process( + connection_string=args.connection_string, + sample_size=args.sample_size, + network=args.network, + start_date=start_date, + end_date=end_date, + is_diagnostic=args.diag, + do_infer=args.infer, + insert_strategy=InsertStrategy[args.insert_strategy], + bulk_chunk_size=args.bulk_chunk_size, + input_stream=f, + ) + + successful_files.append(file_path) + log.info(f"Successfully processed: {os.path.basename(file_path)}") + + # Move processed file immediately if requested + if args.move_processed and processed_dir: + filename = os.path.basename(file_path) + new_path = os.path.join(processed_dir, filename) + os.rename(file_path, new_path) + log.info(f"Moved processed file: {file_path} -> {new_path}") + + except Exception as e: + log.error(f"Failed to process file {os.path.basename(file_path)}: {str(e)}") + failed_files.append((file_path, str(e))) + + if not args.continue_on_error: + log.error( + "Stopping processing due to error (use --continue-on-error to continue)" + ) + break + + # Summary + log.info( + f"Processing complete. Successfully processed: {len(successful_files)}, Failed: {len(failed_files)}" + ) + + if failed_files: + log.error("Failed files:") + for file_path, error in failed_files: + log.error(f" {file_path}: {error}") + + +if __name__ == "__main__": + parser = ArgumentParser(description="Bulk process files using crmprtd.process") + + # File selection options (mutually exclusive groups would be ideal) + parser.add_argument( + "-d", "--directory", help="Directory containing files to process" + ) + parser.add_argument( + "-p", + "--file-pattern", + default="*.xml", + help="File pattern to match in directory (default: *.xml)", + ) + parser.add_argument("-f", "--filename", help="Single file to process") + parser.add_argument( + "--file-list", + help="Text file containing list of files to process (one per line)", + ) + + # Processing options + parser.add_argument( + "--continue-on-error", + action="store_true", + help="Continue processing remaining files if one fails", + ) + parser.add_argument( + "--move-processed", + action="store_true", + help="Move successfully processed files to a 'processed' subdirectory", + ) + + # Arguments passed through to crmprtd.process + parser.add_argument( + "-c", "--connection_string", help="PostgreSQL connection string" + ) + parser.add_argument( + "-N", "--network", help="The network from which the data is coming from" + ) + parser.add_argument( + "-S", + "--start_date", + help="Optional start time to use for processing (interpreted with dateutil.parser.parse)", + ) + parser.add_argument( + "-E", + "--end_date", + help="Optional end time to use for processing (interpreted with dateutil.parser.parse)", + ) + parser.add_argument( + "-D", "--diag", action="store_true", help="Turn on diagnostic mode (no commits)" + ) + parser.add_argument( + "-I", + "--infer", + action="store_true", + help="Run the 'infer' stage of the pipeline", + ) + parser.add_argument( + "-R", + "--insert_strategy", + choices=["BULK", "SINGLE", "CHUNK_BISECT", "ADAPTIVE"], + default="BULK", + help="Strategy to use for inserting observations", + ) + parser.add_argument( + "-C", + "--bulk_chunk_size", + type=int, + default=1000, + help="Fixed-length chunk size to use for BULK insertion strategy", + ) + parser.add_argument( + "--sample_size", + type=int, + default=50, + help="Number of samples to be taken from observations when searching for duplicates", + ) + parser.add_argument( + "-y", + "--log_conf", + help="YAML file to use to override the default logging configuration", + ) + parser.add_argument("-l", "--log_filename", help="Log filename") + parser.add_argument( + "-e", + "--error_email", + help="E-mail address to which the program should report errors", + ) + parser.add_argument("--log-level", help="Logging level") + + # Set defaults similar to other scripts + try: + with (files("crmprtd") / "data/logging.yaml").open("r") as f: + default_log_conf = f.name + except: + default_log_conf = None + + parser.set_defaults( + connection_string="dbname=crmprtd user=crmp", + log_conf=default_log_conf, + log_filename="/tmp/crmp/bulk_process.log", + error_email="bveerman@uvic.ca", + continue_on_error=False, + move_processed=True, + ) + + args = parser.parse_args() + main(args) diff --git a/scripts/ec_recovery.py b/scripts/ec_recovery.py deleted file mode 100755 index b54e5ea9..00000000 --- a/scripts/ec_recovery.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python - -# Standard module -from datetime import datetime, timedelta -from optparse import OptionParser -from pkg_resources import resource_stream - -# Local -from real_time_ec import main as ec_recover - - -def main(opts, args): - # It is assumed that start and end times are correctly formatted on entry - # Exceptions are not handled by design. - stime = datetime.strptime(opts.stime, "%Y/%m/%d %H:%M:%S") - etime = datetime.strptime(opts.etime, "%Y/%m/%d %H:%M:%S") - if opts.filename or opts.time: - raise Exception( - "Incorrect invocation, this wrapper only works on " - "startime-endtime ranges to download data" - ) - assert stime < etime, ( - "Start time must be less than end time... " "otherwise why are you using this?" - ) - - if opts.frequency == "hourly": - timestep = timedelta(hours=1) - elif opts.frequency == "daily": - timestep = timedelta(days=1) - - while stime <= etime: - opts.time = datetime.strftime(stime, "%Y/%m/%d %H:%M:%S") - ec_recover(opts, args) - stime += timestep - opts.log_conf.seek(0) # need to reset the resource stream back to 0 - - -if __name__ == "__main__": - parser = OptionParser() - parser.add_option( - "-c", - "--connection_string", - dest="connection_string", - help="PostgreSQL connection string", - ) - parser.add_option( - "-y", - "--log_conf", - dest="log_conf", - help=("YAML file to use to override the default logging " " configuration"), - ) - parser.add_option("-l", "--log", dest="log", help="log filename") - parser.add_option( - "-e", - "--error_email", - dest="error_email", - help=( - "e-mail address to which the program should " - "report error which require human intervention" - ), - ) - parser.add_option( - "-C", - "--cache_dir", - dest="cache_dir", - help=( - "directory in which to put the downloaded file in " - "the event of a post-download error" - ), - ) - parser.add_option( - "-f", "--filename", dest="filename", help="MPO-XML file to process" - ) - parser.add_option( - "-p", "--province", dest="province", help="2 letter province code" - ) - parser.add_option( - "-L", "--language", dest="language", help="'e' (english) | 'f' (french)" - ) - parser.add_option("-F", "--frequency", dest="frequency", help="daily|hourly") - parser.add_option( - "-t", - "--time", - dest="time", - help=( - "Alternate time to use for downloading " - "(interpreted with " - "strptime(format='%Y/%m/%d %H:%M:%S')" - ), - ) - parser.add_option( - "-T", - "--threshold", - dest="thresh", - help=( - "Distance threshold to use when matching stations." - " Stations are considered a match if they have the" - " same id, name, and are within this threshold" - ), - ) - parser.add_option( - "-D", - "--diag", - dest="diag", - action="store_true", - help="Turn on diagnostic mode (no commits)", - ) - parser.add_option( - "--starttime", - dest="stime", - help=( - "Start time of range to recover (interpreted with " - "strptime(format='%Y/%m/%d %H:%M:%S')" - ), - ) - parser.add_option( - "--endtime", - dest="etime", - help=( - "End time of range to recover (interpreted with " - "strptime(format='%Y/%m/%d %H:%M:%S')" - ), - ) - - parser.set_defaults( - connection_string="dbname=rtcrmp user=crmp", - log_conf=resource_stream("crmprtd", "/data/logging.yaml"), - log="/tmp/crmp/ec_recovery.txt", - error_email="bveerman@uvic.ca", - cache_dir="/home/data/projects/crmp/rtd/EC", - filename=None, - province="BC", - language="e", - frequency="daily", - time=None, - stime=None, - etime=None, - diag=False, - thresh=0, - ) - (opts, args) = parser.parse_args() - main(opts, args) diff --git a/scripts/infill_all.py b/scripts/infill_all.py index 0e5f4b40..6fff30db 100644 --- a/scripts/infill_all.py +++ b/scripts/infill_all.py @@ -13,6 +13,7 @@ can configure the loggging, and select a subset of available networks to infill. """ + import logging from argparse import ArgumentParser import datetime