Update ec network normalize to support variable subsitutions

.devcontainer/Dockerfile

@@ -0,0 +1,14 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3/.devcontainer/base.Dockerfile
+
+
+# [Choice] Ubuntu version (use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon): ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT=ubuntu-24.04
+FROM mcr.microsoft.com/vscode/devcontainers/base:${VARIANT}
+
+# Postgres & our packages. Currently not customizable via VERSION param.
+# RUN apt-get update \
+#     && apt-get -y install --no-install-recommends curl ca-certificates gnupg
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends postgresql-plpython3-14 postgresql-14-postgis-3 libpq-dev

.devcontainer/devcontainer.json

@@ -0,0 +1,67 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3
+{
+    "name": "Python3 & Poetry & Postgres",
+    "build": {
+        "dockerfile": "Dockerfile",
+        "args": {
+            // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+            // Append -bullseye or -buster to pin to an OS version.
+            // Use -bullseye variants on local on arm64/Apple Silicon.
+            "VARIANT": "ubuntu-24.04"
+        }
+    },
+    // Configure tool-specific properties.
+    "customizations": {
+        // Configure properties specific to VS Code.
+        "vscode": {
+            // Set *default* container specific settings.json values on container create.
+            "settings": {
+                "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+                "python.linting.enabled": true,
+                "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+                "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+                "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+                "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+                "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+                "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+                "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+                "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+            },
+            // VSCODE ONLY: Add the IDs of extensions you want installed when the container is created.
+            "extensions": [
+                "ms-python.python",
+                "ms-python.vscode-pylance"
+            ]
+        }
+    },
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    //"forwardPorts": [48423],
+    // Use 'postCreateCommand' to run commands after the container is created.
+    "postCreateCommand": "bash ./.devcontainer/post-install.sh",
+    "postStartCommand": "bash ./.devcontainer/post-start.sh",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": "latest",
+        "ghcr.io/devcontainers/features/git:1": "latest",
+        // add python to container
+        "ghcr.io/devcontainers/features/python:1": {
+            "version": "3.13"
+        },
+        // add poetry to container
+        "ghcr.io/devcontainers-extra/features/poetry:2": {
+            "version": "2.1.3"
+        }
+    },
+    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+    "remoteUser": "vscode",
+    "remoteEnv": {
+        //		"PATH": "${containerEnv:PATH}:${containerEnv:HOME}/.local/bin"
+    },
+    "runArgs": [
+        // allow container to be treated with no network isolation
+        "--network=host",
+        // give a nicer name to the container
+        "--name",
+        "${localEnv:USER}_crmprtd_devcontainer"
+    ]
+}

.devcontainer/post-install.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ex
+
+##
+## Create some aliases
+##
+echo 'alias ll="ls -alF"' >> $HOME/.bashrc
+echo 'alias la="ls -A"' >> $HOME/.bashrc
+echo 'alias l="ls -CF"' >> $HOME/.bashrc
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)
+
+# Change some Poetry settings to better deal with working in a container
+poetry config cache-dir ${WORKSPACE_DIR}/.cache
+poetry config virtualenvs.in-project true
+
+# Now install all dependencies
+poetry install --all-extras
+
+echo "Done!"

.devcontainer/post-start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)
+
+# # Set current workspace as safe for git
+# git config --global --add safe.directory ${WORKSPACE_DIR}

.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Pytest: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}",
+            "justMyCode": false,
+            "env": {
+            },
+            "args": [
+                "${file}"
+            ]
+        }
+    ]
+}

crmprtd/download.py

@@ -19,6 +19,7 @@
 - Cons: Significant changes to existing code. Greater complexity. Significant changes
 to scripts that use it.
 """
+
 from typing import List
 from importlib import import_module
 from argparse import ArgumentParser

crmprtd/more_itertools.py

@@ -1,6 +1,7 @@
 """
 Some additional iteration tools
 """
+
 from itertools import islice, cycle
 
 

crmprtd/networks/_test/download.py

@@ -1,6 +1,7 @@
 """
 A test downloader that does nothing.
 """
+
 import logging
 import os
 from argparse import ArgumentParser

crmprtd/networks/bc_hydro/download.py

@@ -1,4 +1,4 @@
-""" Downloads data from BC hyrdo
+"""Downloads data from BC hyrdo
 
 BC Hydro posts a rolling window (3 months) observing hourly data
 once a week. Data is in txt files.
@@ -8,6 +8,7 @@
 for errors. If the script is run less than once every 3 months
 you will miss data.
 """
+
 from typing import List
 import pysftp
 import logging

crmprtd/networks/crd/download.py

@@ -15,6 +15,7 @@
 can be supplied as the username in the authentication file or via the
 --username paramenter. No password is necessary.
 """
+
 import logging
 import sys
 from argparse import ArgumentParser

crmprtd/networks/ec/normalize.py

@@ -1,7 +1,41 @@
+import logging
+import yaml
+
+from importlib.resources import files
+from crmprtd import Row
 from crmprtd.swob_ml import normalize as swob_ml_normalize
 
+log = logging.getLogger(__name__)
+
 
 def normalize(file_stream):
-    return swob_ml_normalize(
+
+    variable_substitutions_path = "networks/ec/variable_substitutions.yaml"
+    try:
+        with (files("crmprtd") / variable_substitutions_path).open("rb") as f:
+            variable_substitutions = yaml.safe_load(f)
+    except FileNotFoundError:
+        log.warning(
+            f"Cannot open resource file '{variable_substitutions_path}'. "
+            f"Proceeding with normalization, but there's a risk that variable names will not be recognized."
+        )
+        return
+
+    rows = swob_ml_normalize(
         file_stream, "EC_raw", station_id_attr="climate_station_number"
     )
+
+    for row in rows:
+        if row.variable_name in variable_substitutions:
+            yield Row(
+                time=row.time,
+                val=row.val,
+                variable_name=variable_substitutions[row.variable_name],
+                unit=row.unit,
+                network_name=row.network_name,
+                station_id=row.station_id,
+                lat=row.lat,
+                lon=row.lon,
+            )
+        else:
+            yield row

crmprtd/networks/ec/variable_substitutions.yaml

@@ -0,0 +1,10 @@
+# Defines a mapping between variable names given to us by EC
+# c.a. 2022 and what the variables were named in the PCDS (a.k.a. their
+# "historic" name)
+# Values should be of the form: "name_in_near_real_time_feed": "net_var_name-in-pcds"
+
+'air_temperature_yesterday_high': 'air_temperature'
+'air_temperature_yesterday_low': 'air_temperature'
+'total_precipitation': 'total_precipitation'
+'wind_direction': 'wind_from_direction'
+'wind_gust_speed': 'wind_speed'

crmprtd/networks/wamr/download.py

@@ -9,6 +9,7 @@
 the last run). If the script is run less than once per month, you will
 miss data.
 """
+
 from typing import List
 import ftplib
 import logging

crmprtd/process.py

@@ -229,8 +229,9 @@ def gulpy_plus_plus():
         help="The network from which the data is coming from. "
         "Since gulpy input already identifies the network by way of the provided history_ids, the name will only be used for logging.",
     )
-    parser.add_argument('filenames', metavar='filename', nargs='+',
-                        help='CSV files to process')
+    parser.add_argument(
+        "filenames", metavar="filename", nargs="+", help="CSV files to process"
+    )
     args = parser.parse_args()
 
     setup_logging(