Add Archive workflow

docs/contributing.qmd

@@ -14,19 +14,12 @@ We follow a standard feature branch workflow, as described [here.](https://docs.
     $ git clone https://github.com/Earth-Information-System/fireatlas.git
     $ cd fireatlas
 
-    # create virtual environment named `feds_env` with venv 
-    $ python -m venv feds_env
-    $ source feds_env/bin/activate
+    # create and activate environment named `fire_env`
+    $ conda env create -f env.yml
+    $ conda activate fire_env
 
-    # OR create a virtual environment with conda
-    $ conda create --name feds_env python=3.11
-    $ conda activate feds_env
-
-    # install required dependencies 
-    $ pip install -e . 
-
-    # install optional test dependencies if needed
-    $ pip install -e '.[test]'  
+    # install optional dev dependencies if needed 
+    $ pip install -e .[dev]
     ```
 
 2.  Create a new branch and make your changes as needed. Name the branch {your initials}/{descriptive feature name}:
@@ -39,7 +32,7 @@ We follow a standard feature branch workflow, as described [here.](https://docs.
     $ git commit -m "Add NOAA21 inputs"
     ```
 
-3.  Run tests (from fireatlas root dir)
+3.  Run tests from fireatlas root dir (requires optional dependencies) 
 
     ```         
     $ pytest -v tests/ 
@@ -63,6 +56,21 @@ We follow a standard feature branch workflow, as described [here.](https://docs.
 
 6.  When you are ready, [create a pull request](https://docs.github.com/en/get-started/using-github/github-flow#create-a-pull-request) to merge your branch into the main branch, `conus-dps`. Most substantive changes should be reviewed by another team member, and all changes must wait to be merged until the automated tests have passed.
 
+## Juptyer Notebooks 
+
+To use a Jupyter Notebook with the development environment: 
+
+```
+# First, create conda environment following directions in Step 1 above 
+# Then, from the fireatlasroot dir: 
+$ pip install .[dev]
+$ python -m ipykernel install --user --name fire_env --display-name "Python - FEDS development environment"
+# wait 30 seconds or so, then launch a new notebook. You should see this environment as an option to choose from in the launcher. 
+
+```
+
+This assumes you are using a JupyterLab interface such as the MAAP ADE. 
+
 ## Editing Docs
 
 We use Quarto to build this website. Most of the source files are plaintext `.qmd` files written in [Quarto markdown](https://quarto.org/docs/authoring/markdown-basics.html). You can edit them in any text editor, including with the MAAP ADE or even directly on GitHub. However, if you are making more extensive changes or using formatting, you should preview what the rendered website will look like.

env.yml

@@ -1,36 +1,11 @@
+name: fire_env
 channels:
   - conda-forge
+
 dependencies:
   - python=3.11
-  - xarray
-  - pandas
-  - dask
-  - distributed
-  - geopandas=0.14.2
-  - shapely
-  - fsspec
-  - jupyterlab
-  - s3fs=2024.6.0
-  - fiona
-  - rtree
-  - scikit-learn
-  - pyarrow
-  - rasterio
-  - pyproj
-  - ipykernel
-  - ca-certificates
-  - certifi
-  - openssl
-  - pydantic-settings
-  - pytest
-  - pytest-cov
-  - tqdm
-  - scalene
+  - awscli
   - pip
-  - numpy<2.0.0
-  - pyogrio
-  - pip:
-    - git+https://github.com/MAAP-Project/maap-py.git@develop
-    - awscli
-# NOTE: this env.yml is here for convenience but really all DPS and test installs leverage pyproject.toml
-# so if you change things here do community service and change pyproject.toml too :wink:
+
+  - pip: 
+      - -e . # Install most deps from pyproject.toml with pip

fireatlas/DataCheckUpdate.py

@@ -16,7 +16,7 @@
 from fireatlas.FireLog import logger
 from fireatlas.preprocess import preprocess_input_file
 
-MAP_KEY = "0e50658bd8e8ea368db7379b0be28630"
+MAP_KEY = "af58dc9df0eb9c54f4cad1e73c6eae4e"
 N_MAX_RETRIES = 30
 
 # ------------------------------------------------------------------------------
@@ -137,13 +137,13 @@ def update_FIRMS(d:date, sat: Literal["SNPP", "NOAA20", "NOAA21"], product: Lite
         logger.warning(
             f"{product} {sat} data is empty for {d}. This date may be outside range of data availability."
         )
-        return 
+        tst = d
+    else:   
+        daterange = pd.to_datetime(df['acq_date'])
+        tst, ted = daterange.min(), daterange.max() 
 
-    daterange = pd.to_datetime(df['acq_date'])
-    tst, ted = daterange.min(), daterange.max() 
-
-    if tst.date() != ted.date():
-        raise ValueError(f"Unexpected date range for single day file: {tst} to {ted}")
+        if tst.date() != ted.date():
+            raise ValueError(f"Unexpected date range for single day file: {tst} to {ted}")
 
     filename_out = f"FIRMS_VIIRS_{sat}_{product}_{tst.strftime('%Y%m%d')}.csv" 
     downloaded_filepath = os.path.join(data_dir, filename_out)  

fireatlas/FireConsts.py

@@ -3,13 +3,18 @@
 running controls
 """
 
-from typing import Literal
+from typing import Literal, Optional, Tuple
 import os
 import warnings
 from pyproj import CRS
 
 import fsspec
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import (
+    BaseSettings, 
+    SettingsConfigDict, 
+    PydanticBaseSettingsSource, 
+    YamlConfigSettingsSource,
+)
 from pydantic import Field, validator, field_validator
 
 
@@ -18,14 +23,49 @@
 root_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 DOTENV_ABS_PATH = os.path.join(os.path.dirname(__file__), ".env")
-
+YAML_FILENAME = "run_config.yaml"
+YAML_ABS_PATH = os.path.join(os.path.dirname(__file__), YAML_FILENAME)
 
 class Settings(BaseSettings):
-    # read in all env vars prefixed with `FEDS_` they can be in a .env file
 
-    model_config = SettingsConfigDict(
-        env_file=DOTENV_ABS_PATH, extra="ignore", env_prefix="FEDS_"
-    )
+    if os.path.exists(YAML_ABS_PATH):
+        model_config = SettingsConfigDict(
+            yaml_file=YAML_ABS_PATH, 
+            yaml_config_section="settings",
+            env_file=DOTENV_ABS_PATH, 
+            extra="ignore",
+            env_prefix="FEDS_" # note: expects env vars as FEDS_env_var_name
+        )
+    else:
+        model_config = SettingsConfigDict(
+            env_file=DOTENV_ABS_PATH, 
+            extra="ignore",
+            env_prefix="FEDS_"
+        )
+
+    # Settings resolution order (in ascending order of priority): 
+    # 1. Starts with default field values provided in FireConsts.py
+    # 2. Overrides with settings from run_config.yaml if available
+    # 3. Overrides with environment variables from .env file if available
+    # 4. Overrides with environment variables from the system env if set
+    # 5. Overrides with settings passed to the Settings class initializer if passed
+    # Result: init > system env vars > .env file > yaml file > default values
+    @classmethod
+    def settings_customise_sources(
+        cls, 
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource, 
+        env_settings: PydanticBaseSettingsSource, 
+        dotenv_settings: PydanticBaseSettingsSource, 
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings, 
+            env_settings, 
+            dotenv_settings,
+            YamlConfigSettingsSource(settings_cls), 
+            )
+
 
     # ------------------------------------------------------------------------------
     # where data is stored
@@ -48,14 +88,25 @@ class Settings(BaseSettings):
     OUTPUT_DIR: str = Field(
         "FEDSoutput-v3", description="directory where output data is stored"
     )
+    REGIONS_DIR: str = Field(
+        "run_definitions",
+        description="dirctory where region definitions are stored."
+    )
 
     READ_LOCATION: Location = Field(
         "s3",
         description="Final storage place for written files. This is where everything reads from",
     )
 
-    LOG_FILENAME: str = Field("running.log", description="Where to write logs to.")
+    LOG_FILEPATH: str = Field(
+        os.path.join(root_dir, "running.log"),
+        description="Absolute path to the log file."
+    )
 
+    ENV_META_FILEPATH: str = Field(
+        os.path.join(root_dir, "env_metadata.txt"),
+        description="Absolute path to the environment metadata file."
+    )
     # ------------------------------------------------------------------------------
     # spatiotemporal constraints of fire objects
     # ------------------------------------------------------------------------------
@@ -120,6 +171,46 @@ def check_epsg(cls, epsg: int):
         1, description="the connectivity spatial threshold (to previous fire line), km"
     )
 
+    # ------------------------------------------------------------------------------
+    # OPTIONAL: run parameters 
+    # Can be passed from run_config.yaml if using FireRunArchiveCoordinator.py
+    # or passed from the command line for all other scripts
+    # ------------------------------------------------------------------------------
+
+    TST: Optional[Tuple[int, int, int, Literal["AM", "PM"]]] = Field(
+        default=None, 
+        description="start time as [year, month, day, 'AM'/'PM']"
+    )
+
+    TED: Optional[Tuple[int, int, int, Literal["AM", "PM"]]] = Field(
+        default=None, 
+        description="end time as [year, month, day, 'AM'/'PM']"
+    )
+
+    @field_validator("TST", "TED", mode="after")
+    @classmethod
+    def _tuple_to_list(cls, v):
+        # v is already validated as a tuple of (int, int, int, str)
+        if v is None:
+            return None
+        return list(v)
+
+    RUN_NAME: Optional[str] = Field(
+        default=None,
+        description="Run name, e.g. 'ArchiveCONUS' or 'ArchiveCONUS_test'."
+    )
+
+    REGION_SHAPEFILE: Optional[str] = Field(
+        default=None, 
+        description="Name of the file that holds a shapefile that defiens this region. " 
+        "Assumes that this file is in the FEDSinput/run_definitions/RUN_NAME/ directory."
+    )
+
+    REGION_BBOX: Optional[list[float]] = Field(
+        default=None,
+        description="Bounding box of the region, e.g. [-126,24,-61,49]."
+    )
+
     # ------------------------------------------------------------------------------
     # shape parameters
     # ------------------------------------------------------------------------------
@@ -143,7 +234,9 @@ def check_epsg(cls, epsg: int):
     # MODIS pixel size
     MCD64buf: float = Field(231.7, description="MODIS fire perimeter buffer, m")
 
-    # fire source data
+    # ------------------------------------------------------------------------------
+    # fire data source parameters
+    # ------------------------------------------------------------------------------
 
     FIRE_SOURCE: Literal["SNPP", "NOAA20", "VIIRS", "BAMOD"] = Field(
         "VIIRS", description="fire source data"
@@ -184,8 +277,15 @@ def check_epsg(cls, epsg: int):
     export_to_veda: bool = Field(
         False, description="whether to export data from MAAP to VEDA s3"
     )
-    N_DASK_WORKERS: int = Field(6, description="How many dask workers to use for Run.")
 
+    # compute settings
+
+    N_DASK_WORKERS: int = Field(6, description="How many dask workers to use for Run.")
+    ARCHIVE_RUN_JOB_SIZE: int = Field(
+        10, 
+        description="How many days to run in each archive job chunk.")
+
+    # NIFC matching options
     DO_NIFC_MATCHING: bool = Field(
         False, 
         description="If True, reads from the NIFC incident database for current "
@@ -208,6 +308,8 @@ def check_epsg(cls, epsg: int):
     CONT_OPT: Literal["preset", "CA", "global"] = Field(
         "CA", description="continuity threshold option"
     )
+    # ------------------------------------------------------------------------------
+
 
     @validator("LOCAL_PATH")
     def local_path_must_not_end_with_slash(cls, v: str) -> str: