build-bgwcomp.py

#######################################################################################

# PROJECT: SAWS
# FILE NAME: build-bgwcomp.py
# AUTHOR: Caroline Adkins
# LAST UPDATED: July 2025

# Brackish Groundwater Composition (bgwcomp)

#######################################################################################
#######################################################################################

import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..","..")))

from dataset_builder.dataset import Dataset
from dataset_builder.processing import load_data_from_url, configure_columns
from dataset_builder.transformation import convert_to_saws_resolutions

if __name__ == '__main__':
    ##########################################
    ### METADATA
    ##########################################
    metadata_dict = {
        # identifiers
        'code' : 'bgwcomp',
        'name' : 'Brackish Groundwater Composition',
        'level' : 1,
        'type' : 'standard',
        'native_resolution' : 'grid1',
        'category' : 'Brackish Groundwater',

        # source
        'source_institution' : 'Stanford University',
        'source_link' : 'Not Yet Available',
        'download_link' : 'https://saws-database.s3.us-east-2.amazonaws.com/raw_data/brackish_groundwater_compositions.geojson',
        'citation' : 'Under Review',
        'update_frequency' : 'None Planned',

        # attributes, parameters, time
        'attributes' : {
            'concentration' : {
                'title' : 'Concentration',
                'value_type' : 'numerical',
                'categories' : None,
                'legend' : {
                    'colormap' : 'viridis_r',
                    'label' : 'Concentration (mg/L)'
                },
                'description' : 'concentration of species in brackish groundwater',
                'scale' : 'continuous (≥0)',
                'units' : 'mg/L'
            },
        },
        'parameters' : {
            'species' : {
                'title' : 'Species',
                'type' : 'dropdown',
                'opts' : {
                    'B' : 'Boron (B)',
                    'Ba' : 'Barium (Ba)',
                    'Br' : 'Bromine (Br)',
                    'C' : 'Carbon (C)',
                    'Ca' : 'Calcium (Ca)',
                    'Cl' : 'Chlorine (Cl)',
                    'Fe' : 'Iron (Fe)',
                    'K' : 'Potassium (K)',
                    'Li' : 'Lithium (Li)',
                    'Mg' : 'Magnesium (Mg)',
                    'Mn' : 'Manganese (Mn)',
                    'Na' : 'Sodium (Na)',
                    'SO4' : 'Sulfate (SO$_4$)',
                    'Si' : 'Silicon (Si)',
                    'Sr' : 'Strontium (Sr)',
                    'TDS' : 'Total Dissolved Solids (TDS)'
                },
            },
            'stat' : {
                'title' : 'Statistic',
                'type' : 'dropdown',
                'opts' : {
                    'mean' : 'Mean',
                    'min' : 'Min',
                    'q1' : '25th Percentile',
                    'q2' : 'Median (50th Percentile)',
                    'q3' : '75th Percentile',
                    'max' : 'Max',
                },
            }
        },
        'time' : {},
        
        # narrative of processing steps
        'processing_steps' : """Adkins et al (2025, under review) provide a dataset including statistical estimations of ion concentrations in brackish groundwater at a 1˚ grid resolution. We convert these values to SAWS resolutions using our areally weighted averaging transformation."""
    }
    data = Dataset(metadata_dict)

    ##########################################
    ### LOAD RAW DATA
    ##########################################
    print('Loading raw data...', end='\r')
    data.data_dict['raw'] = load_data_from_url(data.download_link)
    print('Finished loading raw data.', end='\n')

    ##########################################
    ### CONFIGURE COLUMNS
    ##########################################
    print('Configuring columns...', end='\r')
    # configure columns with new names and ordering
    col_config_dict = {
        'grid_cell_id' : 'grid_cell_id',
        'state_alpha' : 'state_alpha',
        'ion' : 'ion',
        'mean_conc_mgL' : 'concentration-XX_mean',
        'min_conc_mgL' : 'concentration-XX_min',
        'q1_conc_mgL' : 'concentration-XX_q1',
        'median_conc_mgL' : 'concentration-XX_q2',
        'q3_conc_mgL' : 'concentration-XX_q3',
        'max_conc_mgL' : 'concentration-XX_max',
        'geometry' : 'geometry'
    }
    df = configure_columns(data.data_dict['raw'], col_config_dict)

    # save information on original variables for documentation
    data.original_variables = {
        'mean_conc_mgL' : {
            'description' : 'mean estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        },
        'min_conc_mgL' : {
            'description' : 'minimum estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        },
        'q1_conc_mgL' : {
            'description' : 'first quartile estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        },
        'median_conc_mgL' : {
            'description' : 'median estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        },
        'q3_conc_mgL' : {
            'description' : 'third quartile estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        },
        'max_conc_mgL' : {
            'description' : 'maximum estimated concentration of species in brackish groundwater in grid cell',
            'scale' : 'continuous (≥0)',
            'units' : 'm3'
        }
    }
    print('Finished configuring columns.', end='\n')

    ##########################################
    ### DATASET-SPECIFIC PROCESSING
    ##########################################
    print('Conducting dataset-specific processing...', end='\r')
    # make unique grid cell id's by combining grid cell id with state alpha
    df['grid_cell_id'] = df['grid_cell_id'].astype(str) + '_' + df['state_alpha']

    # pivot dataframe to have one col per stat per ion
    df_wide = df.pivot(index='grid_cell_id', columns='ion', values=['concentration-XX_mean', 'concentration-XX_min', 'concentration-XX_q1', 'concentration-XX_q2', 'concentration-XX_q3', 'concentration-XX_max'])

    # rename columns in format concentration-{ion}_stat
    df_wide.columns = [f'concentration-{ion}_{stat.split('_')[-1]}' for stat, ion in df_wide.columns]
    df_wide.reset_index(inplace=True, drop=False)

    # add geometry back in
    df_wide = df[['grid_cell_id', 'geometry']].drop_duplicates().reset_index(drop=True).merge(df_wide, on='grid_cell_id', how='right')

    df = df_wide.copy()

    print('Finished conducting dataset-specific processing.', end='\n')

    ##########################################
    ### CONVERT TO SAWS RESOLUTIONS
    ##########################################
    print('Converting to other resolutions...', end='\r')
    aggregation_dict = {
        'concentration' : 'mean'
    }
    data.data_dict.update(convert_to_saws_resolutions(df, data.native_resolution, 'grid_cell_id', aggregation_dict, null_to_neg=True, save_matrix=False))
    print('Finished converting to other resolutions.', end='\n')

    ##########################################
    ### EXPORT
    ##########################################
    print('Exporting datasets...', end='\r')
    data.export()
    print('Finished exporting datasets.', end='\n')