From abf4eeff84e6cdc3dfe2c414e409d32afdd6ae9d Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Wed, 5 Mar 2025 17:39:10 -0700 Subject: [PATCH 1/8] Breakout ecm_prep shared methods and variable classes in separate modules. --- scout/config.py | 3 +- scout/ecm_prep.py | 2127 +--------------------------------------- scout/ecm_prep_vars.py | 1975 +++++++++++++++++++++++++++++++++++++ scout/run.py | 15 +- scout/run_batch.py | 5 +- scout/utils.py | 80 ++ tests/ecm_prep_test.py | 91 +- 7 files changed, 2146 insertions(+), 2150 deletions(-) create mode 100644 scout/ecm_prep_vars.py create mode 100644 scout/utils.py diff --git a/scout/config.py b/scout/config.py index 2847db261..9be4b12aa 100644 --- a/scout/config.py +++ b/scout/config.py @@ -9,8 +9,7 @@ class LogConfig: - """Configure the logger - """ + """Configure the logger""" @staticmethod def configure_logging(): diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index a46f1a1b0..30ad024f5 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -17,12 +17,12 @@ import math import pandas as pd import time -from datetime import datetime -from pathlib import PurePath, Path +from pathlib import Path import argparse from scout.ecm_prep_args import ecm_args -from scout.config import FilePaths as fp -from scout.config import LogConfig +from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles, EPlusMapDicts +from scout.utils import JsonIO, PrintFormat as fmt +from scout.config import LogConfig, FilePaths as fp import traceback import logging logger = logging.getLogger(__name__) @@ -49,53 +49,9 @@ def configure_ecm_prep_logger(opts): logger.propagate = False -class MyEncoder(json.JSONEncoder): - """Convert numpy arrays to list for JSON serializing.""" - - def default(self, obj): - """Modify 'default' method from JSONEncoder.""" - # Case where object to be serialized is numpy array - if isinstance(obj, numpy.ndarray): - return obj.tolist() - if isinstance(obj, PurePath): - return str(obj) - # All other cases - else: - return super(MyEncoder, self).default(obj) - - class Utils: - @classmethod - def load_json(cls, filepath: Path) -> dict: - """Loads data from a .json file - - Args: - filepath (pathlib.Path): filepath of .json file - - Returns: - dict: .json data as a dict - """ - with open(filepath, 'r') as handle: - try: - data = json.load(handle) - except ValueError as e: - raise ValueError(f"Error reading in '{filepath}': {str(e)}") from None - return data - - @classmethod - def dump_json(cls, data, filepath: Path): - """Export data to .json file - - Args: - data: data to write to .json file - filepath (pathlib.Path): filepath of .json file - """ - with open(filepath, "w") as handle: - json.dump(data, handle, indent=2, cls=MyEncoder) - - @classmethod - def update_active_measures(cls, - run_setup: dict, + @staticmethod + def update_active_measures(run_setup: dict, to_active: list = [], to_inactive: list = [], to_skipped: list = []) -> dict: @@ -139,1972 +95,6 @@ def update_active_measures(cls, return run_setup -class UsefulInputFiles(object): - """Class of input file paths to be used by this routine. - - Attributes: - msegs_in (tuple): Database of baseline microsegment stock/energy. - msegs_cpl_in (tuple): Database of baseline technology characteristics. - iecc_reg_map (tuple): Maps IECC climates to AIA or EMM regions/states. - ba_reg_map (tuple): Maps Building America climates to AIA/EMM/states. - ash_emm_map (tuple): Maps ASHRAE climates to EMM regions. - aia_altreg_map (tuple): Maps AIA climates to EMM regions or states. - state_emm_map (tuple): Maps states to EMM regions. - state_aia_map (tuple): Maps states to AIA regions. - metadata (str) = Baseline metadata inc. min/max for year range. - glob_vars (str) = Global settings from ecm_prep to use later in run - cost_convert_in (tuple): Database of measure cost unit conversions. - cap_facts (tuple): Database of commercial equip. capacity factors. - cbecs_sf_byvint (tuple): Commercial sq.ft. by vintage data. - indiv_ecms (tuple): Individual ECM JSON definitions folder. - ecm_packages (tuple): Measure package data. - ecm_prep (tuple): Prepared measure attributes data for use in the analysis engine. - ecm_prep_env_cf (tuple): Prepared envelope/HVAC package measure - attributes data with effects of HVAC removed (isolate envelope). - ecm_prep_shapes (tuple): Prepared measure sector shapes data. - ecm_prep_env_cf_shapes (tuple): Prepared envelope/HVAC package measure - sector shapes data with effects of HVAC removed (isolate envelope). - ecm_compete_data (tuple): Folder with contributing microsegment data - needed to run measure competition in the analysis engine. - ecm_eff_fs_splt_data (tuple): Folder with data needed to determine the - fuel splits of efficient case results for fuel switching measures. - run_setup (str): Names of active measures that should be run in - the analysis engine. - cpi_data (tuple): Historical Consumer Price Index data. - ss_data (tuple): Site-source, emissions, and price data, national. - ss_data_nonfs (tuple): Site-source, emissions, and price data, - national, to assign in certain cases to non-fuel switching - microsegments under high grid decarb case. - ss_data_altreg (tuple): Emissions/price data, EMM- or state-resolved. - ss_data_altreg_nonfs (tuple): Base-case emissions/price data, EMM– or - state-resolved, to assign in certain cases to non-fuel switching - microsegments under high grid decarb case. - tsv_load_data (tuple): Time sensitive energy demand data, EMM- or - state-resolved. - tsv_cost_data (tuple): Time sensitive electricity price data, EMM- or - state-resolved. - tsv_carbon_data (tuple): Time sensitive average CO2 emissions data, - EMM- or state-resolved. - tsv_cost_data_nonfs (tuple): Time sensitive electricity price data to - assign in certain cases to non-fuel switching microsegments under - high grid decarb case, EMM- or state-resolved. - tsv_carbon_data_nonfs (tuple): Time sensitive average CO2 emissions - data to assign in certain cases to non-fuel switching microsegments - under high grid decarb case, EMM- or state-resolved. - tsv_shape_data (tuple): Custom hourly savings shape data. - tsv_metrics_data_tot (tuple): Total system load data by EMM region. - tsv_metrics_data_net (tuple): Net system load shape data by EMM region. - health_data (tuple): EPA public health benefits data by EMM region. - hp_convert_rates (tuple): Fuel switching conversion rates. - fug_emissions_dat (tuple): Refrigerant and supply chain methane leakage - data to asses fugitive emissions sources. - """ - - def __init__(self, opts): - if opts.alt_regions == 'AIA': - # UNCOMMENT WITH ISSUE 188 - # self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_cz_2017.json" - # UNCOMMENT WITH ISSUE 188 - # self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_cz_2017.json" - self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_cz.json" - self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_cz.gz" - self.iecc_reg_map = fp.CONVERT_DATA / "geo_map" / "IECC_AIA_ColSums.txt" - self.ba_reg_map = fp.CONVERT_DATA / "geo_map" / "BA_AIA_ColSums.txt" - self.state_aia_map = fp.CONVERT_DATA / "geo_map" / "AIA_State_RowSums.txt" - self.tsv_load_data = None - elif opts.alt_regions == 'EMM': - self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_emm.gz" - self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_emm.gz" - self.ash_emm_map = fp.CONVERT_DATA / "geo_map" / "ASH_EMM_ColSums.txt" - self.aia_altreg_map = fp.CONVERT_DATA / "geo_map" / "AIA_EMM_ColSums.txt" - self.iecc_reg_map = fp.CONVERT_DATA / "geo_map" / "IECC_EMM_ColSums.txt" - self.ba_reg_map = fp.CONVERT_DATA / "geo_map" / "BA_EMM_ColSums.txt" - self.state_emm_map = fp.CONVERT_DATA / "geo_map" / "EMM_State_RowSums.txt" - self.tsv_load_data = fp.TSV_DATA / "tsv_load_EMM.gz" - elif opts.alt_regions == 'State': - self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_state.gz" - self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_cdiv.gz" - self.aia_altreg_map = fp.CONVERT_DATA / "geo_map" / "AIA_State_ColSums.txt" - self.iecc_reg_map = fp.CONVERT_DATA / "geo_map" / "IECC_State_ColSums.txt" - self.ba_reg_map = fp.CONVERT_DATA / "geo_map" / "BA_State_ColSums.txt" - self.tsv_load_data = fp.TSV_DATA / "tsv_load_State.gz" - else: - raise ValueError( - "Unsupported regional breakout (" + opts.alt_regions + ")") - - self.set_decarb_grid_vars(opts) - self.metadata = fp.METADATA_PATH - self.glob_vars = fp.GENERATED / "glob_run_vars.json" - # UNCOMMENT WITH ISSUE 188 - # self.metadata = "metadata_2017.json" - self.cost_convert_in = fp.CONVERT_DATA / "ecm_cost_convert.json" - self.cap_facts = fp.CONVERT_DATA / "cap_facts.json" - self.cbecs_sf_byvint = fp.CONVERT_DATA / "cbecs_sf_byvintage.json" - self.indiv_ecms = fp.ECM_DEF - self.ecm_packages = fp.ECM_DEF / "package_ecms.json" - self.ecm_prep = fp.GENERATED / "ecm_prep.json" - self.ecm_prep_env_cf = fp.GENERATED / "ecm_prep_env_cf.json" - self.ecm_prep_shapes = fp.GENERATED / "ecm_prep_shapes.json" - self.ecm_prep_env_cf_shapes = fp.GENERATED / "ecm_prep_env_cf_shapes.json" - self.ecm_compete_data = fp.ECM_COMP - self.ecm_eff_fs_splt_data = fp.EFF_FS_SPLIT - self.run_setup = fp.GENERATED / "run_setup.json" - self.cpi_data = fp.CONVERT_DATA / "cpi.csv" - self.tsv_shape_data = ( - fp.ECM_DEF / "energyplus_data" / "savings_shapes") - self.tsv_metrics_data_tot_ref = fp.TSV_DATA / "tsv_hrs_tot_ref.csv" - self.tsv_metrics_data_net_ref = fp.TSV_DATA / "tsv_hrs_net_ref.csv" - self.tsv_metrics_data_tot_hr = fp.TSV_DATA / "tsv_hrs_tot_hr.csv" - self.tsv_metrics_data_net_hr = fp.TSV_DATA / "tsv_hrs_net_hr.csv" - self.health_data = fp.CONVERT_DATA / "epa_costs.csv" - self.hp_convert_rates = fp.CONVERT_DATA / "hp_convert_rates.json" - self.fug_emissions_dat = fp.CONVERT_DATA / "fugitive_emissions_convert.json" - - def set_decarb_grid_vars(self, opts: argparse.NameSpace): # noqa: F821 - """Assign instance variables related to grid decarbonization which are dependent on the - alt_regions, alt_ref_carb, grid_decarb_level, and grid_assessment_timing arguments. - - Args: - opts (argparse.NameSpace): argparse object containing the argument attributes - """ - - def get_suffix(arg): - """Return a suffix derived from a user-supplied argument string to append to filepath - variables; if argument is None, return an empty string. - """ - if arg is None: - return '' - else: - return f"-{arg}" - alt_ref_carb_suffix = get_suffix(opts.alt_ref_carb) - grid_decarb_level_suffix = get_suffix(opts.grid_decarb_level) - # Toggle EMM emissions and price data based on whether or not a grid decarbonization - # scenario is used - if opts.alt_regions in ['EMM', "State"]: - emission_var_map = {} # Map UsefulInputFiles instance vars to filenames suffixes - if opts.grid_decarb_level: - # Set grid decarbonization case - emission_var_map["ss_data_altreg"] = grid_decarb_level_suffix - else: - # Set baseline emissions factors - emission_var_map["ss_data_altreg"] = alt_ref_carb_suffix - self.ss_data_altreg_nonfs = None - if opts.grid_assessment_timing and opts.grid_assessment_timing == "before": - # Set emissions/cost reductions for non-fuel switching measures before grid - # decarbonization - emission_var_map["ss_data_altreg_nonfs"] = alt_ref_carb_suffix - elif (not opts.grid_decarb or - (opts.grid_assessment_timing and opts.grid_assessment_timing == "after")): - # Set emissions/cost reductions for non-fuel switching measures after grid - # decarbonization - self.ss_data_altreg_nonfs = None - # Set filepaths for EMM region emissions and prices - for var, filesuffix in emission_var_map.items(): - if opts.alt_regions == "EMM": - filepath = fp.CONVERT_DATA / f"emm_region_emissions_prices{filesuffix}.json" - else: - filepath = fp.CONVERT_DATA / f"state_emissions_prices{filesuffix}.json" - setattr(self, var, filepath) - - # Set site-source conversions and TSV files for grid decarbonization case - if opts.grid_decarb: - self.ss_data = (fp.CONVERT_DATA / - f"site_source_co2_conversions{grid_decarb_level_suffix}.json") - # Update tsv data file suffixes for DECARB levels - if "DECARB" in grid_decarb_level_suffix: - grid_decarb_level_suffix = { - "-DECARB-mid": "-95by2050", - "-DECARB-high": "-100by2035"}[grid_decarb_level_suffix] - self.tsv_cost_data = ( - fp.TSV_DATA / - f"tsv_cost-{opts.alt_regions.lower()}-{grid_decarb_level_suffix}.json") - self.tsv_carbon_data = ( - fp.TSV_DATA / - f"tsv_carbon-{opts.alt_regions.lower()}-{grid_decarb_level_suffix}.json") - - # Set site-source conversions and TSV files for non-fuel switching measures - # before grid decarbonization - if opts.grid_assessment_timing and opts.grid_assessment_timing == "before": - self.ss_data_nonfs = (fp.CONVERT_DATA / - f"site_source_co2_conversions{alt_ref_carb_suffix}.json") - self.tsv_cost_data_nonfs = (fp.TSV_DATA / - f"tsv_cost-{opts.alt_regions.lower()}-MidCase.json") - self.tsv_carbon_data_nonfs = (fp.TSV_DATA / - f"tsv_carbon-{opts.alt_regions.lower()}-MidCase.json") - # Set site-source conversions and TSV files for non-fuel switching measures - # after grid decarbonization - elif (not opts.grid_decarb or - (opts.grid_assessment_timing and opts.grid_assessment_timing == "after")): - self.ss_data_nonfs, self.tsv_cost_data_nonfs, \ - self.tsv_carbon_data_nonfs = (None for n in range(3)) - - # Set site-source conversions and TSV files for captured energy method - if opts.captured_energy: - self.ss_data = fp.CONVERT_DATA / "site_source_co2_conversions-ce.json" - elif not opts.grid_decarb: - self.ss_data = (fp.CONVERT_DATA / - f"site_source_co2_conversions{alt_ref_carb_suffix}.json") - self.tsv_cost_data = fp.TSV_DATA / f"tsv_cost-{opts.alt_regions.lower()}-MidCase.json" - self.tsv_carbon_data = (fp.TSV_DATA / - f"tsv_carbon-{opts.alt_regions.lower()}-MidCase.json") - self.ss_data_nonfs, self.tsv_cost_data_nonfs, \ - self.tsv_carbon_data_nonfs = (None for n in range(3)) - - -class UsefulVars(object): - """Class of variables that are used globally across functions. - - Attributes: - adopt_schemes_prep (list): Adopt schemes to use in preparing ECM data. - adopt_schemes_run (list): Adopt schemes to be used in competing ECMs. - full_dat_out (dict): Flag that limits technical potential (TP) data - prep/reporting when TP is not in user-specified adoption schemes. - discount_rate (float): Rate to use in discounting costs/savings. - nsamples (int): Number of samples to draw from probability distribution - on measure inputs. - regions (string): User region settings. - aeo_years (list): Modeling time horizon. - aeo_years_summary (list): Reduced set of snapshot years in the horizon. - retro_rate (dict): Annual rate of deep retrofitting existing stock. - demand_tech (list): All demand-side heating/cooling technologies. - zero_cost_tech (list): All baseline technologies with cost of zero. - inverted_relperf_list (list) = Performance units that require - an inverted relative performance calculation (e.g., an air change - rate where lower numbers indicate higher performance). - valid_submkt_urls (list) = Valid URLs for sub-market scaling fractions. - consumer_price_ind (numpy.ndarray) = Historical Consumer Price Index. - ss_conv (dict): Site-source conversion factors by fuel type. - fuel_switch_conv (dict): Performance unit conversions for expected - fuel switching cases. - carb_int (dict): Carbon intensities by fuel type (MMTon/quad). - ecosts (dict): Energy costs by building and fuel type ($/MMBtu). - ccosts (dict): Carbon costs ($/MTon). - com_timeprefs (dict): Commercial adoption time preference premiums. - hp_rates (dict): Exogenous rates of conversions from baseline - equipment to heat pumps, if applicable. - link_htcl_tover_anchor_tech_opts = For measures that apply to separate - heating and cooling technologies, stock turnover and exogenous - switching rates will be anchored on whichever technology in the - measure's definition appears first in the lists in this dict, - given the anchor end use above and applicable bldg. type (res/com) - fug_emissions (dict): Refrigerant leakage data and supply chain - methane data to support assessments of fugitive emissions. - in_all_map (dict): Maps any user-defined measure inputs marked 'all' to - list of climates, buildings, fuels, end uses, or technologies. - valid_mktnames (list): List of all valid applicable baseline market - input names for a measure. - out_break_czones (OrderedDict): Maps measure climate zone names to - the climate zone categories used in summarizing measure outputs. - out_break_bldgtypes (OrderedDict): Maps measure building type names to - the building sector categories used in summarizing measure outputs. - out_break_enduses (OrderedDict): Maps measure end use names to - the end use categories used in summarizing measure outputs. - out_break_eus_w_fsplits (List): List of end use categories that - would potentially apply across multiple fuels. - out_break_fuels (OrderedDict): Maps measure fuel types to electric vs. - non-electric fuels (for heating, cooling, WH, and cooking). - out_break_in (OrderedDict): Breaks out key measure results by - climate zone, building sector, and end use. - cconv_topkeys_map (dict): Maps measure cost units to top-level keys in - an input cost conversion data dict. - tech_units_rmv (list): Flags baseline performance units that cannot - currently be handled, thus the associated segment must be removed. - tech_units_map (dict): Maps baseline performance units to measure units - in cases where the conversion is expected (e.g., EER to COP). - sf_to_house (dict): Stores information for mapping stock units in - sf to number of households, as applicable. - com_eqp_eus_nostk (list): Flags commercial equipment end uses for - which no service demand data (which are used to represent com. - "stock") are available and square footage should be used for stock. - res_lts_per_home (list): RECS 2015 Table HC5.1 number of lights per - household, by building type, used to get from $/home to $/bulb - cconv_tech_mltstage_map (dict): Maps measure cost units to cost - conversion dict keys for demand-side heating/cooling - technologies and controls technologies requiring multiple - conversion steps (e.g., $/ft^2 glazing -> $/ft^2 wall -> - $/ft^2 floor). - cconv_bybldg_units (list): Flags cost unit conversions that must - be re-initiated for each new microsegment building type. - deflt_choice (list): Residential technology choice capital/operating - cost parameters to use when choice data are missing. - regions (str): Regions to use in geographically breaking out the data. - warm_cold_regs (dict): Warm and cold climate subsets of current - region set. - region_cpl_mapping (str or dict): Maps states to census divisions for - the case where states are used; otherwise empty string. - self.com_RTU_fs_tech (list): Flag heating tech. that pairs with RTU. - self.com_nRTU_fs_tech (list): Flag heating tech. that pairs with - larger commercial cooling equipment (not RTU). - resist_ht_wh_tech (list): Flag for resistance-based heat/WH technology. - minor_hvac_tech (list): Minor/secondary HVAC tech. to remove stock/ - stock/cost data for when major tech. is also in measure definition. - alt_attr_brk_map (dict): Mapping factors used to handle alternate - regional breakouts in measure performance, cost, or mkt. scaling. - months (str): Month sequence for accessing time-sensitive data. - tsv_feature_types (list): Possible types of TSV features. - tsv_climate_regions (list): Possible ASHRAE climate regions for - time-sensitive analysis and metrics. - tsv_nerc_regions (list): Possible NERC regions for time-sensitive data. - tsv_metrics_data (str): Includes information on max/min net system load - hours, peak/take net system load windows, and peak days by EMM - region/season, as well as days of year to attribute to each season. - tsv_hourly_price (dict): Dict for storing hourly price factors. - tsv_hourly_emissions (dict): Dict for storing hourly emissions factors. - tsv_hourly_lafs (dict): Dict for storing annual energy, cost, and - carbon adjustment factors by region, building type, and end use. - emm_name_num_map (dict): Maps EMM region names to EIA region numbers. - cz_emm_map (dict): Maps climate zones to EMM region net system load - shape data. - state_emm_map (dict): Maps states to the EMM region with the largest - geographical overlap. - health_scn_names (list): List of public health data scenario names. - health_scn_data (numpy.ndarray): Public health cost data. - env_heat_ls_scrn (tuple): Envelope heat gains to screen out of time- - sensitive valuation for heating (no load shapes for these gains). - skipped_ecms (int): List of names for ECMs skipped due to errors. - save_shp_warn (list): Tracks missing savings shape error history. - """ - - def __init__(self, base_dir, handyfiles, opts): - # Set adoption schemes to use in preparing ECM data. Note that high- - # level technical potential (TP) market data are always prepared, even - # if the user has excluded the TP adoption scheme from the run, because - # these data are later required to derive unit-level metrics in the - # ECM competition module - self.adopt_schemes_prep = ["Technical potential"] - if opts.adopt_scn_restrict is False or \ - "Max adoption potential" in opts.adopt_scn_restrict: - self.adopt_schemes_prep.append("Max adoption potential") - # Assume default adoption scenarios will be used in the competition - # scheme if user doesn't specify otherwise - if opts.adopt_scn_restrict is False: - self.adopt_schemes_run = self.adopt_schemes_prep - # Otherwise set adoption scenarios to be used in the competition - # scheme to the user-specified choice - else: - self.adopt_schemes_run = opts.adopt_scn_restrict - # Only prepare full datasets (including high-level and detailed market - # information) for adoption scenarios that will be used in the - # competition scheme. If a user has excluded the technical potential - # scheme, a limited set of high-level market data are prepared; these - # data are needed to calculate unit-level cost metrics for competition - self.full_dat_out = { - a_s: (True if a_s in self.adopt_schemes_run else False) - for a_s in self.adopt_schemes_prep} - - self.discount_rate = 0.07 - self.nsamples = 100 - self.regions = opts.alt_regions - # Load metadata including AEO year range - aeo_yrs = Utils.load_json(handyfiles.metadata) - # Set minimum modeling year to current year - aeo_min = datetime.today().year - # Set maximum modeling year - aeo_max = aeo_yrs["max year"] - # Derive time horizon from min/max years - self.aeo_years = [ - str(i) for i in range(aeo_min, aeo_max + 1)] - self.aeo_years_summary = ["2030", "2050"] - # Set early retrofit rate assumptions - - # Default case (zero early retrofits) or user has set early retrofits - # to zero - if opts.retro_set is False or opts.retro_set[0] == "1": - self.retro_rate = {yr: 0 for yr in self.aeo_years} - # User has set early retrofits to non-zero - else: - # Set default assumptions about starting values for early - # retrofits at the technology component-level. Values are based - # on survey questions about renovations in CBECS and the American - # Housing Survey, which cover lighting, HVAC, and envelope for - # commercial and HVAC and envelope for residential, respectively. - # Water heating values are assumed to be identical to HVAC - # values for the given building type, and residential lighting - # values are assumed to be identical to commercial values. Values - # for all other components are set to zero. - start_vals = { - "commercial": { - "lighting": 0.015, "HVAC": 0.009, "roof": 0.006, - "windows": 0.003, "wall": 0.003, - "water heating": 0.009, "other": 0 - }, - "residential": { - "lighting": 0.015, "HVAC": 0.005, "roof": 0.0027, - "windows": 0.0023, "wall": 0.0006, - "water heating": 0.005, "other": 0 - } - } - - # Set multipliers that progressively scale up the early retrofit - # values over time - - # User desires no change in starting values for early retrofits - # across the modeled time horizon; set multipliers to 1 across yrs. - if opts.retro_set[0] == "2": - multipliers = {yr: 1 for yr in self.aeo_years} - # User specified a rate multiplier and year by which it is - # achieved; assume linear increase in early retrofit rates from - # starting values to the increased values by the indicated year, - # and maintain increased value for all years thereafter - else: - # Pull in user-defined rate multiplier and year by which it - # is achieved - rate_inc, yr_inc = opts.retro_set[1:3] - # Calculate progressively increasing multipliers to the early - # retrofit rate based on user settings - multipliers = {yr: 1 + ((rate_inc - 1) / (yr_inc - aeo_min)) * - (int(yr) - aeo_min) if int(yr) < yr_inc else - rate_inc for yr in self.aeo_years} - # For each year, multiply starting early retrofit rate values by - # rate multipliers to obtain final early retrofit rates by year; - # nest by building type and technology component, consistent with - # the structure of the starting values above - self.retro_rate = {bldg: {cmpo: { - yr: start_vals[bldg][cmpo] * multipliers[yr] - for yr in self.aeo_years} for cmpo in start_vals[bldg].keys()} - for bldg in start_vals.keys()} - - self.demand_tech = [ - 'roof', 'ground', 'lighting gain', 'windows conduction', - 'equipment gain', 'floor', 'infiltration', 'people gain', - 'windows solar', 'ventilation', 'other heat gain', 'wall'] - # Note: ASHP costs are zero by convention in EIA data for new - # construction - self.zero_cost_tech = ['infiltration', 'ASHP'] - self.inverted_relperf_list = ["ACH", "CFM/ft^2 @ 0.3 in. w.c.", - "kWh/yr", "kWh/day", "SHGC", "HP/CFM", - "kWh/cycle"] - self.valid_submkt_urls = [ - '.eia.gov', '.doe.gov', '.energy.gov', '.data.gov', - '.energystar.gov', '.epa.gov', '.census.gov', '.pnnl.gov', - '.lbl.gov', '.nrel.gov', 'www.sciencedirect.com', 'www.costar.com', - 'www.navigantresearch.com'] - try: - self.consumer_price_ind = numpy.genfromtxt( - handyfiles.cpi_data, - names=True, delimiter=',', - dtype=[('DATE', 'U10'), ('VALUE', ' AIA mapping - try: - iecc_reg_map = numpy.genfromtxt( - handyfiles.iecc_reg_map, - names=True, delimiter='\t', dtype=( - [' AIA mapping - try: - ba_reg_map = numpy.genfromtxt( - handyfiles.ba_reg_map, names=True, delimiter='\t', - dtype=([' AIA mapping data only when methane leakage is - # assessed - if opts.fugitive_emissions is not False and \ - opts.fugitive_emissions[0] in ['1', '3']: - try: - self.fugitive_emissions_map = numpy.genfromtxt( - handyfiles.state_aia_map, names=True, - delimiter='\t', dtype=([' EMM mapping data only when methane leakage - # is assessed - if opts.fugitive_emissions is not False and \ - opts.fugitive_emissions[0] in ['1', '3']: - try: - self.fugitive_emissions_map = numpy.genfromtxt( - handyfiles.state_emm_map, names=True, - delimiter='\t', dtype=([' EMM or State mapping - try: - # Hard code number of valid states at 51 (includes DC) to avoid - # potential issues later when indexing numpy columns by state - if opts.alt_regions == "State": - len_reg = 51 - else: - len_reg = len(valid_regions) - # Read in the data - aia_altreg_map = numpy.genfromtxt( - handyfiles.aia_altreg_map, names=True, delimiter='\t', - dtype=([' EMM or State mapping - try: - iecc_altreg_map = numpy.genfromtxt( - handyfiles.iecc_reg_map, names=True, delimiter='\t', - dtype=([' EMM or State mapping - try: - ba_altreg_map = numpy.genfromtxt( - handyfiles.ba_reg_map, names=True, delimiter='\t', - dtype=(['= handydicts.structure_type[ - 'retrofit'][k][0] and \ - cbecs_yr < handydicts.structure_type[ - 'retrofit'][k][1]: - eplus_vintage_weights[k] += self.vintage_sf[k2] - total_retro_sf += self.vintage_sf[k2] - - # Run through all EnergyPlus vintage weights, normalizing the - # square footage-based weights for each 'retrofit' vintage to the - # total square footage across all 'retrofit' vintage categories - for k in eplus_vintage_weights.keys(): - # If running through the 'new' EnergyPlus vintage bin, register - # the value of its weight (should be 1) - if k == handydicts.structure_type['new']: - new_weight_sum = eplus_vintage_weights[k] - # If running through a 'retrofit' EnergyPlus vintage bin, - # normalize the square footage for that vintage by total - # square footages across 'retrofit' vintages to arrive at the - # final weight for that EnergyPlus vintage - else: - eplus_vintage_weights[k] /= total_retro_sf - retro_weight_sum += eplus_vintage_weights[k] - - # Check that the 'new' EnergyPlus vintage weight equals 1 and that - # all 'retrofit' EnergyPlus vintage weights sum to 1 - if new_weight_sum != 1: - raise ValueError("Incorrect new vintage weight total when " - "instantiating 'EPlusGlobals' object") - elif retro_weight_sum != 1: - raise ValueError("Incorrect retrofit vintage weight total when" - "instantiating 'EPlusGlobals' object") - - else: - raise KeyError( - "Unexpected EnergyPlus vintage(s) when instantiating " - "'EPlusGlobals' object; " - "check EnergyPlus vintage assumptions in structure_type " - "attribute of 'EPlusMapDict' object") - - return eplus_vintage_weights - - class Measure(object): """Set up a class representing efficiency measures as objects. @@ -4073,7 +2063,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, elif any([x == {} for x in [mseg["stock"], mseg["energy"]]]): if mskeys[-2] not in stk_energy_warn: stk_energy_warn.append(mskeys[-2]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM {self.name} missing valid baseline stock/energy data for technology " f"'{str(mskeys[-2])}'; removing technology from analysis", @@ -4604,7 +2594,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, if mskeys[-2] is not None and \ mskeys[-2] not in cpl_warn: cpl_warn.append(mskeys[-2]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' uses invalid performance units for " f"technology '{str(mskeys[-2])}' (requires " @@ -4627,7 +2617,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # Warn the user of the value/units conversions if mskeys[-2] is not None and \ mskeys[-2] not in cpl_warn: - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' uses units of {str(perf_units)} for " f"technology '{str(mskeys[-2])}' (requires " @@ -4679,7 +2669,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # also multiplied by 2 below) for this segment if mskeys[-2] not in hp_warn: hp_warn.append(mskeys[-2]) - verboseprint( + fmt.verboseprint( opts.verbose, "Stock/stock cost data for comparable residential baseline " f"technology '{str(mskeys[-2])}' multiplied by two to account " @@ -4751,7 +2741,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # special exception and how it's handled if mskeys[-2] not in cpl_warn: cpl_warn.append(mskeys[-2]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' missing valid baseline cost/performance" f"/lifetime data for technology '{str(mskeys[-2])}'; " @@ -4794,7 +2784,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # exception and how it's handled if mskeys[-2] not in cpl_warn: cpl_warn.append(mskeys[-2]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' missing valid baseline cost/performance/" f"lifetime data for technology '{str(mskeys[-2])}'; " @@ -5035,7 +3025,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, (1 - perf_meas_orig)) / perf_base[yr]) except ZeroDivisionError: - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' has baseline or measure " "performance of zero; baseline and measure " @@ -5089,7 +3079,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, rel_perf[yr] = ( perf_base[yr] / perf_meas) except ZeroDivisionError: - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' has measure performance of zero; " "baseline and measure performance set equal", @@ -5109,7 +3099,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, try: rel_perf[yr] = (perf_meas / perf_base[yr]) except ZeroDivisionError: - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' has baseline performance of zero; " "baseline and measure performance set equal", @@ -5368,7 +3358,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, if mskeys[0] == "primary": if mskeys[4] not in consume_warn: consume_warn.append(mskeys[4]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' missing valid consumer choice " f"data for end use '{str(mskeys[4])}'; using default " @@ -5420,7 +3410,7 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # for the given technology, print warning message if mskeys[4] not in consume_warn: consume_warn.append(mskeys[4]) - verboseprint( + fmt.verboseprint( opts.verbose, f"ECM '{self.name}' missing valid consumer choice data for " f"end use '{str(mskeys[4])}'; using default choice data for " @@ -7135,7 +5125,7 @@ def apply_tsv(self, load_fact, ash_cz_wts, eplus_bldg_wts, # Warn user if hasn't been done already for this mseg info. if mseg_warn not in self.handyvars.save_shp_warn: self.handyvars.save_shp_warn.append(mseg_warn) - verboseprint( + fmt.verboseprint( opts.verbose, f"Measure '{self.name}', requires custom savings shape " "data, but none were found or all values were zero for " @@ -7755,7 +5745,7 @@ def convert_costs(self, convert_data, bldg_sect, mskeys, cost_meas, user_message += " for building type '" + mskeys[2] + "'" # Print user message - verboseprint(verbose, user_message, "info") + fmt.verboseprint(verbose, user_message, "info") # Case where cost conversion has not succeeded else: raise ValueError( @@ -8127,7 +6117,7 @@ def partition_microsegment( comp_frac_diffuse_linked = { yr: 0 for yr in self.handyvars.aeo_years} cum_frac_linked = 0 - verboseprint( + fmt.verboseprint( opts.verbose, f"No data available to link mseg {str(mskeys)} for measure '{self.name}' " f"with {self.linked_htcl_tover_anchor_tech} " @@ -13486,7 +11476,7 @@ def prep_error(meas_name, handyvars, handyfiles): # Add ECM to skipped list handyvars.skipped_ecms.append(meas_name) # Print error message if in verbose mode - # verboseprint(opts.verbose, err_msg, "error") + # fmt.verboseprint(opts.verbose, err_msg, "error") # # Log error message to file (see ./generated) logger.error(err_msg) @@ -13603,35 +11593,6 @@ def split_clean_data(meas_prepped_objs, full_dat_out): meas_eff_fs_splt -def custom_formatwarning(msg, *a): - """Given a warning object, return only the warning message.""" - return str(msg) + '\n' - - -def custom_showwarning(message, category, filename, lineno, file=None, line=None): - """Define a custom warning message format.""" - # Other message details suppressed because error location and type are not relevant - print(message) - - -def verboseprint(verbose, msg, log_type): - """Print input message when the code is run in verbose mode. - - Args: - verbose (boolean): Indicator of verbose mode - msg (string): Message to print to console when in verbose mode - """ - if not verbose: - return - - if log_type == "info": - logger.info(msg) - elif log_type == "warning": - logger.warning(msg) - elif log_type == "error": - logger.error(msg) - - def tsv_cost_carb_yrmap(tsv_data, aeo_years): """Map 8760 TSV cost/carbon data years to AEO years. @@ -14100,7 +12061,7 @@ def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, # Yield warning if current contributing microsegment cannot # be mapped to an output breakout category except KeyError: - verboseprint( + fmt.verboseprint( opts.verbose, f"Baseline market key chain: '{str(mskeys)}' for ECM '{self.name}' does not map to " "output breakout categories, thus will not be reflected in output breakout data", @@ -14115,10 +12076,6 @@ def downselect_packages(existing_pkgs: list[dict], pkg_subset: list) -> list: return downselected_pkgs -def format_console_list(list_to_format): - return [f" {elem}\n" for elem in list_to_format] - - def retrieve_valid_ecms(packages: list, opts: argparse.NameSpace, # noqa: F821 handyfiles: UsefulInputFiles) -> list: @@ -14205,17 +12162,6 @@ def initialize_run_setup(input_files: UsefulInputFiles) -> dict: return run_setup -def dict_raise_on_duplicates(ordered_pairs): - """Reject duplicate keys in individual measure JSONs.""" - d = {} - for k, v in ordered_pairs: - if k in d: - raise ValueError("duplicate key %r" % (k,)) - else: - d[k] = v - return d - - def main(opts: argparse.NameSpace): # noqa: F821 """Import and prepare measure attributes for analysis engine. @@ -14235,9 +12181,6 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Set current working directory base_dir = getcwd() - # Custom format all warning messages (ignore everything but - # message itself) *** Note: sometimes yields error; investigate *** - # warnings.formatwarning = custom_formatwarning # Instantiate useful input files object handyfiles = UsefulInputFiles(opts) @@ -14269,7 +12212,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 ecm_prep_exists = "" # Import packages JSON, filter as needed - meas_toprep_package_init = Utils.load_json(handyfiles.ecm_packages) + meas_toprep_package_init = JsonIO.load_json(handyfiles.ecm_packages) meas_toprep_package_init = downselect_packages(meas_toprep_package_init, opts.ecm_packages) # If applicable, import file to write prepared measure sector shapes to @@ -14356,7 +12299,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Import all individual measure JSONs for mi in meas_toprep_indiv_names: # Load each JSON into a dict - meas_dict = Utils.load_json(handyfiles.indiv_ecms / mi) + meas_dict = JsonIO.load_json(handyfiles.indiv_ecms / mi) try: # Shorthand for previously prepared measured data that match # current measure @@ -14608,7 +12551,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 str(e)) from None # Set custom warning formatting - warnings.showwarning = custom_showwarning + warnings.showwarning = fmt.custom_showwarning # Find package measure definitions that are new or were edited since # the last time 'ecm_prep.py' routine was run, or are comprised of @@ -14639,7 +12582,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 to_active=non_ctrb_ms, to_inactive=excluded_ind_ecms) if excluded_ind_ecms: - excluded_ind_ecms_txt = format_console_list(excluded_ind_ecms) + excluded_ind_ecms_txt = fmt.format_console_list(excluded_ind_ecms) warnings.warn("The following ECMs were selected to be prepared, but due to their" " presence in one or more packages, they will not be run individually and" " will only be included as part of the package(s):" @@ -14744,16 +12687,16 @@ def main(opts: argparse.NameSpace): # noqa: F821 with gzip.GzipFile(bjszip, 'r') as zip_ref: msegs = json.loads(zip_ref.read().decode('utf-8')) else: - msegs = Utils.load_json(handyfiles.msegs_in) + msegs = JsonIO.load_json(handyfiles.msegs_in) # Import baseline cost, performance, and lifetime data bjszip = handyfiles.msegs_cpl_in with gzip.GzipFile(bjszip, 'r') as zip_ref: msegs_cpl = json.loads(zip_ref.read().decode('utf-8')) # Import measure cost unit conversion data - convert_data = Utils.load_json(handyfiles.cost_convert_in) + convert_data = JsonIO.load_json(handyfiles.cost_convert_in) # Import CBECS square footage by vintage data (used to map EnergyPlus # commercial building vintages to Scout building vintages) - cbecs_sf_byvint = Utils.load_json(handyfiles.cbecs_sf_byvint)[ + cbecs_sf_byvint = JsonIO.load_json(handyfiles.cbecs_sf_byvint)[ "commercial square footage by vintage"] if (opts.alt_regions in ['EMM', 'State'] and (( opts.tsv_metrics is not False or any([ @@ -14985,20 +12928,20 @@ def main(opts: argparse.NameSpace): # noqa: F821 with gzip.open(fs_splt_folder_name / meas_file_name, 'w') as zp: pickle.dump(meas_eff_fs_splt[ind], zp, -1) # Write prepared high-level measure attributes data to JSON - Utils.dump_json(meas_summary, handyfiles.ecm_prep) + JsonIO.dump_json(meas_summary, handyfiles.ecm_prep) # If applicable, write sector shape data to JSON if opts.sect_shapes is True: - Utils.dump_json(meas_shapes, handyfiles.ecm_prep_shapes) + JsonIO.dump_json(meas_shapes, handyfiles.ecm_prep_shapes) # Write prepared high-level counterfactual measure attributes data to # JSON (e.g., a separate file with data that will be used to isolate # the effects of envelope within envelope/HVAC packages) if opts is not None and opts.pkg_env_sep is True and \ meas_summary_env_cf is not None: - Utils.dump_json(meas_summary_env_cf, handyfiles.ecm_prep_env_cf) + JsonIO.dump_json(meas_summary_env_cf, handyfiles.ecm_prep_env_cf) # If applicable, write out envelope counterfactual sector shapes if opts.sect_shapes is True: - Utils.dump_json(meas_shapes_env_cf, handyfiles.ecm_prep_env_cf_shapes) + JsonIO.dump_json(meas_shapes_env_cf, handyfiles.ecm_prep_env_cf_shapes) # Write metadata for consistent use later in the analysis engine glob_vars = { @@ -15012,12 +12955,12 @@ def main(opts: argparse.NameSpace): # noqa: F821 "out_break_fuels": handyvars.out_break_fuels, "out_break_eus_w_fsplits": handyvars.out_break_eus_w_fsplits } - Utils.dump_json(glob_vars, handyfiles.glob_vars) + JsonIO.dump_json(glob_vars, handyfiles.glob_vars) else: logger.info("No new ECM updates available") # Write lists of active/inactive measures to be used in the analysis engine - Utils.dump_json(run_setup, handyfiles.run_setup) + JsonIO.dump_json(run_setup, handyfiles.run_setup) if __name__ == "__main__": diff --git a/scout/ecm_prep_vars.py b/scout/ecm_prep_vars.py new file mode 100644 index 000000000..c96adc101 --- /dev/null +++ b/scout/ecm_prep_vars.py @@ -0,0 +1,1975 @@ +from __future__ import annotations +import argparse +import itertools +import numpy +import re +from datetime import datetime +from collections import OrderedDict +from scout.utils import JsonIO +from scout.config import FilePaths as fp + + +class UsefulVars(object): + """Class of variables that are used globally across functions. + + Attributes: + adopt_schemes_prep (list): Adopt schemes to use in preparing ECM data. + adopt_schemes_run (list): Adopt schemes to be used in competing ECMs. + full_dat_out (dict): Flag that limits technical potential (TP) data + prep/reporting when TP is not in user-specified adoption schemes. + discount_rate (float): Rate to use in discounting costs/savings. + nsamples (int): Number of samples to draw from probability distribution + on measure inputs. + regions (string): User region settings. + aeo_years (list): Modeling time horizon. + aeo_years_summary (list): Reduced set of snapshot years in the horizon. + retro_rate (dict): Annual rate of deep retrofitting existing stock. + demand_tech (list): All demand-side heating/cooling technologies. + zero_cost_tech (list): All baseline technologies with cost of zero. + inverted_relperf_list (list) = Performance units that require + an inverted relative performance calculation (e.g., an air change + rate where lower numbers indicate higher performance). + valid_submkt_urls (list) = Valid URLs for sub-market scaling fractions. + consumer_price_ind (numpy.ndarray) = Historical Consumer Price Index. + ss_conv (dict): Site-source conversion factors by fuel type. + fuel_switch_conv (dict): Performance unit conversions for expected + fuel switching cases. + carb_int (dict): Carbon intensities by fuel type (MMTon/quad). + ecosts (dict): Energy costs by building and fuel type ($/MMBtu). + ccosts (dict): Carbon costs ($/MTon). + com_timeprefs (dict): Commercial adoption time preference premiums. + hp_rates (dict): Exogenous rates of conversions from baseline + equipment to heat pumps, if applicable. + link_htcl_tover_anchor_tech_opts = For measures that apply to separate + heating and cooling technologies, stock turnover and exogenous + switching rates will be anchored on whichever technology in the + measure's definition appears first in the lists in this dict, + given the anchor end use above and applicable bldg. type (res/com) + fug_emissions (dict): Refrigerant leakage data and supply chain + methane data to support assessments of fugitive emissions. + in_all_map (dict): Maps any user-defined measure inputs marked 'all' to + list of climates, buildings, fuels, end uses, or technologies. + valid_mktnames (list): List of all valid applicable baseline market + input names for a measure. + out_break_czones (OrderedDict): Maps measure climate zone names to + the climate zone categories used in summarizing measure outputs. + out_break_bldgtypes (OrderedDict): Maps measure building type names to + the building sector categories used in summarizing measure outputs. + out_break_enduses (OrderedDict): Maps measure end use names to + the end use categories used in summarizing measure outputs. + out_break_eus_w_fsplits (List): List of end use categories that + would potentially apply across multiple fuels. + out_break_fuels (OrderedDict): Maps measure fuel types to electric vs. + non-electric fuels (for heating, cooling, WH, and cooking). + out_break_in (OrderedDict): Breaks out key measure results by + climate zone, building sector, and end use. + cconv_topkeys_map (dict): Maps measure cost units to top-level keys in + an input cost conversion data dict. + tech_units_rmv (list): Flags baseline performance units that cannot + currently be handled, thus the associated segment must be removed. + tech_units_map (dict): Maps baseline performance units to measure units + in cases where the conversion is expected (e.g., EER to COP). + sf_to_house (dict): Stores information for mapping stock units in + sf to number of households, as applicable. + com_eqp_eus_nostk (list): Flags commercial equipment end uses for + which no service demand data (which are used to represent com. + "stock") are available and square footage should be used for stock. + res_lts_per_home (list): RECS 2015 Table HC5.1 number of lights per + household, by building type, used to get from $/home to $/bulb + cconv_tech_mltstage_map (dict): Maps measure cost units to cost + conversion dict keys for demand-side heating/cooling + technologies and controls technologies requiring multiple + conversion steps (e.g., $/ft^2 glazing -> $/ft^2 wall -> + $/ft^2 floor). + cconv_bybldg_units (list): Flags cost unit conversions that must + be re-initiated for each new microsegment building type. + deflt_choice (list): Residential technology choice capital/operating + cost parameters to use when choice data are missing. + regions (str): Regions to use in geographically breaking out the data. + warm_cold_regs (dict): Warm and cold climate subsets of current + region set. + region_cpl_mapping (str or dict): Maps states to census divisions for + the case where states are used; otherwise empty string. + self.com_RTU_fs_tech (list): Flag heating tech. that pairs with RTU. + self.com_nRTU_fs_tech (list): Flag heating tech. that pairs with + larger commercial cooling equipment (not RTU). + resist_ht_wh_tech (list): Flag for resistance-based heat/WH technology. + minor_hvac_tech (list): Minor/secondary HVAC tech. to remove stock/ + stock/cost data for when major tech. is also in measure definition. + alt_attr_brk_map (dict): Mapping factors used to handle alternate + regional breakouts in measure performance, cost, or mkt. scaling. + months (str): Month sequence for accessing time-sensitive data. + tsv_feature_types (list): Possible types of TSV features. + tsv_climate_regions (list): Possible ASHRAE climate regions for + time-sensitive analysis and metrics. + tsv_nerc_regions (list): Possible NERC regions for time-sensitive data. + tsv_metrics_data (str): Includes information on max/min net system load + hours, peak/take net system load windows, and peak days by EMM + region/season, as well as days of year to attribute to each season. + tsv_hourly_price (dict): Dict for storing hourly price factors. + tsv_hourly_emissions (dict): Dict for storing hourly emissions factors. + tsv_hourly_lafs (dict): Dict for storing annual energy, cost, and + carbon adjustment factors by region, building type, and end use. + emm_name_num_map (dict): Maps EMM region names to EIA region numbers. + cz_emm_map (dict): Maps climate zones to EMM region net system load + shape data. + state_emm_map (dict): Maps states to the EMM region with the largest + geographical overlap. + health_scn_names (list): List of public health data scenario names. + health_scn_data (numpy.ndarray): Public health cost data. + env_heat_ls_scrn (tuple): Envelope heat gains to screen out of time- + sensitive valuation for heating (no load shapes for these gains). + skipped_ecms (int): List of names for ECMs skipped due to errors. + save_shp_warn (list): Tracks missing savings shape error history. + """ + + def __init__(self, base_dir, handyfiles, opts): + # Set adoption schemes to use in preparing ECM data. Note that high- + # level technical potential (TP) market data are always prepared, even + # if the user has excluded the TP adoption scheme from the run, because + # these data are later required to derive unit-level metrics in the + # ECM competition module + self.adopt_schemes_prep = ["Technical potential"] + if opts.adopt_scn_restrict is False or \ + "Max adoption potential" in opts.adopt_scn_restrict: + self.adopt_schemes_prep.append("Max adoption potential") + # Assume default adoption scenarios will be used in the competition + # scheme if user doesn't specify otherwise + if opts.adopt_scn_restrict is False: + self.adopt_schemes_run = self.adopt_schemes_prep + # Otherwise set adoption scenarios to be used in the competition + # scheme to the user-specified choice + else: + self.adopt_schemes_run = opts.adopt_scn_restrict + # Only prepare full datasets (including high-level and detailed market + # information) for adoption scenarios that will be used in the + # competition scheme. If a user has excluded the technical potential + # scheme, a limited set of high-level market data are prepared; these + # data are needed to calculate unit-level cost metrics for competition + self.full_dat_out = { + a_s: (True if a_s in self.adopt_schemes_run else False) + for a_s in self.adopt_schemes_prep} + + self.discount_rate = 0.07 + self.nsamples = 100 + self.regions = opts.alt_regions + # Load metadata including AEO year range + aeo_yrs = JsonIO.load_json(handyfiles.metadata) + # Set minimum modeling year to current year + aeo_min = datetime.today().year + # Set maximum modeling year + aeo_max = aeo_yrs["max year"] + # Derive time horizon from min/max years + self.aeo_years = [ + str(i) for i in range(aeo_min, aeo_max + 1)] + self.aeo_years_summary = ["2030", "2050"] + # Set early retrofit rate assumptions + + # Default case (zero early retrofits) or user has set early retrofits + # to zero + if opts.retro_set is False or opts.retro_set[0] == "1": + self.retro_rate = {yr: 0 for yr in self.aeo_years} + # User has set early retrofits to non-zero + else: + # Set default assumptions about starting values for early + # retrofits at the technology component-level. Values are based + # on survey questions about renovations in CBECS and the American + # Housing Survey, which cover lighting, HVAC, and envelope for + # commercial and HVAC and envelope for residential, respectively. + # Water heating values are assumed to be identical to HVAC + # values for the given building type, and residential lighting + # values are assumed to be identical to commercial values. Values + # for all other components are set to zero. + start_vals = { + "commercial": { + "lighting": 0.015, "HVAC": 0.009, "roof": 0.006, + "windows": 0.003, "wall": 0.003, + "water heating": 0.009, "other": 0 + }, + "residential": { + "lighting": 0.015, "HVAC": 0.005, "roof": 0.0027, + "windows": 0.0023, "wall": 0.0006, + "water heating": 0.005, "other": 0 + } + } + + # Set multipliers that progressively scale up the early retrofit + # values over time + + # User desires no change in starting values for early retrofits + # across the modeled time horizon; set multipliers to 1 across yrs. + if opts.retro_set[0] == "2": + multipliers = {yr: 1 for yr in self.aeo_years} + # User specified a rate multiplier and year by which it is + # achieved; assume linear increase in early retrofit rates from + # starting values to the increased values by the indicated year, + # and maintain increased value for all years thereafter + else: + # Pull in user-defined rate multiplier and year by which it + # is achieved + rate_inc, yr_inc = opts.retro_set[1:3] + # Calculate progressively increasing multipliers to the early + # retrofit rate based on user settings + multipliers = {yr: 1 + ((rate_inc - 1) / (yr_inc - aeo_min)) * + (int(yr) - aeo_min) if int(yr) < yr_inc else + rate_inc for yr in self.aeo_years} + # For each year, multiply starting early retrofit rate values by + # rate multipliers to obtain final early retrofit rates by year; + # nest by building type and technology component, consistent with + # the structure of the starting values above + self.retro_rate = {bldg: {cmpo: { + yr: start_vals[bldg][cmpo] * multipliers[yr] + for yr in self.aeo_years} for cmpo in start_vals[bldg].keys()} + for bldg in start_vals.keys()} + + self.demand_tech = [ + 'roof', 'ground', 'lighting gain', 'windows conduction', + 'equipment gain', 'floor', 'infiltration', 'people gain', + 'windows solar', 'ventilation', 'other heat gain', 'wall'] + # Note: ASHP costs are zero by convention in EIA data for new + # construction + self.zero_cost_tech = ['infiltration', 'ASHP'] + self.inverted_relperf_list = ["ACH", "CFM/ft^2 @ 0.3 in. w.c.", + "kWh/yr", "kWh/day", "SHGC", "HP/CFM", + "kWh/cycle"] + self.valid_submkt_urls = [ + '.eia.gov', '.doe.gov', '.energy.gov', '.data.gov', + '.energystar.gov', '.epa.gov', '.census.gov', '.pnnl.gov', + '.lbl.gov', '.nrel.gov', 'www.sciencedirect.com', 'www.costar.com', + 'www.navigantresearch.com'] + try: + self.consumer_price_ind = numpy.genfromtxt( + handyfiles.cpi_data, + names=True, delimiter=',', + dtype=[('DATE', 'U10'), ('VALUE', ' AIA mapping + try: + iecc_reg_map = numpy.genfromtxt( + handyfiles.iecc_reg_map, + names=True, delimiter='\t', dtype=( + [' AIA mapping + try: + ba_reg_map = numpy.genfromtxt( + handyfiles.ba_reg_map, names=True, delimiter='\t', + dtype=([' AIA mapping data only when methane leakage is + # assessed + if opts.fugitive_emissions is not False and \ + opts.fugitive_emissions[0] in ['1', '3']: + try: + self.fugitive_emissions_map = numpy.genfromtxt( + handyfiles.state_aia_map, names=True, + delimiter='\t', dtype=([' EMM mapping data only when methane leakage + # is assessed + if opts.fugitive_emissions is not False and \ + opts.fugitive_emissions[0] in ['1', '3']: + try: + self.fugitive_emissions_map = numpy.genfromtxt( + handyfiles.state_emm_map, names=True, + delimiter='\t', dtype=([' EMM or State mapping + try: + # Hard code number of valid states at 51 (includes DC) to avoid + # potential issues later when indexing numpy columns by state + if opts.alt_regions == "State": + len_reg = 51 + else: + len_reg = len(valid_regions) + # Read in the data + aia_altreg_map = numpy.genfromtxt( + handyfiles.aia_altreg_map, names=True, delimiter='\t', + dtype=([' EMM or State mapping + try: + iecc_altreg_map = numpy.genfromtxt( + handyfiles.iecc_reg_map, names=True, delimiter='\t', + dtype=([' EMM or State mapping + try: + ba_altreg_map = numpy.genfromtxt( + handyfiles.ba_reg_map, names=True, delimiter='\t', + dtype=(['= handydicts.structure_type[ + 'retrofit'][k][0] and \ + cbecs_yr < handydicts.structure_type[ + 'retrofit'][k][1]: + eplus_vintage_weights[k] += self.vintage_sf[k2] + total_retro_sf += self.vintage_sf[k2] + + # Run through all EnergyPlus vintage weights, normalizing the + # square footage-based weights for each 'retrofit' vintage to the + # total square footage across all 'retrofit' vintage categories + for k in eplus_vintage_weights.keys(): + # If running through the 'new' EnergyPlus vintage bin, register + # the value of its weight (should be 1) + if k == handydicts.structure_type['new']: + new_weight_sum = eplus_vintage_weights[k] + # If running through a 'retrofit' EnergyPlus vintage bin, + # normalize the square footage for that vintage by total + # square footages across 'retrofit' vintages to arrive at the + # final weight for that EnergyPlus vintage + else: + eplus_vintage_weights[k] /= total_retro_sf + retro_weight_sum += eplus_vintage_weights[k] + + # Check that the 'new' EnergyPlus vintage weight equals 1 and that + # all 'retrofit' EnergyPlus vintage weights sum to 1 + if new_weight_sum != 1: + raise ValueError("Incorrect new vintage weight total when " + "instantiating 'EPlusGlobals' object") + elif retro_weight_sum != 1: + raise ValueError("Incorrect retrofit vintage weight total when" + "instantiating 'EPlusGlobals' object") + + else: + raise KeyError( + "Unexpected EnergyPlus vintage(s) when instantiating " + "'EPlusGlobals' object; " + "check EnergyPlus vintage assumptions in structure_type " + "attribute of 'EPlusMapDict' object") + + return eplus_vintage_weights diff --git a/scout/run.py b/scout/run.py index 6ceff38d3..21c81b818 100644 --- a/scout/run.py +++ b/scout/run.py @@ -13,8 +13,8 @@ import numpy_financial as npf from datetime import datetime from scout.plots import run_plot -from scout.config import FilePaths as fp -from scout.config import Config +from scout.config import Config, FilePaths as fp +from scout.utils import PrintFormat as fmt import warnings @@ -5122,9 +5122,6 @@ def main(opts: argparse.NameSpace): # noqa: F821 of key results to an output JSON. """ - # Set function that only prints message when in verbose mode - verboseprint = print if opts.verbose else lambda *a, **k: None - # Raise numpy errors as exceptions numpy.seterr('raise') # Initialize user opts variable (elements: S-S calculation method; @@ -5368,7 +5365,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Reset measure fuel split attribute to imported values m.eff_fs_splt = meas_eff_fs_data # Print data import message for each ECM if in verbose mode - verboseprint("Imported ECM '" + m.name + "' competition data") + fmt.verboseprint(opts.verbose, f"Imported ECM {m.name} competition data") # Import total absolute heating and cooling energy use data, used in # removing overlaps between supply-side and demand-side heating/cooling @@ -5455,12 +5452,12 @@ def main(opts: argparse.NameSpace): # noqa: F821 try: elec_carb = elec_cost_carb['CO2 intensity of electricity']['data'] elec_cost = elec_cost_carb['End-use electricity price']['data'] - fmt = True # Boolean for indicating data key substructure + format_data = True # Boolean for indicating data key substructure except KeyError: # Data are structured as in the site_source_co2_conversions files elec_carb = elec_cost_carb['electricity']['CO2 intensity']['data'] elec_cost = elec_cost_carb['electricity']['price']['data'] - fmt = False + format_data = False # Determine regions and building types used by active measures for # aggregating onsite generation data @@ -5501,7 +5498,7 @@ def variable_depth_dict(): return defaultdict(variable_depth_dict) else: bt_bin = 'commercial' # Get CO2 intensity and electricity cost data and convert units - if fmt: # Data (and data structure) from emm_region files + if format_data: # Data (and data structure) from emm_region files # Convert Mt/TWh to Mt/MMBtu carbtmp = {k: elec_carb[cz].get(k, 0)/3.41214e6 for k in elec_carb[cz].keys()} diff --git a/scout/run_batch.py b/scout/run_batch.py index 2c5fefba0..e6a985d7a 100644 --- a/scout/run_batch.py +++ b/scout/run_batch.py @@ -3,6 +3,7 @@ from scout.config import LogConfig, Config, FilePaths as fp from scout.ecm_prep_args import ecm_args from scout.ecm_prep import Utils, main as ecm_prep_main +from scout.utils import JsonIO from scout import run from argparse import ArgumentParser import logging @@ -122,7 +123,7 @@ def run_batch(self): ecm_prep_main(ecm_prep_opts) # Run run.main() for each yml in the group, set custom results directories - run_setup = Utils.load_json(fp.GENERATED / "run_setup.json") + run_setup = JsonIO.load_json(fp.GENERATED / "run_setup.json") ecm_files_list = self.get_ecm_files(yml_grp) for ct, config in enumerate(yml_grp): # Set all ECMs inactive @@ -130,7 +131,7 @@ def run_batch(self): to_inactive=ecm_prep_opts.ecm_files) # Set yml-specific ECMs active run_setup = Utils.update_active_measures(run_setup, to_active=ecm_files_list[ct]) - Utils.dump_json(run_setup, fp.GENERATED / "run_setup.json") + JsonIO.dump_json(run_setup, fp.GENERATED / "run_setup.json") run_opts = self.get_run_opts(config) logger.info(f"Running run.py for {config}") run.main(run_opts) diff --git a/scout/utils.py b/scout/utils.py new file mode 100644 index 000000000..b9966657a --- /dev/null +++ b/scout/utils.py @@ -0,0 +1,80 @@ +import json +import numpy +from pathlib import Path, PurePath + + +class JsonIO: + @staticmethod + def load_json(filepath: Path) -> dict: + """Loads data from a .json file + + Args: + filepath (pathlib.Path): filepath of .json file + + Returns: + dict: .json data as a dict + """ + with open(filepath, 'r') as handle: + try: + data = json.load(handle) + except ValueError as e: + raise ValueError(f"Error reading in '{filepath}': {str(e)}") from None + return data + + @staticmethod + def dump_json(data, filepath: Path): + """Export data to .json file + + Args: + data: data to write to .json file + filepath (pathlib.Path): filepath of .json file + """ + with open(filepath, "w") as handle: + json.dump(data, handle, indent=2, cls=MyEncoder) + + +class MyEncoder(json.JSONEncoder): + """Convert numpy arrays to list for JSON serializing.""" + + def default(self, obj): + """Modify 'default' method from JSONEncoder.""" + # Case where object to be serialized is numpy array + if isinstance(obj, numpy.ndarray): + return obj.tolist() + if isinstance(obj, PurePath): + return str(obj) + # All other cases + else: + return super(MyEncoder, self).default(obj) + + +class PrintFormat: + """Class for customizing print messages.""" + + @staticmethod + def custom_showwarning(message, category, filename, lineno, file=None, line=None): + """Define a custom warning message format.""" + # Other message details suppressed because error location and type are not relevant + print(message) + + @staticmethod + def verboseprint(verbose, msg, log_type): + """Print input message when the code is run in verbose mode. + + Args: + verbose (boolean): Indicator of verbose mode + msg (string): Message to print to console when in verbose mode + """ + if not verbose: + return + + if log_type == "info": + logger.info(msg) + elif log_type == "warning": + logger.warning(msg) + elif log_type == "error": + logger.error(msg) + + @staticmethod + def format_console_list(list_to_format): + return [f" {elem}\n" for elem in list_to_format] diff --git a/tests/ecm_prep_test.py b/tests/ecm_prep_test.py index d1ea290c5..4afdc818e 100644 --- a/tests/ecm_prep_test.py +++ b/tests/ecm_prep_test.py @@ -4,6 +4,7 @@ # Import code to be tested from scout import ecm_prep +from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles, EPlusGlobals from scout.config import FilePaths as fp from scout.ecm_prep_args import ecm_args # Import needed packages @@ -169,7 +170,7 @@ def setUpClass(cls): '1990 to 1999': 13803.0, '2000 to 2003': 7215.0, 'Before 1920': 3980.0, '2008 to 2012': 5726.0, '1920 to 1945': 6020.0, '1980 to 1989': 15185.0} - cls.eplus_globals_ok = ecm_prep.EPlusGlobals( + cls.eplus_globals_ok = EPlusGlobals( fp.ECM_DEF / "energyplus_data" / "energyplus_test_ok", cls.cbecs_sf_byvint) cls.eplus_failpath = fp.ECM_DEF / "energyplus_data" / "energyplus_test_fail" @@ -204,7 +205,7 @@ def test_vintageweights_fail(self): AssertionError: If KeyError is not raised. """ with self.assertRaises(KeyError): - ecm_prep.EPlusGlobals( + EPlusGlobals( self.eplus_failpath, self.cbecs_sf_byvint).find_vintage_weights() @@ -292,8 +293,8 @@ def setUpClass(cls): # Null user options/options dict opts, opts_dict = (NullOpts().opts, NullOpts().opts_dict) # Useful global variables for the sample measure object - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) cls.meas = ecm_prep.Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) # Finalize the measure's 'technology_type' attribute (handled by the @@ -879,8 +880,8 @@ def setUpClass(cls): # Null user options/options dict cls.opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] # National-level variables (AIA regions) - handyfiles = ecm_prep.UsefulInputFiles(cls.opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, cls.opts) + handyfiles = UsefulInputFiles(cls.opts) + handyvars = UsefulVars(base_dir, handyfiles, cls.opts) handyvars.com_eqp_eus_nostk = ["lighting", "PCs", "MELs"] # Site energy assessment settings cls.opts_site_energy, opts_site_energy_dict = [ @@ -894,8 +895,8 @@ def setUpClass(cls): "EMM" for n in range(2)) cls.opts_emm.site_energy, opts_emm_dict["site_energy"] = ( True for n in range(2)) - handyfiles_emm = ecm_prep.UsefulInputFiles(cls.opts_emm) - handyvars_emm = ecm_prep.UsefulVars( + handyfiles_emm = UsefulInputFiles(cls.opts_emm) + handyvars_emm = UsefulVars( base_dir, handyfiles_emm, cls.opts_emm) # Regional-level variables (states) cls.opts_state, opts_state_dict = [ @@ -904,8 +905,8 @@ def setUpClass(cls): "State" for n in range(2)) cls.opts_state.site_energy, opts_state_dict["site_energy"] = ( True for n in range(2)) - handyfiles_state = ecm_prep.UsefulInputFiles(cls.opts_state) - handyvars_state = ecm_prep.UsefulVars( + handyfiles_state = UsefulInputFiles(cls.opts_state) + handyvars_state = UsefulVars( base_dir, handyfiles_state, cls.opts_state) # Fuel switching with exogenous rates cls.opts_hp_rates, opts_hp_rates_dict = [ @@ -916,7 +917,7 @@ def setUpClass(cls): "exog_hp_rates"] = (["aggressive", '1'] for n in range(2)) cls.opts_hp_rates.adopt_scn_usr, opts_hp_rates_dict[ "adopt_scn_usr"] = (["Max adoption potential"] for n in range(2)) - handyvars_hp_rates = ecm_prep.UsefulVars( + handyvars_hp_rates = UsefulVars( base_dir, handyfiles_emm, cls.opts_hp_rates) handyvars_hp_rates.hp_rates_reg_map = { "midwest": [ @@ -976,7 +977,7 @@ def setUpClass(cls): copy.deepcopy(x) for x in [cls.opts_hp_rates, opts_hp_rates_dict]] cls.opts_hp_no_rates.exog_hp_rates, opts_hp_no_rates_dict[ "exog_hp_rates"] = (False for n in range(2)) - handyvars_hp_norates = ecm_prep.UsefulVars( + handyvars_hp_norates = UsefulVars( base_dir, handyfiles_emm, cls.opts_hp_no_rates) handyvars_hp_rates.out_break_in, handyvars_hp_norates.out_break_in = ( {'TRE': {'Residential (Existing)': { @@ -1096,7 +1097,7 @@ def setUpClass(cls): "fugitive_emissions"] = ( mth_lkg_settings[x] for n in range(2)) # Measure-specific handy vars - handyvars_fmeth[x] = ecm_prep.UsefulVars( + handyvars_fmeth[x] = UsefulVars( base_dir, handyfiles_emm, cls.opts_fmeth[x]) handyvars_fmeth[x].fugitive_emissions_map = numpy.array([ ('TRE', 0., 0., @@ -1779,7 +1780,7 @@ def setUpClass(cls): "exog_hp_rates"] = ( exog_hp_rates_settings[x] for n in range(2)) # Measure-specific handy vars - handyvars_frefr[x] = ecm_prep.UsefulVars( + handyvars_frefr[x] = UsefulVars( base_dir, handyfiles_emm, cls.opts_frefr[x]) # Reset exogenous HP rate settings to test vals where applicable if handyvars_frefr[x].hp_rates is not None: @@ -18624,7 +18625,7 @@ def setUpClass(cls): opts_tsv_dummy = copy.deepcopy(opts) opts_tsv_dummy.alt_regions = "EMM" opts_tsv_dummy.tsv_metrics = ['1', '3', '1', '2', '2', '2'] - handyfiles = ecm_prep.UsefulInputFiles(opts_tsv_dummy) + handyfiles = UsefulInputFiles(opts_tsv_dummy) handyfiles.ash_emm_map = ( fp.CONVERT_DATA / "test" / "ASH_EMM_Mapping_USAMainland.txt") # Set supporting custom TSV shape test data location @@ -18637,7 +18638,7 @@ def setUpClass(cls): handyfiles.tsv_metrics_data_net_ref, \ handyfiles.tsv_metrics_data_tot_hr = (( fp.TSV_DATA / "test" / "tsv_hrs.csv") for n in range(4)) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts_tsv_dummy) + handyvars = UsefulVars(base_dir, handyfiles, opts_tsv_dummy) # Hard code aeo_years to fit test years handyvars.aeo_years = ["2009", "2010"] # Develop weekend day flags @@ -63344,8 +63345,8 @@ def setUpClass(cls): cls.time_horizons = ["2009", "2010", "2011"] # Base directory base_dir = os.getcwd() - cls.handyfiles = ecm_prep.UsefulInputFiles(cls.opts) - cls.handyvars = ecm_prep.UsefulVars( + cls.handyfiles = UsefulInputFiles(cls.opts) + cls.handyvars = UsefulVars( base_dir, cls.handyfiles, cls.opts) cls.handyvars.aeo_years = ["2009", "2010", "2011"] cls.handyvars.ccosts = numpy.array( @@ -66871,8 +66872,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measures_fail = [{ "name": "sample measure 5", "market_entry_year": None, @@ -66962,8 +66963,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measures = [{ "name": "sample measure 1", "market_entry_year": None, @@ -67519,8 +67520,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measure = { "name": "sample measure 2", "active": 1, @@ -67740,8 +67741,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measure_in = { "name": "sample measure 1", "active": 1, @@ -67860,8 +67861,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measure_in = { "name": "sample measure 1", "active": 1, @@ -67952,8 +67953,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) sample_measure_in = { "name": "sample measure 1", "active": 1, @@ -68083,8 +68084,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options opts = NullOpts().opts - cls.handyfiles = ecm_prep.UsefulInputFiles(opts) - cls.handyvars = ecm_prep.UsefulVars(base_dir, cls.handyfiles, opts) + cls.handyfiles = UsefulInputFiles(opts) + cls.handyvars = UsefulVars(base_dir, cls.handyfiles, opts) cls.ok_mktnames_out = [ "AIA_CZ1", "AIA_CZ2", "AIA_CZ3", "AIA_CZ4", "AIA_CZ5", "single family home", @@ -68225,8 +68226,8 @@ def setUpClass(cls): base_dir = os.getcwd() # Null user options/options dict opts, opts_dict = [NullOpts().opts, NullOpts().opts_dict] - handyfiles = ecm_prep.UsefulInputFiles(opts) - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, opts) + handyfiles = UsefulInputFiles(opts) + handyvars = UsefulVars(base_dir, handyfiles, opts) # Hardcode AEO years – first year in AEO time horizon (which is set # to the current year in actual runs of ecm_prep) is the # year that cost conversion assumes when no year is given in measure @@ -68848,8 +68849,8 @@ def setUpClass(cls): cls.opts_health = [copy.deepcopy(opts)] cls.opts_health[0].alt_regions = "EMM" cls.opts_health[0].health_costs = True - cls.handyfiles_aia = ecm_prep.UsefulInputFiles(cls.opts_aia) - cls.handyfiles_emm = ecm_prep.UsefulInputFiles(cls.opts_emm[0]) + cls.handyfiles_aia = UsefulInputFiles(cls.opts_aia) + cls.handyfiles_emm = UsefulInputFiles(cls.opts_emm[0]) cls.handyfiles_emm.ash_emm_map = ( fp.CONVERT_DATA / "test" / "ASH_EMM_Mapping_USAMainland.txt") cls.handyfiles_emm.tsv_metrics_data_tot_ref, \ @@ -68857,17 +68858,17 @@ def setUpClass(cls): cls.handyfiles_emm.tsv_metrics_data_net_ref, \ cls.handyfiles_emm.tsv_metrics_data_net_hr = (( fp.TSV_DATA / "test" / "tsv_hrs.csv") for n in range(4)) - cls.handyvars_aia = ecm_prep.UsefulVars( + cls.handyvars_aia = UsefulVars( cls.base_dir, cls.handyfiles_aia, cls.opts_aia) # Dummy user settings needed to generate TSV metrics data params below opts_tsv_dummy = copy.deepcopy(opts) opts_tsv_dummy.alt_regions = "EMM" opts_tsv_dummy.tsv_metrics = ['2', '2', '1', '1', '2', '2'] # Initialize TSV metrics settings for measures affecting EMM regions - cls.handyvars_emm = ecm_prep.UsefulVars( + cls.handyvars_emm = UsefulVars( cls.base_dir, cls.handyfiles_emm, opts_tsv_dummy) cls.handyvars_emm.aeo_years_summary = ["2009", "2010"] - cls.handyvars_health = ecm_prep.UsefulVars( + cls.handyvars_health = UsefulVars( cls.base_dir, cls.handyfiles_emm, cls.opts_health[0]) # Set dummy commercial equipment capacity factors cf_ones = { @@ -124544,14 +124545,14 @@ def setUpClass(cls): opts_sect_shapes_dict["sect_shapes"] = (True for n in range(2)) # Useful files for the sample package measure objects - handyfiles = ecm_prep.UsefulInputFiles(cls.opts) + handyfiles = UsefulInputFiles(cls.opts) # Version of files to use in tests of pkg. sector shapes - handyfiles_sect_shapes = ecm_prep.UsefulInputFiles( + handyfiles_sect_shapes = UsefulInputFiles( cls.opts_sect_shapes) # Useful global vars for the sample package measure objects - handyvars = ecm_prep.UsefulVars(base_dir, handyfiles, cls.opts) + handyvars = UsefulVars(base_dir, handyfiles, cls.opts) # Version of global vars to use in tests of pkg. sector shapes - handyvars_sect_shapes = ecm_prep.UsefulVars( + handyvars_sect_shapes = UsefulVars( base_dir, handyfiles_sect_shapes, cls.opts_sect_shapes) # Hard code aeo_years to fit test years handyvars.aeo_years, handyvars_sect_shapes.aeo_years = ( @@ -127940,8 +127941,8 @@ def setUpClass(cls): benefits = { "energy savings increase": None, "cost reduction": None} - cls.handyfiles = ecm_prep.UsefulInputFiles(opts) - cls.handyvars = ecm_prep.UsefulVars( + cls.handyfiles = UsefulInputFiles(opts) + cls.handyvars = UsefulVars( base_dir, cls.handyfiles, opts) sample_measindiv_dicts = [{ "name": "cleanup 1", From 6b1fc41a886a57f42c4360ed9ac10e965f91070c Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 11 Mar 2025 10:43:23 -0600 Subject: [PATCH 2/8] Make breakout_mseg an instance method in Measure. --- scout/ecm_prep.py | 892 ++++++++++++++++++++--------------------- scout/ecm_prep_vars.py | 6 - 2 files changed, 442 insertions(+), 456 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index 30ad024f5..3d15ab576 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -4107,8 +4107,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, # for current adoption scenario, and if so, prepare data if self.handyvars.full_dat_out[adopt_scheme]: # Populate detailed breakout information for measure - breakout_mseg( - self, mskeys, contrib_mseg_key, adopt_scheme, opts, + self.breakout_mseg( + mskeys, contrib_mseg_key, adopt_scheme, opts, add_stock_total, add_energy_total, add_energy_cost, add_carb_total, add_stock_total_meas, add_energy_total_eff, add_energy_total_eff_capt, @@ -8893,6 +8893,446 @@ def build_array(self, eplus_coltyp, files_to_build): return eplus_perf_array + def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, + add_stock_total, add_energy_total, add_energy_cost, + add_carb_total, add_stock_total_meas, add_energy_total_eff, + add_energy_total_eff_capt, add_energy_cost_eff, + add_carb_total_eff, add_fs_stk_eff_remain, + add_fs_energy_eff_remain, add_fs_energy_cost_eff_remain, + add_fs_carb_eff_remain): + """Record mseg contributions to breakouts by region/bldg/end use/fuel. + + Args: + contrib_mseg_key (tuple): Dictionary key information for the current + market microsegment being updated (mseg type->reg->bldg-> + fuel->end use->technology type->structure type). + adopt_scheme (string): Assumed consumer adoption scenario. + opts (object): Stores user-specified execution options. + add_stock_total (dict): Total stock associated w/ mseg. + add_energy_total (dict): Total energy associated w/ mseg. + add_energy_cost (dict): Total energy cost associated w/ mseg. + add_carb_total (dict): Total carbon emissions associated w/ mseg. + add_stock_total_meas (dict): Total measure-captured stock in mseg. + add_energy_total_eff (dict): Total mseg energy after measure adoption. + add_energy_total_eff_capt (dict): Total mseg energy specifically + associated with measure stock units (vs. baseline). + add_energy_cost_eff (dict): Total mseg energy cost after measure + adoption. + add_carb_total_eff (dict): Total mseg carbon after measure adoption. + add_fs_stk_eff_remain (dict): Portion of efficient mseg stock that is + served by base fuel after measure application (applies to fuel + switching measures) + add_fs_energy_eff_remain (dict): Portion of efficient mseg energy that + is served by base fuel after measure application (applies to fuel + switching measures) + add_fs_energy_cost_eff_remain (dict): Portion of efficient mseg energy + cost that is associated with base fuel after measure application + (applies to fuel switching measures) + add_fs_carb_eff_remain (dict): Portion of efficient mseg carbon that is + from base fuel after measure application (applies to fuel switching + measures) + + Returns: + Updated measure market breakouts by region, building type, end use, and + fuel type that reflect the influence of the current mseg being looped. + + """ + + # Using the key chain for the current microsegment, determine the output + # climate zone, building type, and end use breakout categories to which the + # current microsegment applies + + # Establish applicable climate zone breakout + for cz in self.handyvars.out_break_czones.items(): + if mskeys[1] in cz[1]: + out_cz = cz[0] + # Establish applicable building type breakout + for bldg in self.handyvars.out_break_bldgtypes.items(): + if all([x in bldg[1] for x in [ + mskeys[2], mskeys[-1]]]): + out_bldg = bldg[0] + # Establish applicable end use breakout + for eu in self.handyvars.out_break_enduses.items(): + # * Note: The 'other' microsegment end use may map to either the + # 'Refrigeration' output breakout or the 'Other' output breakout, + # depending on the technology type specified in the measure + # definition. Also note that 'supply' side heating/cooling + # microsegments map to the 'Heating (Equip.)'/'Cooling (Equip.)' end + # uses, while 'demand' side heating/cooling microsegments map to the + # 'Heating (Env.)'/'Cooling (Env.) end use, with the exception of + # 'demand' side heating/cooling microsegments that represent waste heat + # from lights - these are categorized as part of 'Lighting' end use + if mskeys[4] == "other": + if mskeys[5] == "freezers": + out_eu = "Refrigeration" + else: + out_eu = "Other" + elif mskeys[4] in eu[1]: + if (eu[0] in ["Heating (Equip.)", "Cooling (Equip.)"] and + mskeys[5] == "supply") or ( + eu[0] in ["Heating (Env.)", "Cooling (Env.)"] and + mskeys[5] == "demand") or ( + eu[0] not in ["Heating (Equip.)", "Cooling (Equip.)", + "Heating (Env.)", "Cooling (Env.)"]): + out_eu = eu[0] + elif "lighting gain" in mskeys: + out_eu = "Lighting" + + # If applicable, establish fuel type breakout (electric vs. non-electric); + # note – only applicable to end uses that are at least in part fossil-fired + if (len(self.handyvars.out_break_fuels.keys()) != 0) and ( + out_eu in self.handyvars.out_break_eus_w_fsplits): + # Flag for detailed fuel type breakout + detail = len(self.handyvars.out_break_fuels.keys()) > 2 + # Establish breakout of fuel type that is being reduced (e.g., through + # efficiency or fuel switching away from the fuel) + for f in self.handyvars.out_break_fuels.items(): + if mskeys[3] in f[1]: + # Special handling for other fuel tech., under detailed fuel + # type breakouts; this tech. may fit into multiple fuel + # categories + if detail and mskeys[3] == "other fuel": + # Assign coal/kerosene tech. + if f[0] == "Distillate/Other" and ( + mskeys[-2] is not None and any( + [x in mskeys[-2] for x in ["coal", "kerosene"]])): + out_fuel_save = f[0] + # Assign commercial other fuel to Distillate/Other + elif f[0] == "Distillate/Other" and ( + mskeys[2] in self.handyvars.in_all_map['bldg_type'][ + 'commercial']): + out_fuel_save = f[0] + # Assign wood tech. + elif f[0] == "Biomass" and ( + mskeys[-2] is not None and "wood" in mskeys[-2]): + out_fuel_save = f[0] + # All other tech. goes to propane + elif f[0] == "Propane": + out_fuel_save = f[0] + else: + out_fuel_save = f[0] + # Establish breakout of fuel type that is being added to via fuel + # switching, if applicable + if self.fuel_switch_to == "electricity" and \ + out_fuel_save != "Electric": + out_fuel_gain = "Electric" + elif self.fuel_switch_to not in [None, "electricity"] \ + and out_fuel_save == "Electric": + # Check for detailed fuel types + if detail: + for f in self.handyvars.out_break_fuels.items(): + # Special handling for other fuel tech., under detailed + # fuel type breakouts; this tech. may fit into multiple + # fuel cats. + if self.fuel_switch_to in f[1] and \ + mskeys[3] == "other fuel": + # Assign coal/kerosene tech. + if f[0] == "Distillate/Other" and ( + mskeys[-2] is not None and any([ + x in mskeys[-2] for x in [ + "coal", "kerosene"]])): + out_fuel_gain = f[0] + # Assign commercial other fuel to Distillate/Other + elif f[0] == "Distillate/Other" and ( + mskeys[2] in self.handyvars.in_all_map[ + 'bldg_type']['commercial']): + out_fuel_gain = f[0] + # Assign wood tech. + elif f[0] == "Biomass" and ( + mskeys[-2] is not None and "wood" + in mskeys[-2]): + out_fuel_gain = f[0] + # All other tech. goes to propane + elif f[0] == "Propane": + out_fuel_gain = f[0] + elif self.fuel_switch_to in f[1]: + out_fuel_gain = f[0] + else: + out_fuel_gain = "Non-Electric" + else: + out_fuel_gain = "" + else: + out_fuel_save, out_fuel_gain = ("" for n in range(2)) + + # Given the contributing microsegment's applicable climate zone, building + # type, and end use categories, add the microsegment's stock/energy/ecost/ + # carbon baseline, efficient stock/energy/ecost/carbon, and energy/ecost/ + # carbon savings values to the appropriate leaf node of the dictionary used + # to store measure output breakout information. * Note: the values in this + # dictionary will be normalized in run.py by the measure's stock/energy/ + # ecost/carbon baseline, efficient stock/energy/ecost/carbon, and stock/ + # energy/ecost/carbon savings totals (post-competition) to yield the + # fractions of measure stock, energy, carbon, and cost markets/savings that + # are attributable to each climate zone, building type, and end use that + # the measure applies to. Note that savings breakouts are not provided for + # stock data as "savings" is not meaningful in this context. + try: + # Define data indexing and reporting variables + breakout_vars = ["stock", "energy", "cost", "carbon"] + # Create a shorthand for baseline and efficient stock/energy/carbon/ + # cost data to add to the breakout dict + base_data = [add_stock_total, add_energy_total, + add_energy_cost, add_carb_total] + eff_data = [add_stock_total_meas, add_energy_total_eff, + add_energy_cost_eff, add_carb_total_eff] + + # Create a shorthand for efficient captured energy data to add to the + # breakout dict + if add_energy_total_eff_capt: + capt_e = add_energy_total_eff_capt + else: + capt_e = "" + # For a fuel switching case where the user desires that the outputs + # be split by fuel, create shorthands for any efficient-case + # stock/energy/carbon/cost that remains with the baseline fuel + if self.fuel_switch_to is not None and out_fuel_save: + eff_data_fs = [add_fs_stk_eff_remain, + add_fs_energy_eff_remain, + add_fs_energy_cost_eff_remain, + add_fs_carb_eff_remain] + # Record the efficient energy that has not yet fuel switched and + # total efficient energy for the current mseg for later use in + # packaging and/or competing measures + self.eff_fs_splt[adopt_scheme][ + str(contrib_mseg_key)] = { + "energy": [add_fs_energy_eff_remain, add_energy_total_eff], + "cost": [add_fs_energy_cost_eff_remain, + add_energy_cost_eff], + "carbon": [add_fs_carb_eff_remain, add_carb_total_eff]} + # Handle case where output breakout includes fuel type breakout or not + if out_fuel_save: + # Update results for the baseline fuel; handle case where results + # for the current region, bldg., end use, and fuel have not yet + # been initialized + try: + for yr in self.handyvars.aeo_years: + for ind, key in enumerate(breakout_vars): + self.markets[adopt_scheme]["mseg_out_break"][key][ + "baseline"][out_cz][out_bldg][out_eu][ + out_fuel_save][yr] += base_data[ind][yr] + # Efficient and savings; if there is fuel switching, + # only the portion of the efficient case results that + # have not yet switched (due to stock turnover + # limitations) remain, and savings are the delta + # between what remains unswitched in the efficient + # case and the baseline + if not out_fuel_gain: + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_save][yr] += eff_data[ind][yr] + if key == "energy" and capt_e: + self.markets[adopt_scheme]["mseg_out_break"][ + key]["efficient-captured"][out_cz][ + out_bldg][out_eu][out_fuel_save][yr] += \ + capt_e[yr] + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][ + key]["savings"][out_cz][out_bldg][out_eu][ + out_fuel_save][yr] += ( + base_data[ind][yr] - eff_data[ind][yr]) + else: + # Note that efficient-captured variable is not + # relevant for the original fuel (measure captured + # is only of the switched to fuel), and not updated + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_save][yr] += eff_data_fs[ind][yr] + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][ + key]["savings"][out_cz][out_bldg][out_eu][ + out_fuel_save][yr] += ( + base_data[ind][yr] - + eff_data_fs[ind][yr]) + except KeyError: + for ind, key in enumerate(breakout_vars): + # Baseline; add in baseline data as-is + self.markets[adopt_scheme]["mseg_out_break"][key][ + "baseline"][out_cz][out_bldg][out_eu][ + out_fuel_save] = {yr: base_data[ind][yr] for + yr in self.handyvars.aeo_years} + # Efficient and savings; if there is fuel switching, only + # the portion of the efficient case results that have not + # yet switched (due to stock turnover limitations) remain, + # and savings are the delta between what remains + # unswitched in the efficient case and the baseline + if not out_fuel_gain: + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_save] = { + yr: eff_data[ind][yr] for + yr in self.handyvars.aeo_years} + if key == "energy" and capt_e: + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient-captured"][out_cz][out_bldg][ + out_eu][out_fuel_save] = { + yr: capt_e[yr] for yr in + self.handyvars.aeo_years} + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][key][ + "savings"][out_cz][out_bldg][out_eu][ + out_fuel_save] = {yr: ( + base_data[ind][yr] - eff_data[ind][yr]) for + yr in self.handyvars.aeo_years} + else: + # Note that efficient-captured variable is not relevant + # for the original fuel measure captured is only of the + # switched to fuel), and not updated + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_save] = { + yr: eff_data_fs[ind][yr] for + yr in self.handyvars.aeo_years} + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][key][ + "savings"][out_cz][out_bldg][out_eu][ + out_fuel_save] = {yr: ( + base_data[ind][yr] - eff_data_fs[ind][yr]) + for yr in self.handyvars.aeo_years} + # In a fuel switching case, update results for the fuel being + # switched/added to + if out_fuel_gain: + # Handle case where results for the current region, bldg., + # end use, and fuel have not yet been initialized + try: + for yr in self.handyvars.aeo_years: + for ind, key in enumerate(breakout_vars): + # Note: no need to add to baseline for fuel being + # switched to, which remains zero + + # Efficient and savings; efficient case energy/ + # emissions/cost that do not remain with the + # baseline fuel are added to the switched to + # fuel and represented as negative savings for the + # switched to fuel + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][ + key]["efficient"][out_cz][out_bldg][ + out_eu][out_fuel_gain][yr] += \ + (eff_data[ind][yr] - eff_data_fs[ind][yr]) + # All captured efficient energy goes to + # switched to fuel + if key == "energy" and capt_e: + self.markets[adopt_scheme][ + "mseg_out_break"][key][ + "efficient-captured"][ + out_cz][out_bldg][out_eu][ + out_fuel_gain][yr] += capt_e[yr] + self.markets[adopt_scheme]["mseg_out_break"][ + key]["savings"][out_cz][out_bldg][out_eu][ + out_fuel_gain][yr] -= ( + eff_data[ind][yr] - + eff_data_fs[ind][yr]) + else: + self.markets[adopt_scheme]["mseg_out_break"][ + key]["efficient"][out_cz][out_bldg][ + out_eu][out_fuel_gain][yr] += \ + eff_data[ind][yr] + if key == "energy" and capt_e: + self.markets[adopt_scheme][ + "mseg_out_break"][key][ + "efficient-captured"][ + out_cz][out_bldg][out_eu][ + out_fuel_gain][yr] += capt_e[yr] + except KeyError: + for ind, key in enumerate(breakout_vars): + # Baseline for the fuel being switched to is + # initialized as zero + self.markets[adopt_scheme]["mseg_out_break"][key][ + "baseline"][out_cz][out_bldg][out_eu][ + out_fuel_gain] = {yr: 0 for yr in + self.handyvars.aeo_years} + # Efficient and savings; efficient case energy/ + # emissions/cost that do not remain with the baseline + # fuel are added to the switched to fuel and + # represented as negative savings for the switched to + # fuel + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_gain] = { + yr: (eff_data[ind][yr] - + eff_data_fs[ind][yr]) + for yr in self.handyvars.aeo_years} + # All captured efficient energy + # goes to switched to fuel + if key == "energy" and capt_e: + self.markets[adopt_scheme][ + "mseg_out_break"][key][ + "efficient-captured"][ + out_cz][out_bldg][out_eu][ + out_fuel_gain] = { + yr: capt_e[yr] for yr in + self.handyvars.aeo_years} + self.markets[adopt_scheme][ + "mseg_out_break"][key]["savings"][out_cz][ + out_bldg][out_eu][out_fuel_gain] = { + yr: -(eff_data[ind][yr] - + eff_data_fs[ind][yr]) + for yr in self.handyvars.aeo_years} + else: + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][ + out_fuel_gain] = { + yr: eff_data[ind][yr] + for yr in self.handyvars.aeo_years} + if key == "energy" and capt_e: + self.markets[adopt_scheme]["mseg_out_break"][ + key]["efficient-captured"][out_cz][ + out_bldg][out_eu][out_fuel_gain] = { + yr: capt_e[yr] + for yr in self.handyvars.aeo_years} + else: + # Handle case where results for the current region, bldg., end use, + # and fuel have not yet been initialized + try: + for yr in self.handyvars.aeo_years: + for ind, key in enumerate(breakout_vars): + self.markets[adopt_scheme]["mseg_out_break"][key][ + "baseline"][out_cz][out_bldg][out_eu][yr] += \ + base_data[ind][yr] + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu][yr] += \ + eff_data[ind][yr] + if key == "energy" and capt_e: + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient-captured"][out_cz][out_bldg][ + out_eu][yr] += capt_e[yr] + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][key][ + "savings"][out_cz][out_bldg][out_eu][yr] += ( + base_data[ind][yr] - eff_data[ind][yr]) + except KeyError: + for ind, key in enumerate(breakout_vars): + self.markets[adopt_scheme]["mseg_out_break"][key][ + "baseline"][out_cz][out_bldg][out_eu] = { + yr: base_data[ind][yr] for + yr in self.handyvars.aeo_years} + self.markets[adopt_scheme]["mseg_out_break"][key][ + "efficient"][out_cz][out_bldg][out_eu] = { + yr: eff_data[ind][yr] for + yr in self.handyvars.aeo_years} + if key == "energy" and capt_e: + self.markets[adopt_scheme][ + "mseg_out_break"][key]["efficient-captured"][ + out_cz][out_bldg][out_eu] = { + yr: capt_e[yr] for + yr in self.handyvars.aeo_years} + if key != "stock": # no stk save + self.markets[adopt_scheme]["mseg_out_break"][key][ + "savings"][out_cz][out_bldg][out_eu] = { + yr: (base_data[ind][yr] - + eff_data[ind][yr]) for + yr in self.handyvars.aeo_years} + + # Yield warning if current contributing microsegment cannot + # be mapped to an output breakout category + except KeyError: + fmt.verboseprint( + opts.verbose, + f"Baseline market key chain: '{str(mskeys)}' for ECM '{self.name}' does not map to " + "output breakout categories, thus will not be reflected in output breakout data", + "warning") + class MeasurePackage(Measure): """Set up a class representing packaged efficiency measures as objects. @@ -11627,447 +12067,6 @@ def tsv_cost_carb_yrmap(tsv_data, aeo_years): return tsv_yr_map -def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, - add_stock_total, add_energy_total, add_energy_cost, - add_carb_total, add_stock_total_meas, add_energy_total_eff, - add_energy_total_eff_capt, add_energy_cost_eff, - add_carb_total_eff, add_fs_stk_eff_remain, - add_fs_energy_eff_remain, add_fs_energy_cost_eff_remain, - add_fs_carb_eff_remain): - """Record mseg contributions to breakouts by region/bldg/end use/fuel. - - Args: - contrib_mseg_key (tuple): Dictionary key information for the current - market microsegment being updated (mseg type->reg->bldg-> - fuel->end use->technology type->structure type). - adopt_scheme (string): Assumed consumer adoption scenario. - opts (object): Stores user-specified execution options. - add_stock_total (dict): Total stock associated w/ mseg. - add_energy_total (dict): Total energy associated w/ mseg. - add_energy_cost (dict): Total energy cost associated w/ mseg. - add_carb_total (dict): Total carbon emissions associated w/ mseg. - add_stock_total_meas (dict): Total measure-captured stock in mseg. - add_energy_total_eff (dict): Total mseg energy after measure adoption. - add_energy_total_eff_capt (dict): Total mseg energy specifically - associated with measure stock units (vs. baseline). - add_energy_cost_eff (dict): Total mseg energy cost after measure - adoption. - add_carb_total_eff (dict): Total mseg carbon after measure adoption. - add_fs_stk_eff_remain (dict): Portion of efficient mseg stock that is - served by base fuel after measure application (applies to fuel - switching measures) - add_fs_energy_eff_remain (dict): Portion of efficient mseg energy that - is served by base fuel after measure application (applies to fuel - switching measures) - add_fs_energy_cost_eff_remain (dict): Portion of efficient mseg energy - cost that is associated with base fuel after measure application - (applies to fuel switching measures) - add_fs_carb_eff_remain (dict): Portion of efficient mseg carbon that is - from base fuel after measure application (applies to fuel switching - measures) - - Returns: - Updated measure market breakouts by region, building type, end use, and - fuel type that reflect the influence of the current mseg being looped. - - """ - - # Using the key chain for the current microsegment, determine the output - # climate zone, building type, and end use breakout categories to which the - # current microsegment applies - - # Establish applicable climate zone breakout - for cz in self.handyvars.out_break_czones.items(): - if mskeys[1] in cz[1]: - out_cz = cz[0] - # Establish applicable building type breakout - for bldg in self.handyvars.out_break_bldgtypes.items(): - if all([x in bldg[1] for x in [ - mskeys[2], mskeys[-1]]]): - out_bldg = bldg[0] - # Establish applicable end use breakout - for eu in self.handyvars.out_break_enduses.items(): - # * Note: The 'other' microsegment end use may map to either the - # 'Refrigeration' output breakout or the 'Other' output breakout, - # depending on the technology type specified in the measure - # definition. Also note that 'supply' side heating/cooling - # microsegments map to the 'Heating (Equip.)'/'Cooling (Equip.)' end - # uses, while 'demand' side heating/cooling microsegments map to the - # 'Heating (Env.)'/'Cooling (Env.) end use, with the exception of - # 'demand' side heating/cooling microsegments that represent waste heat - # from lights - these are categorized as part of 'Lighting' end use - if mskeys[4] == "other": - if mskeys[5] == "freezers": - out_eu = "Refrigeration" - else: - out_eu = "Other" - elif mskeys[4] in eu[1]: - if (eu[0] in ["Heating (Equip.)", "Cooling (Equip.)"] and - mskeys[5] == "supply") or ( - eu[0] in ["Heating (Env.)", "Cooling (Env.)"] and - mskeys[5] == "demand") or ( - eu[0] not in ["Heating (Equip.)", "Cooling (Equip.)", - "Heating (Env.)", "Cooling (Env.)"]): - out_eu = eu[0] - elif "lighting gain" in mskeys: - out_eu = "Lighting" - - # If applicable, establish fuel type breakout (electric vs. non-electric); - # note – only applicable to end uses that are at least in part fossil-fired - if (len(self.handyvars.out_break_fuels.keys()) != 0) and ( - out_eu in self.handyvars.out_break_eus_w_fsplits): - # Flag for detailed fuel type breakout - detail = len(self.handyvars.out_break_fuels.keys()) > 2 - # Establish breakout of fuel type that is being reduced (e.g., through - # efficiency or fuel switching away from the fuel) - for f in self.handyvars.out_break_fuels.items(): - if mskeys[3] in f[1]: - # Special handling for other fuel tech., under detailed fuel - # type breakouts; this tech. may fit into multiple fuel - # categories - if detail and mskeys[3] == "other fuel": - # Assign coal/kerosene tech. - if f[0] == "Distillate/Other" and ( - mskeys[-2] is not None and any( - [x in mskeys[-2] for x in ["coal", "kerosene"]])): - out_fuel_save = f[0] - # Assign commercial other fuel to Distillate/Other - elif f[0] == "Distillate/Other" and ( - mskeys[2] in self.handyvars.in_all_map['bldg_type'][ - 'commercial']): - out_fuel_save = f[0] - # Assign wood tech. - elif f[0] == "Biomass" and ( - mskeys[-2] is not None and "wood" in mskeys[-2]): - out_fuel_save = f[0] - # All other tech. goes to propane - elif f[0] == "Propane": - out_fuel_save = f[0] - else: - out_fuel_save = f[0] - # Establish breakout of fuel type that is being added to via fuel - # switching, if applicable - if self.fuel_switch_to == "electricity" and \ - out_fuel_save != "Electric": - out_fuel_gain = "Electric" - elif self.fuel_switch_to not in [None, "electricity"] \ - and out_fuel_save == "Electric": - # Check for detailed fuel types - if detail: - for f in self.handyvars.out_break_fuels.items(): - # Special handling for other fuel tech., under detailed - # fuel type breakouts; this tech. may fit into multiple - # fuel cats. - if self.fuel_switch_to in f[1] and \ - mskeys[3] == "other fuel": - # Assign coal/kerosene tech. - if f[0] == "Distillate/Other" and ( - mskeys[-2] is not None and any([ - x in mskeys[-2] for x in [ - "coal", "kerosene"]])): - out_fuel_gain = f[0] - # Assign commercial other fuel to Distillate/Other - elif f[0] == "Distillate/Other" and ( - mskeys[2] in self.handyvars.in_all_map[ - 'bldg_type']['commercial']): - out_fuel_gain = f[0] - # Assign wood tech. - elif f[0] == "Biomass" and ( - mskeys[-2] is not None and "wood" - in mskeys[-2]): - out_fuel_gain = f[0] - # All other tech. goes to propane - elif f[0] == "Propane": - out_fuel_gain = f[0] - elif self.fuel_switch_to in f[1]: - out_fuel_gain = f[0] - else: - out_fuel_gain = "Non-Electric" - else: - out_fuel_gain = "" - else: - out_fuel_save, out_fuel_gain = ("" for n in range(2)) - - # Given the contributing microsegment's applicable climate zone, building - # type, and end use categories, add the microsegment's stock/energy/ecost/ - # carbon baseline, efficient stock/energy/ecost/carbon, and energy/ecost/ - # carbon savings values to the appropriate leaf node of the dictionary used - # to store measure output breakout information. * Note: the values in this - # dictionary will be normalized in run.py by the measure's stock/energy/ - # ecost/carbon baseline, efficient stock/energy/ecost/carbon, and stock/ - # energy/ecost/carbon savings totals (post-competition) to yield the - # fractions of measure stock, energy, carbon, and cost markets/savings that - # are attributable to each climate zone, building type, and end use that - # the measure applies to. Note that savings breakouts are not provided for - # stock data as "savings" is not meaningful in this context. - try: - # Define data indexing and reporting variables - breakout_vars = ["stock", "energy", "cost", "carbon"] - # Create a shorthand for baseline and efficient stock/energy/carbon/ - # cost data to add to the breakout dict - base_data = [add_stock_total, add_energy_total, - add_energy_cost, add_carb_total] - eff_data = [add_stock_total_meas, add_energy_total_eff, - add_energy_cost_eff, add_carb_total_eff] - - # Create a shorthand for efficient captured energy data to add to the - # breakout dict - if add_energy_total_eff_capt: - capt_e = add_energy_total_eff_capt - else: - capt_e = "" - # For a fuel switching case where the user desires that the outputs - # be split by fuel, create shorthands for any efficient-case - # stock/energy/carbon/cost that remains with the baseline fuel - if self.fuel_switch_to is not None and out_fuel_save: - eff_data_fs = [add_fs_stk_eff_remain, - add_fs_energy_eff_remain, - add_fs_energy_cost_eff_remain, - add_fs_carb_eff_remain] - # Record the efficient energy that has not yet fuel switched and - # total efficient energy for the current mseg for later use in - # packaging and/or competing measures - self.eff_fs_splt[adopt_scheme][ - str(contrib_mseg_key)] = { - "energy": [add_fs_energy_eff_remain, add_energy_total_eff], - "cost": [add_fs_energy_cost_eff_remain, - add_energy_cost_eff], - "carbon": [add_fs_carb_eff_remain, add_carb_total_eff]} - # Handle case where output breakout includes fuel type breakout or not - if out_fuel_save: - # Update results for the baseline fuel; handle case where results - # for the current region, bldg., end use, and fuel have not yet - # been initialized - try: - for yr in self.handyvars.aeo_years: - for ind, key in enumerate(breakout_vars): - self.markets[adopt_scheme]["mseg_out_break"][key][ - "baseline"][out_cz][out_bldg][out_eu][ - out_fuel_save][yr] += base_data[ind][yr] - # Efficient and savings; if there is fuel switching, - # only the portion of the efficient case results that - # have not yet switched (due to stock turnover - # limitations) remain, and savings are the delta - # between what remains unswitched in the efficient - # case and the baseline - if not out_fuel_gain: - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_save][yr] += eff_data[ind][yr] - if key == "energy" and capt_e: - self.markets[adopt_scheme]["mseg_out_break"][ - key]["efficient-captured"][out_cz][ - out_bldg][out_eu][out_fuel_save][yr] += \ - capt_e[yr] - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][ - key]["savings"][out_cz][out_bldg][out_eu][ - out_fuel_save][yr] += ( - base_data[ind][yr] - eff_data[ind][yr]) - else: - # Note that efficient-captured variable is not - # relevant for the original fuel (measure captured - # is only of the switched to fuel), and not updated - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_save][yr] += eff_data_fs[ind][yr] - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][ - key]["savings"][out_cz][out_bldg][out_eu][ - out_fuel_save][yr] += ( - base_data[ind][yr] - - eff_data_fs[ind][yr]) - except KeyError: - for ind, key in enumerate(breakout_vars): - # Baseline; add in baseline data as-is - self.markets[adopt_scheme]["mseg_out_break"][key][ - "baseline"][out_cz][out_bldg][out_eu][ - out_fuel_save] = {yr: base_data[ind][yr] for - yr in self.handyvars.aeo_years} - # Efficient and savings; if there is fuel switching, only - # the portion of the efficient case results that have not - # yet switched (due to stock turnover limitations) remain, - # and savings are the delta between what remains - # unswitched in the efficient case and the baseline - if not out_fuel_gain: - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_save] = { - yr: eff_data[ind][yr] for - yr in self.handyvars.aeo_years} - if key == "energy" and capt_e: - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient-captured"][out_cz][out_bldg][ - out_eu][out_fuel_save] = { - yr: capt_e[yr] for yr in - self.handyvars.aeo_years} - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][key][ - "savings"][out_cz][out_bldg][out_eu][ - out_fuel_save] = {yr: ( - base_data[ind][yr] - eff_data[ind][yr]) for - yr in self.handyvars.aeo_years} - else: - # Note that efficient-captured variable is not relevant - # for the original fuel measure captured is only of the - # switched to fuel), and not updated - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_save] = { - yr: eff_data_fs[ind][yr] for - yr in self.handyvars.aeo_years} - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][key][ - "savings"][out_cz][out_bldg][out_eu][ - out_fuel_save] = {yr: ( - base_data[ind][yr] - eff_data_fs[ind][yr]) - for yr in self.handyvars.aeo_years} - # In a fuel switching case, update results for the fuel being - # switched/added to - if out_fuel_gain: - # Handle case where results for the current region, bldg., - # end use, and fuel have not yet been initialized - try: - for yr in self.handyvars.aeo_years: - for ind, key in enumerate(breakout_vars): - # Note: no need to add to baseline for fuel being - # switched to, which remains zero - - # Efficient and savings; efficient case energy/ - # emissions/cost that do not remain with the - # baseline fuel are added to the switched to - # fuel and represented as negative savings for the - # switched to fuel - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][ - key]["efficient"][out_cz][out_bldg][ - out_eu][out_fuel_gain][yr] += \ - (eff_data[ind][yr] - eff_data_fs[ind][yr]) - # All captured efficient energy goes to - # switched to fuel - if key == "energy" and capt_e: - self.markets[adopt_scheme][ - "mseg_out_break"][key][ - "efficient-captured"][ - out_cz][out_bldg][out_eu][ - out_fuel_gain][yr] += capt_e[yr] - self.markets[adopt_scheme]["mseg_out_break"][ - key]["savings"][out_cz][out_bldg][out_eu][ - out_fuel_gain][yr] -= ( - eff_data[ind][yr] - - eff_data_fs[ind][yr]) - else: - self.markets[adopt_scheme]["mseg_out_break"][ - key]["efficient"][out_cz][out_bldg][ - out_eu][out_fuel_gain][yr] += \ - eff_data[ind][yr] - if key == "energy" and capt_e: - self.markets[adopt_scheme][ - "mseg_out_break"][key][ - "efficient-captured"][ - out_cz][out_bldg][out_eu][ - out_fuel_gain][yr] += capt_e[yr] - except KeyError: - for ind, key in enumerate(breakout_vars): - # Baseline for the fuel being switched to is - # initialized as zero - self.markets[adopt_scheme]["mseg_out_break"][key][ - "baseline"][out_cz][out_bldg][out_eu][ - out_fuel_gain] = {yr: 0 for yr in - self.handyvars.aeo_years} - # Efficient and savings; efficient case energy/ - # emissions/cost that do not remain with the baseline - # fuel are added to the switched to fuel and - # represented as negative savings for the switched to - # fuel - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_gain] = { - yr: (eff_data[ind][yr] - - eff_data_fs[ind][yr]) - for yr in self.handyvars.aeo_years} - # All captured efficient energy - # goes to switched to fuel - if key == "energy" and capt_e: - self.markets[adopt_scheme][ - "mseg_out_break"][key][ - "efficient-captured"][ - out_cz][out_bldg][out_eu][ - out_fuel_gain] = { - yr: capt_e[yr] for yr in - self.handyvars.aeo_years} - self.markets[adopt_scheme][ - "mseg_out_break"][key]["savings"][out_cz][ - out_bldg][out_eu][out_fuel_gain] = { - yr: -(eff_data[ind][yr] - - eff_data_fs[ind][yr]) - for yr in self.handyvars.aeo_years} - else: - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][ - out_fuel_gain] = { - yr: eff_data[ind][yr] - for yr in self.handyvars.aeo_years} - if key == "energy" and capt_e: - self.markets[adopt_scheme]["mseg_out_break"][ - key]["efficient-captured"][out_cz][ - out_bldg][out_eu][out_fuel_gain] = { - yr: capt_e[yr] - for yr in self.handyvars.aeo_years} - else: - # Handle case where results for the current region, bldg., end use, - # and fuel have not yet been initialized - try: - for yr in self.handyvars.aeo_years: - for ind, key in enumerate(breakout_vars): - self.markets[adopt_scheme]["mseg_out_break"][key][ - "baseline"][out_cz][out_bldg][out_eu][yr] += \ - base_data[ind][yr] - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu][yr] += \ - eff_data[ind][yr] - if key == "energy" and capt_e: - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient-captured"][out_cz][out_bldg][ - out_eu][yr] += capt_e[yr] - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][key][ - "savings"][out_cz][out_bldg][out_eu][yr] += ( - base_data[ind][yr] - eff_data[ind][yr]) - except KeyError: - for ind, key in enumerate(breakout_vars): - self.markets[adopt_scheme]["mseg_out_break"][key][ - "baseline"][out_cz][out_bldg][out_eu] = { - yr: base_data[ind][yr] for - yr in self.handyvars.aeo_years} - self.markets[adopt_scheme]["mseg_out_break"][key][ - "efficient"][out_cz][out_bldg][out_eu] = { - yr: eff_data[ind][yr] for - yr in self.handyvars.aeo_years} - if key == "energy" and capt_e: - self.markets[adopt_scheme][ - "mseg_out_break"][key]["efficient-captured"][ - out_cz][out_bldg][out_eu] = { - yr: capt_e[yr] for - yr in self.handyvars.aeo_years} - if key != "stock": # no stk save - self.markets[adopt_scheme]["mseg_out_break"][key][ - "savings"][out_cz][out_bldg][out_eu] = { - yr: (base_data[ind][yr] - - eff_data[ind][yr]) for - yr in self.handyvars.aeo_years} - - # Yield warning if current contributing microsegment cannot - # be mapped to an output breakout category - except KeyError: - fmt.verboseprint( - opts.verbose, - f"Baseline market key chain: '{str(mskeys)}' for ECM '{self.name}' does not map to " - "output breakout categories, thus will not be reflected in output breakout data", - "warning") - - def downselect_packages(existing_pkgs: list[dict], pkg_subset: list) -> list: if "*" in pkg_subset: return existing_pkgs @@ -12184,13 +12183,6 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Instantiate useful input files object handyfiles = UsefulInputFiles(opts) - # UNCOMMENT WITH ISSUE 188 - # # Ensure that all AEO-based JSON data are drawn from the same AEO version - # if len(numpy.unique([splitext(x)[0][-4:] for x in [ - # handyfiles.msegs_in, handyfiles.msegs_cpl_in, - # handyfiles.metadata]])) > 1: - # raise ValueError("Inconsistent AEO version used across input files") - # Instantiate useful variables object handyvars = UsefulVars(base_dir, handyfiles, opts) diff --git a/scout/ecm_prep_vars.py b/scout/ecm_prep_vars.py index c96adc101..a85a5fff3 100644 --- a/scout/ecm_prep_vars.py +++ b/scout/ecm_prep_vars.py @@ -1562,10 +1562,6 @@ class UsefulInputFiles(object): def __init__(self, opts): if opts.alt_regions == 'AIA': - # UNCOMMENT WITH ISSUE 188 - # self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_cz_2017.json" - # UNCOMMENT WITH ISSUE 188 - # self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_cz_2017.json" self.msegs_in = fp.STOCK_ENERGY / "mseg_res_com_cz.json" self.msegs_cpl_in = fp.STOCK_ENERGY / "cpl_res_com_cz.gz" self.iecc_reg_map = fp.CONVERT_DATA / "geo_map" / "IECC_AIA_ColSums.txt" @@ -1595,8 +1591,6 @@ def __init__(self, opts): self.set_decarb_grid_vars(opts) self.metadata = fp.METADATA_PATH self.glob_vars = fp.GENERATED / "glob_run_vars.json" - # UNCOMMENT WITH ISSUE 188 - # self.metadata = "metadata_2017.json" self.cost_convert_in = fp.CONVERT_DATA / "ecm_cost_convert.json" self.cap_facts = fp.CONVERT_DATA / "cap_facts.json" self.cbecs_sf_byvint = fp.CONVERT_DATA / "cbecs_sf_byvintage.json" From 975ceaa004a7348489650be0065d5b7f060252b3 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 11 Mar 2025 15:51:33 -0600 Subject: [PATCH 3/8] Further encapsulate ecm_prep methods; remove unused EPlusGlobals class. --- scout/ecm_prep.py | 126 +++++++++++++---------------- scout/ecm_prep_vars.py | 178 ----------------------------------------- tests/ecm_prep_test.py | 69 +--------------- 3 files changed, 58 insertions(+), 315 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index 3d15ab576..fab622052 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -50,6 +50,35 @@ def configure_ecm_prep_logger(opts): class Utils: + """Shared methods used throughout ecm_prep.py""" + + @staticmethod + def initialize_run_setup(input_files: UsefulInputFiles) -> dict: + """Reads in analysis engine setup file, run_setup.json, and initializes values. If the file + exists and has measures set to 'active', those will be moved to 'inactive'. If the file + does not exist, return a dictionary with empty 'active' and 'inactive' lists. + + Args: + input_files (UsefulInputFiles): UsefulInputFiles instance + + Returns: + dict: run_setup data with active and inactive lists + """ + try: + am = open(input_files.run_setup, 'r') + try: + run_setup = json.load(am, object_pairs_hook=OrderedDict) + except ValueError as e: + raise ValueError( + f"Error reading in '{input_files.run_setup}': {str(e)}") from None + am.close() + # Initialize all measures as inactive + run_setup = Utils.update_active_measures(run_setup, to_inactive=run_setup["active"]) + except FileNotFoundError: + run_setup = {"active": [], "inactive": [], "skipped": []} + + return run_setup + @staticmethod def update_active_measures(run_setup: dict, to_active: list = [], @@ -94,6 +123,30 @@ def update_active_measures(run_setup: dict, return run_setup + @staticmethod + def prep_error(meas_name, handyvars, handyfiles): + """Prepare and write out error messages for skipped measures/packages. + + Args: + meas_name (str): Measure or package name. + handyvars (object): Global variables of use across Measure methods. + handyfiles (object): Input files of use across Measure methods. + """ + # # Complete the update to the console for each measure being processed + # print("Error") + # Pull full error traceback + err_dets = traceback.format_exc() + # Construct error message to write out + err_msg = ( + "\nECM '" + meas_name + "' produced the following exception that " + "prevented its preparation:\n" + str(err_dets) + "\n") + # Add ECM to skipped list + handyvars.skipped_ecms.append(meas_name) + # Print error message if in verbose mode + # fmt.verboseprint(opts.verbose, err_msg, "error") + # # Log error message to file (see ./generated) + logger.error(err_msg) + class Measure(object): """Set up a class representing efficiency measures as objects. @@ -11714,20 +11767,6 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, "undefined" in m.energy_efficiency.keys() and m.energy_efficiency["undefined"] == "From EnergyPlus") for m in meas_update_objs]): - # NOTE: Comment out operations related to the import of ECM performance - # data from EnergyPlus and yield an error until these operations are - # fully supported in the future - - # Set default directory for EnergyPlus simulation output files - # eplus_dir = fp.ECM_DEF / "energyplus_data" - # # Set EnergyPlus global variables - # handyeplusvars = EPlusGlobals(eplus_dir, cbecs_sf_byvint) - # # Fill in EnergyPlus-based measure performance information - # [m.fill_eplus( - # msegs, eplus_dir, handyeplusvars.eplus_coltypes, - # handyeplusvars.eplus_files, handyeplusvars.eplus_vintage_weights, - # handyeplusvars.eplus_basecols) for m in meas_update_objs - # if 'EnergyPlus file' in m.energy_efficiency.keys()] raise ValueError( 'One or more ECMs require EnergyPlus data for ECM performance; ' 'EnergyPlus-based ECM performance data are currently unsupported.') @@ -11744,7 +11783,7 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, # are valid before attempting to retrieve data on this baseline market m.check_meas_inputs() except Exception: - prep_error(m.name, handyvars, handyfiles) + Utils.prep_error(m.name, handyvars, handyfiles) # Add measure index to removal list remove_inds.append(m_ind) @@ -11756,7 +11795,7 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, msegs, msegs_cpl, convert_data, tsv_data, opts, ctrb_ms_pkg_prep, tsv_data_nonfs) except Exception: - prep_error(m.name, handyvars, handyfiles) + Utils.prep_error(m.name, handyvars, handyfiles) # Add measure index to removal list remove_inds.append(m_ind) @@ -11892,35 +11931,11 @@ def prepare_packages(packages, meas_update_objs, meas_summary, if packaged_measure is not False: meas_update_objs.append(packaged_measure) except Exception: - prep_error(p["name"], handyvars, handyfiles) + Utils.prep_error(p["name"], handyvars, handyfiles) return meas_update_objs -def prep_error(meas_name, handyvars, handyfiles): - """Prepare and write out error messages for skipped measures/packages. - - Args: - meas_name (str): Measure or package name. - handyvars (object): Global variables of use across Measure methods. - handyfiles (object): Input files of use across Measure methods. - """ - # # Complete the update to the console for each measure being processed - # print("Error") - # Pull full error traceback - err_dets = traceback.format_exc() - # Construct error message to write out - err_msg = ( - "\nECM '" + meas_name + "' produced the following exception that " - "prevented its preparation:\n" + str(err_dets) + "\n") - # Add ECM to skipped list - handyvars.skipped_ecms.append(meas_name) - # Print error message if in verbose mode - # fmt.verboseprint(opts.verbose, err_msg, "error") - # # Log error message to file (see ./generated) - logger.error(err_msg) - - def split_clean_data(meas_prepped_objs, full_dat_out): """Reorganize and remove data from input Measure objects. @@ -12134,33 +12149,6 @@ def filter_invalid_packages(packages: list[dict], return filtered_packages, invalid_pkgs -def initialize_run_setup(input_files: UsefulInputFiles) -> dict: - """Reads in analysis engine setup file, run_setup.json, and initializes values. If the file - exists and has measures set to 'active', those will be moved to 'inactive'. If the file - does not exist, return a dictionary with empty 'active' and 'inactive' lists. - - Args: - input_files (UsefulInputFiles): UsefulInputFiles instance - - Returns: - dict: run_setup data with active and inactive lists - """ - try: - am = open(input_files.run_setup, 'r') - try: - run_setup = json.load(am, object_pairs_hook=OrderedDict) - except ValueError as e: - raise ValueError( - f"Error reading in '{input_files.run_setup}': {str(e)}") from None - am.close() - # Initialize all measures as inactive - run_setup = Utils.update_active_measures(run_setup, to_inactive=run_setup["active"]) - except FileNotFoundError: - run_setup = {"active": [], "inactive": [], "skipped": []} - - return run_setup - - def main(opts: argparse.NameSpace): # noqa: F821 """Import and prepare measure attributes for analysis engine. @@ -12564,7 +12552,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Write initial data for run_setup.json # Import analysis engine setup file - run_setup = initialize_run_setup(handyfiles) + run_setup = Utils.initialize_run_setup(handyfiles) # Set contributing ECMs as inactive in run_setup and throw warning, set all others as active ctrb_ms = [ecm for pkg in meas_toprep_package_init for ecm in pkg["contributing_ECMs"]] diff --git a/scout/ecm_prep_vars.py b/scout/ecm_prep_vars.py index a85a5fff3..590e95fdc 100644 --- a/scout/ecm_prep_vars.py +++ b/scout/ecm_prep_vars.py @@ -2,7 +2,6 @@ import argparse import itertools import numpy -import re from datetime import datetime from collections import OrderedDict from scout.utils import JsonIO @@ -1790,180 +1789,3 @@ def __init__(self): '90.1-2010': [2010, 2012], 'DOE Ref 1980-2004': [1980, 2003], 'DOE Ref Pre-1980': [0, 1979]}} - - -class EPlusGlobals(object): - """Class of global variables used in parsing EnergyPlus results file. - - Attributes: - cbecs_sh (xlrd sheet object): CBECS square footages Excel sheet. - vintage_sf (dict): Summary of CBECS square footages by vintage. - eplus_coltypes (list): Expected EnergyPlus variable data types. - eplus_basecols (list): Variable columns that should never be removed. - eplus_perf_files (list): EnergyPlus simulation output file names. - eplus_vintages (list): EnergyPlus building vintage types. - eplus_vintage_weights (dicts): Square-footage-based weighting factors - for EnergyPlus vintages. - """ - - def __init__(self, eplus_dir, cbecs_sf_byvint): - # Set building vintage square footage data from CBECS - self.vintage_sf = cbecs_sf_byvint - self.eplus_coltypes = [ - ('building_type', '= handydicts.structure_type[ - 'retrofit'][k][0] and \ - cbecs_yr < handydicts.structure_type[ - 'retrofit'][k][1]: - eplus_vintage_weights[k] += self.vintage_sf[k2] - total_retro_sf += self.vintage_sf[k2] - - # Run through all EnergyPlus vintage weights, normalizing the - # square footage-based weights for each 'retrofit' vintage to the - # total square footage across all 'retrofit' vintage categories - for k in eplus_vintage_weights.keys(): - # If running through the 'new' EnergyPlus vintage bin, register - # the value of its weight (should be 1) - if k == handydicts.structure_type['new']: - new_weight_sum = eplus_vintage_weights[k] - # If running through a 'retrofit' EnergyPlus vintage bin, - # normalize the square footage for that vintage by total - # square footages across 'retrofit' vintages to arrive at the - # final weight for that EnergyPlus vintage - else: - eplus_vintage_weights[k] /= total_retro_sf - retro_weight_sum += eplus_vintage_weights[k] - - # Check that the 'new' EnergyPlus vintage weight equals 1 and that - # all 'retrofit' EnergyPlus vintage weights sum to 1 - if new_weight_sum != 1: - raise ValueError("Incorrect new vintage weight total when " - "instantiating 'EPlusGlobals' object") - elif retro_weight_sum != 1: - raise ValueError("Incorrect retrofit vintage weight total when" - "instantiating 'EPlusGlobals' object") - - else: - raise KeyError( - "Unexpected EnergyPlus vintage(s) when instantiating " - "'EPlusGlobals' object; " - "check EnergyPlus vintage assumptions in structure_type " - "attribute of 'EPlusMapDict' object") - - return eplus_vintage_weights diff --git a/tests/ecm_prep_test.py b/tests/ecm_prep_test.py index 4afdc818e..c105cc661 100644 --- a/tests/ecm_prep_test.py +++ b/tests/ecm_prep_test.py @@ -4,7 +4,7 @@ # Import code to be tested from scout import ecm_prep -from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles, EPlusGlobals +from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles from scout.config import FilePaths as fp from scout.ecm_prep_args import ecm_args # Import needed packages @@ -143,73 +143,6 @@ def __init__(self): self.opts_dict = vars(self.opts) -class EPlusGlobalsTest(unittest.TestCase, CommonMethods): - """Test 'find_vintage_weights' function. - - Ensure building vintage square footages are read in properly from a - cbecs data file and that the proper weights are derived for mapping - EnergyPlus building vintages to Scout's 'new' and 'retrofit' building - structure types. - - Attributes: - cbecs_sf_byvint (dict): Commercial square footage by vintage data. - eplus_globals_ok (object): EPlusGlobals object with square footage and - vintage weights attributes to test against expected outputs. - eplus_failpath (string): Path to invalid EnergyPlus simulation data - file that should cause EPlusGlobals object instantiation to fail. - ok_out_weights (dict): Correct vintage weights output for - 'find_vintage_weights'function given valid inputs. - """ - - @classmethod - def setUpClass(cls): - """Define variables for use across all class functions.""" - cls.cbecs_sf_byvint = { - '2004 to 2007': 6524.0, '1960 to 1969': 10362.0, - '1946 to 1959': 7381.0, '1970 to 1979': 10846.0, - '1990 to 1999': 13803.0, '2000 to 2003': 7215.0, - 'Before 1920': 3980.0, '2008 to 2012': 5726.0, - '1920 to 1945': 6020.0, '1980 to 1989': 15185.0} - cls.eplus_globals_ok = EPlusGlobals( - fp.ECM_DEF / "energyplus_data" / "energyplus_test_ok", - cls.cbecs_sf_byvint) - cls.eplus_failpath = fp.ECM_DEF / "energyplus_data" / "energyplus_test_fail" - cls.ok_out_weights = { - 'DOE Ref 1980-2004': 0.42, '90.1-2004': 0.07, - '90.1-2010': 0.07, 'DOE Ref Pre-1980': 0.44, - '90.1-2013': 1} - - def test_vintageweights(self): - """Test find_vintage_weights function given valid inputs. - - Note: - Ensure EnergyPlus building vintage type data are correctly weighted - by their square footages (derived from CBECs data). - - Raises: - AssertionError: If function yields unexpected results. - """ - self.dict_check( - self.eplus_globals_ok.find_vintage_weights(), - self.ok_out_weights) - - # Test that an error is raised when unexpected eplus vintages are present - def test_vintageweights_fail(self): - """Test find_vintage_weights function given invalid inputs. - - Note: - Ensure that KeyError is raised when an unexpected EnergyPlus - building vintage is present. - - Raises: - AssertionError: If KeyError is not raised. - """ - with self.assertRaises(KeyError): - EPlusGlobals( - self.eplus_failpath, - self.cbecs_sf_byvint).find_vintage_weights() - - class EPlusUpdateTest(unittest.TestCase, CommonMethods): """Test the 'fill_eplus' function and its supporting functions. From daa945512ffead3fa43b0a4b25d6d663b8edad8c Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Tue, 18 Mar 2025 17:31:09 -0600 Subject: [PATCH 4/8] New class ECMPrep, store all ecm_prep functions in classes --- scout/ecm_prep.py | 949 +++++++++++++++++++++-------------------- scout/run_batch.py | 8 +- tests/ecm_prep_test.py | 109 +++-- 3 files changed, 542 insertions(+), 524 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index fab622052..738abd455 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -28,30 +28,30 @@ logger = logging.getLogger(__name__) -def configure_ecm_prep_logger(opts): - # Set file name for prep error logs using current date and time - err_f_name = fp.GENERATED / ("log_ecm_prep_" + time.strftime("%Y%m%d-%H%M%S") + ".txt") - # Ensure root logger is set up - LogConfig.configure_logging() - logger.handlers.clear() # Remove existing handlers - filehandler = logging.FileHandler(err_f_name, mode='a', delay=True) - # Set new handler to match root formatter - root_format = logging.getLogger().handlers[0].formatter - filehandler.setFormatter(root_format) - logger.addHandler(filehandler) - - # Write logger to console - console_handler = logging.StreamHandler() - console_handler.setFormatter(root_format) - logger.addHandler(console_handler) - - # Disable propagation to root logger - logger.propagate = False - - -class Utils: +class ECMUtils: """Shared methods used throughout ecm_prep.py""" + @staticmethod + def configure_ecm_prep_logger(): + # Set file name for prep error logs using current date and time + err_f_name = fp.GENERATED / ("log_ecm_prep_" + time.strftime("%Y%m%d-%H%M%S") + ".txt") + # Ensure root logger is set up + LogConfig.configure_logging() + logger.handlers.clear() # Remove existing handlers + filehandler = logging.FileHandler(err_f_name, mode='a', delay=True) + # Set new handler to match root formatter + root_format = logging.getLogger().handlers[0].formatter + filehandler.setFormatter(root_format) + logger.addHandler(filehandler) + + # Write logger to console + console_handler = logging.StreamHandler() + console_handler.setFormatter(root_format) + logger.addHandler(console_handler) + + # Disable propagation to root logger + logger.propagate = False + @staticmethod def initialize_run_setup(input_files: UsefulInputFiles) -> dict: """Reads in analysis engine setup file, run_setup.json, and initializes values. If the file @@ -73,7 +73,8 @@ def initialize_run_setup(input_files: UsefulInputFiles) -> dict: f"Error reading in '{input_files.run_setup}': {str(e)}") from None am.close() # Initialize all measures as inactive - run_setup = Utils.update_active_measures(run_setup, to_inactive=run_setup["active"]) + run_setup = ECMUtils.update_active_measures(run_setup, + to_inactive=run_setup["active"]) except FileNotFoundError: run_setup = {"active": [], "inactive": [], "skipped": []} @@ -133,13 +134,11 @@ def prep_error(meas_name, handyvars, handyfiles): handyfiles (object): Input files of use across Measure methods. """ # # Complete the update to the console for each measure being processed - # print("Error") # Pull full error traceback err_dets = traceback.format_exc() # Construct error message to write out - err_msg = ( - "\nECM '" + meas_name + "' produced the following exception that " - "prevented its preparation:\n" + str(err_dets) + "\n") + err_msg = (f"\nECM '{meas_name}' produced the following exception that prevented its " + f"preperation: \n{str(err_dets)}\n") # Add ECM to skipped list handyvars.skipped_ecms.append(meas_name) # Print error message if in verbose mode @@ -147,6 +146,107 @@ def prep_error(meas_name, handyvars, handyfiles): # # Log error message to file (see ./generated) logger.error(err_msg) + @staticmethod + def downselect_packages(existing_pkgs: list[dict], pkg_subset: list) -> list: + if "*" in pkg_subset: + return existing_pkgs + downselected_pkgs = [pkg for pkg in existing_pkgs if pkg["name"] in pkg_subset] + + return downselected_pkgs + + @staticmethod + def retrieve_valid_ecms(packages: list, + opts: argparse.NameSpace, # noqa: F821 + handyfiles: UsefulInputFiles) -> list: + """Determine full list of individual measure JSON names that 1) contribute to selected + packages in opts.ecm_packages, or 2) are included in opts.ecm_files, and 3) exist in the + ecm definitions directory (opts.ecm_directory) + + Args: + packages (list): List of valid packages + opts (argparse.NameSpace): object storing user responses + handyfiles (UsefulInputFiles): object storing input filepaths + + Returns: + list: filtered list of ECMs that meet the criteria above + """ + + contributing_ecms = { + ecm for pkg in packages for ecm in pkg["contributing_ECMs"]} + opts.ecm_files.extend([ecm for ecm in contributing_ecms if ecm not in opts.ecm_files]) + valid_ecms = [ + x for x in handyfiles.indiv_ecms.iterdir() if x.suffix == ".json" and + 'package_ecms' not in x.name and x.stem in opts.ecm_files] + + return valid_ecms + + @staticmethod + def filter_invalid_packages(packages: list[dict], + ecms: list, + opts: argparse.Namespace) -> tuple[list[dict], list]: + """Identify and filter packages whose ECMs are not all present in the individual ECM set + + Args: + packages (list[dict]): List of packages imported from package_ecms.json + ecms (list): List of ECM definitions file names + opts (argparse.Namespace): argparse object containing the argument attributes + + Returns: + filtered_packages (list[dict]): Packages list with invalid packages filtered out + invalid_pkgs (list): List of invalid packages + """ + + invalid_pkgs = [pkg["name"] for pkg in packages if not + set(pkg["contributing_ECMs"]).issubset(set(ecms))] + filtered_packages = [pkg for pkg in packages if pkg["name"] not in invalid_pkgs] + + # Trigger warning message regarding screening of packages + package_opt_txt = "" + if opts.ecm_packages is not None: + package_opt_txt = "specified with the ecm_packages argument " + if invalid_pkgs: + invalid_pkgs_txt = fmt.format_console_list(invalid_pkgs) + msg = (f"WARNING: Package(s) in package_ecms.json {package_opt_txt}have contributing" + " ECMs that are not present among ECM definitions. The following packages will" + f" not be executed: \n{''.join(invalid_pkgs_txt)}") + warnings.warn(msg) + + return filtered_packages, invalid_pkgs + + @staticmethod + def tsv_cost_carb_yrmap(tsv_data, aeo_years): + """Map 8760 TSV cost/carbon data years to AEO years. + + Args: + tsv_data: TSV cost or carbon input datasets. + aeo_years: AEO year range. + + Returns: + Mapping between TSV cost/carbon data years and AEO years. + """ + + # Set up a matrix mapping each AEO year to the years available in the + # TSV data + + # Pull available years from TSV data + tsv_yrs = list(sorted(tsv_data.keys())) + # Establish the mapping from available TSV years to AEO years + tsv_yr_map = { + yr_tsv: [str(x) for x in range( + int(yr_tsv), int(tsv_yrs[ind + 1]))] + if (ind + 1) < len(tsv_yrs) else [str(x) for x in range( + int(yr_tsv), int(aeo_years[-1]) + 1)] + for ind, yr_tsv in enumerate(tsv_yrs) + } + # Prepend AEO years preceding the start year in the TSV data, if needed + if (aeo_years[0] not in tsv_yr_map[tsv_yrs[0]]): + yrs_to_prepend = range(int(aeo_years[0]), min([ + int(x) for x in tsv_yr_map[tsv_yrs[0]]])) + tsv_yr_map[tsv_yrs[0]] = [str(x) for x in yrs_to_prepend] + \ + tsv_yr_map[tsv_yrs[0]] + + return tsv_yr_map + class Measure(object): """Set up a class representing efficiency measures as objects. @@ -11714,439 +11814,351 @@ def merge_out_break(self, pkg_brk, meas_brk): return pkg_brk -def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, - handyfiles, cbecs_sf_byvint, tsv_data, base_dir, opts, - ctrb_ms_pkg_prep, tsv_data_nonfs): - """Finalize measure markets for subsequent use in the analysis engine. - - Note: - Determine which in a list of measures require updates to finalize - stock, energy, carbon, and cost markets for further use in the - analysis engine; instantiate these measures as Measure objects; - execute the necessary updates for each object; and update the - original list of measures accordingly. - - Args: - measures (list): List of dicts with efficiency measure attributes. - convert_data (dict): Measure cost unit conversion data. - msegs (dict): Baseline microsegment stock and energy use. - msegs_cpl (dict): Baseline technology cost, performance, and lifetime. - handyvars (object): Global variables of use across Measure methods. - handyfiles (object): Input files of use across Measure methods. - cbecs_sf_byvint (dict): Commercial square footage by vintage data. - tsv_data (dict): Data needed for time sensitive efficiency valuation. - base_dir (string): Base directory. - opts (object): Stores user-specified execution options. - ctrb_ms_pkg_prep (list): Names of measures that contribute to pkgs. - tsv_data_nonfs (dict): If applicable, base-case TSV data to apply to - non-fuel switching measures under a high decarb. scenario. - - Returns: - A list of dicts, each including a set of measure attributes that has - been prepared for subsequent use in the analysis engine. - - Raises: - ValueError: If more than one Measure object matches the name of a - given input efficiency measure. - """ - logger.info("Initializing measures...") - # Translate user options to a dictionary for further use in Measures - opts_dict = vars(opts) - # Initialize Measure() objects based on 'measures_update' list - meas_update_objs = [Measure( - base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] - logger.info("Measure initialization complete") - - # Fill in EnergyPlus-based performance information for Measure objects - # with a 'From EnergyPlus' flag in their 'energy_efficiency' attribute - - # Handle a superfluous 'undefined' key in the ECM performance field that is - # generated by the 'Add ECM' web form in certain cases *** NOTE: WILL - # FIX IN FUTURE UI VERSION *** - if any([isinstance(m.energy_efficiency, dict) and ( - "undefined" in m.energy_efficiency.keys() and - m.energy_efficiency["undefined"] == "From EnergyPlus") for - m in meas_update_objs]): - raise ValueError( - 'One or more ECMs require EnergyPlus data for ECM performance; ' - 'EnergyPlus-based ECM performance data are currently unsupported.') - - # Initialize list with indices of measures to remove from further - # preparation due to Exceptions - remove_inds = [] - - # Check that all Measure objects have valid market inputs before proceeding - for m_ind, m in enumerate(meas_update_objs): - # Try/except allows continuation past malformed ECMs - try: - # Check that the measure's applicable baseline market input definitions - # are valid before attempting to retrieve data on this baseline market - m.check_meas_inputs() - except Exception: - Utils.prep_error(m.name, handyvars, handyfiles) - # Add measure index to removal list - remove_inds.append(m_ind) - - # Finalize 'markets' attribute for all Measure objects - for m_ind, m in enumerate(meas_update_objs): - # Try/except allows continuation when individual ECMs error - try: - m.fill_mkts( - msegs, msegs_cpl, convert_data, tsv_data, opts, - ctrb_ms_pkg_prep, tsv_data_nonfs) - except Exception: - Utils.prep_error(m.name, handyvars, handyfiles) - # Add measure index to removal list - remove_inds.append(m_ind) - - # Remove measure objects with exceptions from further preparation - meas_update_objs = [ - m for m_ind, m in enumerate(meas_update_objs) if - m_ind not in remove_inds] - - return meas_update_objs - - -def prepare_packages(packages, meas_update_objs, meas_summary, - handyvars, handyfiles, base_dir, opts, convert_data): - """Combine multiple measures into a single packaged measure. - - Args: - packages (dict): Names of packages and measures that comprise them. - meas_update_objs (dict): Attributes of individual efficiency measures. - meas_summary (): List of dicts including previously prepared ECM data. - handyvars (object): Global variables of use across Measure methods. - handyfiles (object): Input files of use across Measure methods. - base_dir (string): Base directory. - opts (object): Stores user-specified execution options. - convert_data (dict): Measure cost unit conversion data. - - Returns: - A dict with packaged measure attributes that can be added to the - existing measures database. - """ - # Run through each unique measure package and merge the measures that - # contribute to this package - for p in packages: - # Try/except allows continuation of routine when individual pkgs error - try: - # Notify user that measure is being updated - print("Updating ECM '" + p["name"] + "'...", end="", flush=True) - - # Establish a list of names for measures that contribute to the - # package - package_measures = p["contributing_ECMs"] - # Determine the subset of all previously initialized measure - # objects that contribute to the current package - measure_list_package = [ - x for x in meas_update_objs if x.name in package_measures] - # Determine which contributing measures have not yet been - # initialized as objects - measures_to_add = [mc for mc in package_measures if mc not in [ - x.name for x in measure_list_package]] - # Initialize any missing contributing measure objects and add to - # the existing list of contributing measure objects for the package - for m in measures_to_add: - # Load and set high level summary data for the missing measure - meas_summary_data = [x for x in meas_summary if x["name"] == m] - if len(meas_summary_data) == 1: - # Translate user options to a dictionary for further use in - # Measures - opts_dict = vars(opts) - # Initialize the missing measure as an object - meas_obj = Measure( - base_dir, handyvars, handyfiles, opts_dict, - **meas_summary_data[0]) - # Reset measure technology type and total energy (used to - # normalize output breakout fractions) to their values in the - # high level summary data (reformatted during initialization) - meas_obj.technology_type = meas_summary_data[0][ - "technology_type"] - # Assemble folder path for measure competition data - meas_folder_name = handyfiles.ecm_compete_data - # Assemble file name for measure competition data - meas_file_name = meas_obj.name + ".pkl.gz" - # Load and set competition data for the missing measure object - with gzip.open(meas_folder_name / meas_file_name, 'r') as zp: - try: - meas_comp_data = pickle.load(zp) - except Exception as e: - raise Exception( - "Error reading in competition data of " + - "contributing ECM '" + meas_obj.name + - "' for package '" + p["name"] + "': " + - str(e)) from None - for adopt_scheme in handyvars.adopt_schemes_prep: - meas_obj.markets[adopt_scheme]["master_mseg"] = \ - meas_summary_data[0]["markets"][adopt_scheme][ - "master_mseg"] - meas_obj.markets[adopt_scheme]["mseg_adjust"] = \ - meas_comp_data[adopt_scheme] - meas_obj.markets[adopt_scheme]["mseg_out_break"] = \ - meas_summary_data[0]["markets"][adopt_scheme][ - "mseg_out_break"] - # Add missing measure object to the existing list - measure_list_package.append(meas_obj) - # Raise an error if no existing data exist for the missing - # contributing measure - elif len(meas_summary_data) == 0: - raise ValueError( - "Contributing ECM '" + m + - "' cannot be added to package '" + p["name"] + - "' due to missing attribute data for this ECM") - else: - raise ValueError( - "More than one set of attribute data for " + - "contributing ECM '" + m + "'; ECM cannot be added to" + - "package '" + p["name"]) - - # Determine which (if any) measure objects that contribute to - # the package are invalid due to unacceptable input data sourcing - measure_list_package_rmv = [ - x for x in measure_list_package if x.remove is True] - - # Warn user of no valid measures to package - if len(measure_list_package_rmv) > 0: - warnings.warn("WARNING (CRITICAL): Package '" + p["name"] + - "' removed due to invalid contributing ECM(s)") - packaged_measure = False - # Update package if valid contributing measures are available - else: - # Instantiate measure package object based on packaged measure - # subset above - packaged_measure = MeasurePackage( - measure_list_package, p["name"], p["benefits"], - handyvars, handyfiles, opts, convert_data) - # Record heating/cooling equipment and envelope overlaps in - # package after confirming that envelope measures are present - if len(packaged_measure.contributing_ECMs_env) > 0: - packaged_measure.htcl_adj_rec(opts) - # Merge measures in the package object - packaged_measure.merge_measures(opts) - # Print update on measure status - print("Success") - - # Add the new packaged measure to the measure list (if it exists) - # for further evaluation like any other regular measure - if packaged_measure is not False: - meas_update_objs.append(packaged_measure) - except Exception: - Utils.prep_error(p["name"], handyvars, handyfiles) - - return meas_update_objs - - -def split_clean_data(meas_prepped_objs, full_dat_out): - """Reorganize and remove data from input Measure objects. - - Note: - The input Measure objects have updated data, which must - be reorganized/condensed for the purposes of writing out - to JSON files. +class ECMPrep(): + """Methods to generate and alter Measure and MeasurePackage instances""" - Args: - meas_prepped_objs (object): Measure objects with data to - be split in to separate dicts or removed. - full_dat_out (dict): Flag that limits technical potential (TP) data - prep/reporting when TP is not in user-specified adoption schemes. - - Returns: - Three to four lists of dicts, one containing competition data for - each updated measure, one containing high level summary - data for each updated measure, another containing sector shape - data for each measure (if applicable), and a final one containing - efficient fuel split data, as applicable to fuel switching measures - when the user has required fuel splits. - """ - # Initialize lists of measure competition/summary data - meas_prepped_compete = [] - meas_prepped_summary = [] - meas_prepped_shapes = [] - meas_eff_fs_splt = [] - # Loop through all Measure objects and reorganize/remove the - # needed data. - for m in meas_prepped_objs: - # Initialize a reorganized measure competition data dict and efficient - # fuel split data dict - comp_data_dict, fs_splits_dict, shapes_dict = ({} for n in range(3)) - # Retrieve measure contributing microsegment data that are relevant to - # markets competition in the analysis engine, then remove these data - # from measure object - for adopt_scheme in m.handyvars.adopt_schemes_prep: - # Delete contributing microsegment data that are - # not relevant to competition in the analysis engine - del m.markets[adopt_scheme]["mseg_adjust"][ - "secondary mseg adjustments"]["sub-market"] - del m.markets[adopt_scheme]["mseg_adjust"][ - "secondary mseg adjustments"]["stock-and-flow"] - # If individual measure, delete markets data used to linked - # heating/cooling turnover and switching rates across msegs (these - # data are not prepared for packages) - if not isinstance(m, MeasurePackage): - del m.markets[adopt_scheme]["mseg_adjust"][ - "paired heat/cool mseg adjustments"] - # Add remaining contributing microsegment data to - # competition data dict, if the adoption scenario will be competed - # in the run.py module, then delete from measure - if full_dat_out[adopt_scheme]: - comp_data_dict[adopt_scheme] = \ - m.markets[adopt_scheme]["mseg_adjust"] - # If applicable, add efficient fuel split data to fuel split - # data dict - if len(m.eff_fs_splt[adopt_scheme].keys()) != 0: - fs_splits_dict[adopt_scheme] = \ - m.eff_fs_splt[adopt_scheme] - # If applicable, add sector shape data - if m.sector_shapes is not None and len( - m.sector_shapes[adopt_scheme].keys()) != 0: - shapes_dict["name"] = m.name - shapes_dict[adopt_scheme] = \ - m.sector_shapes[adopt_scheme] - else: - # If adoption scenario will not be competed in the run.py - # module, remove detailed mseg breakouts - del m.markets[adopt_scheme]["mseg_out_break"] - del m.markets[adopt_scheme]["mseg_adjust"] - # Delete info. about efficient fuel splits for fuel switch measures - del m.eff_fs_splt - # Delete info. about sector shapes - del m.sector_shapes - - # Append updated competition data from measure to - # list of competition data across all measures - meas_prepped_compete.append(comp_data_dict) - # Append fuel switching split information, if applicable - meas_eff_fs_splt.append(fs_splits_dict) - # Append sector shape information, if applicable - meas_prepped_shapes.append(shapes_dict) - # Delete 'handyvars' measure attribute (not relevant to - # analysis engine) - del m.handyvars - # Delete 'tsv_features' measure attributes - # (not relevant) for individual measures - if not isinstance(m, MeasurePackage): - del m.tsv_features - # Delete individual measure attributes used to link heating/ - # cooling microsegment turnover and switching rates - del m.linked_htcl_tover - del m.linked_htcl_tover_anchor_eu - del m.linked_htcl_tover_anchor_tech - # For measure packages, replace 'contributing_ECMs' - # objects list with a list of these measures' names and remove - # unnecessary heating/cooling equip/env overlap data - if isinstance(m, MeasurePackage): - m.contributing_ECMs = [ - x.name for x in m.contributing_ECMs] - del m.htcl_overlaps - del m.contributing_ECMs_eqp - del m.contributing_ECMs_env - # Append updated measure __dict__ attribute to list of - # summary data across all measures - meas_prepped_summary.append(m.__dict__) - - return meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ - meas_eff_fs_splt - - -def tsv_cost_carb_yrmap(tsv_data, aeo_years): - """Map 8760 TSV cost/carbon data years to AEO years. - - Args: - tsv_data: TSV cost or carbon input datasets. - aeo_years: AEO year range. - - Returns: - Mapping between TSV cost/carbon data years and AEO years. - """ - - # Set up a matrix mapping each AEO year to the years available in the - # TSV data - - # Pull available years from TSV data - tsv_yrs = list(sorted(tsv_data.keys())) - # Establish the mapping from available TSV years to AEO years - tsv_yr_map = { - yr_tsv: [str(x) for x in range( - int(yr_tsv), int(tsv_yrs[ind + 1]))] - if (ind + 1) < len(tsv_yrs) else [str(x) for x in range( - int(yr_tsv), int(aeo_years[-1]) + 1)] - for ind, yr_tsv in enumerate(tsv_yrs) - } - # Prepend AEO years preceding the start year in the TSV data, if needed - if (aeo_years[0] not in tsv_yr_map[tsv_yrs[0]]): - yrs_to_prepend = range(int(aeo_years[0]), min([ - int(x) for x in tsv_yr_map[tsv_yrs[0]]])) - tsv_yr_map[tsv_yrs[0]] = [str(x) for x in yrs_to_prepend] + \ - tsv_yr_map[tsv_yrs[0]] - - return tsv_yr_map + @staticmethod + def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, + handyfiles, cbecs_sf_byvint, tsv_data, base_dir, opts, + ctrb_ms_pkg_prep, tsv_data_nonfs): + """Finalize measure markets for subsequent use in the analysis engine. + Note: + Determine which in a list of measures require updates to finalize + stock, energy, carbon, and cost markets for further use in the + analysis engine; instantiate these measures as Measure objects; + execute the necessary updates for each object; and update the + original list of measures accordingly. -def downselect_packages(existing_pkgs: list[dict], pkg_subset: list) -> list: - if "*" in pkg_subset: - return existing_pkgs - downselected_pkgs = [pkg for pkg in existing_pkgs if pkg["name"] in pkg_subset] + Args: + measures (list): List of dicts with efficiency measure attributes. + convert_data (dict): Measure cost unit conversion data. + msegs (dict): Baseline microsegment stock and energy use. + msegs_cpl (dict): Baseline technology cost, performance, and lifetime. + handyvars (object): Global variables of use across Measure methods. + handyfiles (object): Input files of use across Measure methods. + cbecs_sf_byvint (dict): Commercial square footage by vintage data. + tsv_data (dict): Data needed for time sensitive efficiency valuation. + base_dir (string): Base directory. + opts (object): Stores user-specified execution options. + ctrb_ms_pkg_prep (list): Names of measures that contribute to pkgs. + tsv_data_nonfs (dict): If applicable, base-case TSV data to apply to + non-fuel switching measures under a high decarb. scenario. - return downselected_pkgs + Returns: + A list of dicts, each including a set of measure attributes that has + been prepared for subsequent use in the analysis engine. + Raises: + ValueError: If more than one Measure object matches the name of a + given input efficiency measure. + """ -def retrieve_valid_ecms(packages: list, - opts: argparse.NameSpace, # noqa: F821 - handyfiles: UsefulInputFiles) -> list: - """Determine full list of individual measure JSON names that 1) contribute to selected - packages in opts.ecm_packages, or 2) are included in opts.ecm_files, and 3) exist in the - ecm definitions directory (opts.ecm_directory) + logger.info("Initializing measures...") + # Translate user options to a dictionary for further use in Measures + opts_dict = vars(opts) + # Initialize Measure() objects based on 'measures_update' list + meas_update_objs = [Measure( + base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] + logger.info("Measure initialization complete") + + print('Initializing measures...', end="", flush=True) + # Translate user options to a dictionary for further use in Measures + opts_dict = vars(opts) + # Initialize Measure() objects based on 'measures_update' list + meas_update_objs = [Measure( + base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] + print("Complete") + + # Fill in EnergyPlus-based performance information for Measure objects + # with a 'From EnergyPlus' flag in their 'energy_efficiency' attribute + + # Handle a superfluous 'undefined' key in the ECM performance field that is + # generated by the 'Add ECM' web form in certain cases *** NOTE: WILL + # FIX IN FUTURE UI VERSION *** + if any([isinstance(m.energy_efficiency, dict) and ( + "undefined" in m.energy_efficiency.keys() and + m.energy_efficiency["undefined"] == "From EnergyPlus") for + m in meas_update_objs]): + raise ValueError( + 'One or more ECMs require EnergyPlus data for ECM performance; ' + 'EnergyPlus-based ECM performance data are currently unsupported.') - Args: - packages (list): List of valid packages - opts (argparse.NameSpace): object storing user responses - handyfiles (UsefulInputFiles): object storing input filepaths + # Initialize list with indices of measures to remove from further + # preparation due to Exceptions + remove_inds = [] - Returns: - list: filtered list of ECMs that meet the criteria above - """ + # Check that all Measure objects have valid market inputs before proceeding + for m_ind, m in enumerate(meas_update_objs): + # Try/except allows continuation past malformed ECMs + try: + # Check that the measure's applicable baseline market input definitions + # are valid before attempting to retrieve data on this baseline market + m.check_meas_inputs() + except Exception: + ECMUtils.prep_error(m.name, handyvars, handyfiles) + # Add measure index to removal list + remove_inds.append(m_ind) + + # Finalize 'markets' attribute for all Measure objects + for m_ind, m in enumerate(meas_update_objs): + # Try/except allows continuation when individual ECMs error + try: + m.fill_mkts( + msegs, msegs_cpl, convert_data, tsv_data, opts, + ctrb_ms_pkg_prep, tsv_data_nonfs) + except Exception: + ECMUtils.prep_error(m.name, handyvars, handyfiles) + # Add measure index to removal list + remove_inds.append(m_ind) - contributing_ecms = { - ecm for pkg in packages for ecm in pkg["contributing_ECMs"]} - opts.ecm_files.extend([ecm for ecm in contributing_ecms if ecm not in opts.ecm_files]) - valid_ecms = [ - x for x in handyfiles.indiv_ecms.iterdir() if x.suffix == ".json" and - 'package_ecms' not in x.name and x.stem in opts.ecm_files] + # Remove measure objects with exceptions from further preparation + meas_update_objs = [ + m for m_ind, m in enumerate(meas_update_objs) if + m_ind not in remove_inds] - return valid_ecms + return meas_update_objs + @staticmethod + def prepare_packages(packages, meas_update_objs, meas_summary, + handyvars, handyfiles, base_dir, opts, convert_data): + """Combine multiple measures into a single packaged measure. -def filter_invalid_packages(packages: list[dict], - ecms: list, - opts: argparse.Namespace) -> tuple[list[dict], list]: - """Identify and filter packages whose ECMs are not all present in the individual ECM set + Args: + packages (dict): Names of packages and measures that comprise them. + meas_update_objs (dict): Attributes of individual efficiency measures. + meas_summary (): List of dicts including previously prepared ECM data. + handyvars (object): Global variables of use across Measure methods. + handyfiles (object): Input files of use across Measure methods. + base_dir (string): Base directory. + opts (object): Stores user-specified execution options. + convert_data (dict): Measure cost unit conversion data. - Args: - packages (list[dict]): List of packages imported from package_ecms.json - ecms (list): List of ECM definitions file names - opts (argparse.Namespace): argparse object containing the argument attributes + Returns: + A dict with packaged measure attributes that can be added to the + existing measures database. + """ + # Run through each unique measure package and merge the measures that + # contribute to this package + for p in packages: + # Try/except allows continuation of routine when individual pkgs error + try: + # Notify user that measure is being updated + print("Updating ECM '" + p["name"] + "'...", end="", flush=True) + + # Establish a list of names for measures that contribute to the + # package + package_measures = p["contributing_ECMs"] + # Determine the subset of all previously initialized measure + # objects that contribute to the current package + measure_list_package = [ + x for x in meas_update_objs if x.name in package_measures] + # Determine which contributing measures have not yet been + # initialized as objects + measures_to_add = [mc for mc in package_measures if mc not in [ + x.name for x in measure_list_package]] + # Initialize any missing contributing measure objects and add to + # the existing list of contributing measure objects for the package + for m in measures_to_add: + # Load and set high level summary data for the missing measure + meas_summary_data = [x for x in meas_summary if x["name"] == m] + if len(meas_summary_data) == 1: + # Translate user options to a dictionary for further use in + # Measures + opts_dict = vars(opts) + # Initialize the missing measure as an object + meas_obj = Measure( + base_dir, handyvars, handyfiles, opts_dict, + **meas_summary_data[0]) + # Reset measure technology type and total energy (used to + # normalize output breakout fractions) to their values in the + # high level summary data (reformatted during initialization) + meas_obj.technology_type = meas_summary_data[0][ + "technology_type"] + # Assemble folder path for measure competition data + meas_folder_name = handyfiles.ecm_compete_data + # Assemble file name for measure competition data + meas_file_name = meas_obj.name + ".pkl.gz" + # Load and set competition data for the missing measure object + with gzip.open(meas_folder_name / meas_file_name, 'r') as zp: + try: + meas_comp_data = pickle.load(zp) + except Exception as e: + raise Exception( + "Error reading in competition data of " + + "contributing ECM '" + meas_obj.name + + "' for package '" + p["name"] + "': " + + str(e)) from None + for adopt_scheme in handyvars.adopt_schemes_prep: + meas_obj.markets[adopt_scheme]["master_mseg"] = \ + meas_summary_data[0]["markets"][adopt_scheme][ + "master_mseg"] + meas_obj.markets[adopt_scheme]["mseg_adjust"] = \ + meas_comp_data[adopt_scheme] + meas_obj.markets[adopt_scheme]["mseg_out_break"] = \ + meas_summary_data[0]["markets"][adopt_scheme][ + "mseg_out_break"] + # Add missing measure object to the existing list + measure_list_package.append(meas_obj) + # Raise an error if no existing data exist for the missing + # contributing measure + elif len(meas_summary_data) == 0: + raise ValueError( + "Contributing ECM '" + m + + "' cannot be added to package '" + p["name"] + + "' due to missing attribute data for this ECM") + else: + raise ValueError( + "More than one set of attribute data for " + + "contributing ECM '" + m + "'; ECM cannot be added to" + + "package '" + p["name"]) + + # Determine which (if any) measure objects that contribute to + # the package are invalid due to unacceptable input data sourcing + measure_list_package_rmv = [ + x for x in measure_list_package if x.remove is True] + + # Warn user of no valid measures to package + if len(measure_list_package_rmv) > 0: + warnings.warn("WARNING (CRITICAL): Package '" + p["name"] + + "' removed due to invalid contributing ECM(s)") + packaged_measure = False + # Update package if valid contributing measures are available + else: + # Instantiate measure package object based on packaged measure + # subset above + packaged_measure = MeasurePackage( + measure_list_package, p["name"], p["benefits"], + handyvars, handyfiles, opts, convert_data) + # Record heating/cooling equipment and envelope overlaps in + # package after confirming that envelope measures are present + if len(packaged_measure.contributing_ECMs_env) > 0: + packaged_measure.htcl_adj_rec(opts) + # Merge measures in the package object + packaged_measure.merge_measures(opts) + # Print update on measure status + print("Success") + + # Add the new packaged measure to the measure list (if it exists) + # for further evaluation like any other regular measure + if packaged_measure is not False: + meas_update_objs.append(packaged_measure) + except Exception: + ECMUtils.prep_error(p["name"], handyvars, handyfiles) + + return meas_update_objs - Returns: - filtered_packages (list[dict]): Packages list with invalid packages filtered out - invalid_pkgs (list): List of invalid packages - """ + @staticmethod + def split_clean_data(meas_prepped_objs, full_dat_out): + """Reorganize and remove data from input Measure objects. - invalid_pkgs = [pkg["name"] for pkg in packages if not - set(pkg["contributing_ECMs"]).issubset(set(ecms))] - filtered_packages = [pkg for pkg in packages if pkg["name"] not in invalid_pkgs] + Note: + The input Measure objects have updated data, which must + be reorganized/condensed for the purposes of writing out + to JSON files. - # Trigger warning message regarding screening of packages - package_opt_txt = "" - if opts.ecm_packages is not None: - package_opt_txt = "specified with the ecm_packages argument " - if invalid_pkgs: - invalid_pkgs_txt = format_console_list(invalid_pkgs) - msg = (f"WARNING: Package(s) in package_ecms.json {package_opt_txt}have contributing ECMs" - " that are not present among ECM definitions. The following packages will not be" - f" executed: \n{''.join(invalid_pkgs_txt)}") - warnings.warn(msg) + Args: + meas_prepped_objs (object): Measure objects with data to + be split in to separate dicts or removed. + full_dat_out (dict): Flag that limits technical potential (TP) data + prep/reporting when TP is not in user-specified adoption schemes. - return filtered_packages, invalid_pkgs + Returns: + Three to four lists of dicts, one containing competition data for + each updated measure, one containing high level summary + data for each updated measure, another containing sector shape + data for each measure (if applicable), and a final one containing + efficient fuel split data, as applicable to fuel switching measures + when the user has required fuel splits. + """ + # Initialize lists of measure competition/summary data + meas_prepped_compete = [] + meas_prepped_summary = [] + meas_prepped_shapes = [] + meas_eff_fs_splt = [] + # Loop through all Measure objects and reorganize/remove the + # needed data. + for m in meas_prepped_objs: + # Initialize a reorganized measure competition data dict and efficient + # fuel split data dict + comp_data_dict, fs_splits_dict, shapes_dict = ({} for n in range(3)) + # Retrieve measure contributing microsegment data that are relevant to + # markets competition in the analysis engine, then remove these data + # from measure object + for adopt_scheme in m.handyvars.adopt_schemes_prep: + # Delete contributing microsegment data that are + # not relevant to competition in the analysis engine + del m.markets[adopt_scheme]["mseg_adjust"][ + "secondary mseg adjustments"]["sub-market"] + del m.markets[adopt_scheme]["mseg_adjust"][ + "secondary mseg adjustments"]["stock-and-flow"] + # If individual measure, delete markets data used to linked + # heating/cooling turnover and switching rates across msegs (these + # data are not prepared for packages) + if not isinstance(m, MeasurePackage): + del m.markets[adopt_scheme]["mseg_adjust"][ + "paired heat/cool mseg adjustments"] + # Add remaining contributing microsegment data to + # competition data dict, if the adoption scenario will be competed + # in the run.py module, then delete from measure + if full_dat_out[adopt_scheme]: + comp_data_dict[adopt_scheme] = \ + m.markets[adopt_scheme]["mseg_adjust"] + # If applicable, add efficient fuel split data to fuel split + # data dict + if len(m.eff_fs_splt[adopt_scheme].keys()) != 0: + fs_splits_dict[adopt_scheme] = \ + m.eff_fs_splt[adopt_scheme] + # If applicable, add sector shape data + if m.sector_shapes is not None and len( + m.sector_shapes[adopt_scheme].keys()) != 0: + shapes_dict["name"] = m.name + shapes_dict[adopt_scheme] = \ + m.sector_shapes[adopt_scheme] + else: + # If adoption scenario will not be competed in the run.py + # module, remove detailed mseg breakouts + del m.markets[adopt_scheme]["mseg_out_break"] + del m.markets[adopt_scheme]["mseg_adjust"] + # Delete info. about efficient fuel splits for fuel switch measures + del m.eff_fs_splt + # Delete info. about sector shapes + del m.sector_shapes + + # Append updated competition data from measure to + # list of competition data across all measures + meas_prepped_compete.append(comp_data_dict) + # Append fuel switching split information, if applicable + meas_eff_fs_splt.append(fs_splits_dict) + # Append sector shape information, if applicable + meas_prepped_shapes.append(shapes_dict) + # Delete 'handyvars' measure attribute (not relevant to + # analysis engine) + del m.handyvars + # Delete 'tsv_features' measure attributes + # (not relevant) for individual measures + if not isinstance(m, MeasurePackage): + del m.tsv_features + # Delete individual measure attributes used to link heating/ + # cooling microsegment turnover and switching rates + del m.linked_htcl_tover + del m.linked_htcl_tover_anchor_eu + del m.linked_htcl_tover_anchor_tech + # For measure packages, replace 'contributing_ECMs' + # objects list with a list of these measures' names and remove + # unnecessary heating/cooling equip/env overlap data + if isinstance(m, MeasurePackage): + m.contributing_ECMs = [ + x.name for x in m.contributing_ECMs] + del m.htcl_overlaps + del m.contributing_ECMs_eqp + del m.contributing_ECMs_env + # Append updated measure __dict__ attribute to list of + # summary data across all measures + meas_prepped_summary.append(m.__dict__) + + return meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ + meas_eff_fs_splt def main(opts: argparse.NameSpace): # noqa: F821 @@ -12163,7 +12175,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 """ # Configure logger specific to ecm_prep - configure_ecm_prep_logger(opts) + ECMUtils.configure_ecm_prep_logger() # Set current working directory base_dir = getcwd() @@ -12193,7 +12205,8 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Import packages JSON, filter as needed meas_toprep_package_init = JsonIO.load_json(handyfiles.ecm_packages) - meas_toprep_package_init = downselect_packages(meas_toprep_package_init, opts.ecm_packages) + meas_toprep_package_init = ECMUtils.downselect_packages(meas_toprep_package_init, + opts.ecm_packages) # If applicable, import file to write prepared measure sector shapes to # (if file does not exist, provide empty list as substitute, since file @@ -12210,7 +12223,9 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_shapes = [] # Determine full list of individual measure JSON names - meas_toprep_indiv_names = retrieve_valid_ecms(meas_toprep_package_init, opts, handyfiles) + meas_toprep_indiv_names = ECMUtils.retrieve_valid_ecms(meas_toprep_package_init, + opts, + handyfiles) # Initialize list of all individual measures that require updates meas_toprep_indiv = [] @@ -12546,21 +12561,23 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_prepped_pkgs = [mpkg for mpkg in meas_summary if "contributing_ECMs" in mpkg.keys()] # Identify and filter packages whose ECMs are not all present in ECM list ecm_names = [meas.stem for meas in meas_toprep_indiv_names] - meas_toprep_package_init, pkgs_skipped = filter_invalid_packages(meas_toprep_package_init, - ecm_names, - opts) + meas_toprep_package_init, pkgs_skipped = ECMUtils.filter_invalid_packages( + meas_toprep_package_init, + ecm_names, + opts + ) # Write initial data for run_setup.json # Import analysis engine setup file - run_setup = Utils.initialize_run_setup(handyfiles) + run_setup = ECMUtils.initialize_run_setup(handyfiles) # Set contributing ECMs as inactive in run_setup and throw warning, set all others as active ctrb_ms = [ecm for pkg in meas_toprep_package_init for ecm in pkg["contributing_ECMs"]] non_ctrb_ms = [ecm for ecm in opts.ecm_files if ecm not in ctrb_ms] excluded_ind_ecms = [ecm for ecm in opts.ecm_files_user if ecm in ctrb_ms] - run_setup = Utils.update_active_measures(run_setup, - to_active=non_ctrb_ms, - to_inactive=excluded_ind_ecms) + run_setup = ECMUtils.update_active_measures(run_setup, + to_active=non_ctrb_ms, + to_inactive=excluded_ind_ecms) if excluded_ind_ecms: excluded_ind_ecms_txt = fmt.format_console_list(excluded_ind_ecms) warnings.warn("The following ECMs were selected to be prepared, but due to their" @@ -12570,7 +12587,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Set packages to active in run_setup valid_packages = [pkg["name"] for pkg in meas_toprep_package_init] - run_setup = Utils.update_active_measures(run_setup, to_active=valid_packages) + run_setup = ECMUtils.update_active_measures(run_setup, to_active=valid_packages) # Loop through each package dict in the current list and determine which # of these package measures require further preparation @@ -12738,10 +12755,10 @@ def main(opts: argparse.NameSpace): # noqa: F821 tsv_carbon_nonfs_data = None # Map years available in 8760 TSV cost/carbon data to AEO yrs. - tsv_cost_yrmap = tsv_cost_carb_yrmap( + tsv_cost_yrmap = ECMUtils.tsv_cost_carb_yrmap( tsv_cost_data["electricity price shapes"], handyvars.aeo_years) - tsv_carbon_yrmap = tsv_cost_carb_yrmap( + tsv_carbon_yrmap = ECMUtils.tsv_cost_carb_yrmap( tsv_carbon_data["average carbon emissions rates"], handyvars.aeo_years) # Stitch together load shape, cost, emissions, and year @@ -12770,7 +12787,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 logger.info("Supporting data import complete") # Prepare new or edited measures for use in analysis engine - meas_prepped_objs = prepare_measures( + meas_prepped_objs = ECMPrep.prepare_measures( meas_toprep_indiv, convert_data, msegs, msegs_cpl, handyvars, handyfiles, cbecs_sf_byvint, tsv_data, base_dir, opts, ctrb_ms_pkg_prep, tsv_data_nonfs) @@ -12781,15 +12798,17 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_check_list = [mo.name for mo in meas_prepped_objs] # User is warned later, after being warned that ECMs have been skipped if len(handyvars.skipped_ecms) != 0: - meas_toprep_package, pkgs_skipped = filter_invalid_packages(meas_toprep_package, - meas_check_list, - opts) + meas_toprep_package, pkgs_skipped = ECMUtils.filter_invalid_packages( + meas_toprep_package, + meas_check_list, + opts + ) # Move package name to skipped list - run_setup = Utils.update_active_measures(run_setup, to_skipped=pkgs_skipped) + run_setup = ECMUtils.update_active_measures(run_setup, to_skipped=pkgs_skipped) # Prepare measure packages for use in analysis engine (if needed) if meas_toprep_package: - meas_prepped_objs = prepare_packages( + meas_prepped_objs = ECMPrep.prepare_packages( meas_toprep_package, meas_prepped_objs, meas_summary, handyvars, handyfiles, base_dir, opts, convert_data) @@ -12812,14 +12831,14 @@ def main(opts: argparse.NameSpace): # noqa: F821 "corresponding timestamp in ./generated for details.") # Add names of skipped measures to run setup list if not already there - run_setup = Utils.update_active_measures(run_setup, to_skipped=handyvars.skipped_ecms) + run_setup = ECMUtils.update_active_measures(run_setup, to_skipped=handyvars.skipped_ecms) logger.info("All ECM updates complete; finalizing data...") # Split prepared measure data into subsets needed to set high-level # measure attributes information and to execute measure competition # in the analysis engine meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ - meas_eff_fs_splt = split_clean_data( + meas_eff_fs_splt = ECMPrep.split_clean_data( meas_prepped_objs, handyvars.full_dat_out) # Add all prepared high-level measure information to existing @@ -12859,7 +12878,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Remove measures from active list; when public health costs are assumed, only # the "high" health costs versions of prepared measures remain active if opts.health_costs is True and "PHC-EE (high)" not in m["name"]: - run_setup = Utils.update_active_measures(run_setup, to_inactive=[m["name"]]) + run_setup = ECMUtils.update_active_measures(run_setup, to_inactive=[m["name"]]) # Measure serves as counterfactual for isolating envelope impacts # within packages; append data to separate list, which will # be written to a separate ecm_prep file diff --git a/scout/run_batch.py b/scout/run_batch.py index e6a985d7a..72b7bff2e 100644 --- a/scout/run_batch.py +++ b/scout/run_batch.py @@ -2,7 +2,7 @@ from pathlib import Path from scout.config import LogConfig, Config, FilePaths as fp from scout.ecm_prep_args import ecm_args -from scout.ecm_prep import Utils, main as ecm_prep_main +from scout.ecm_prep import ECMUtils, main as ecm_prep_main from scout.utils import JsonIO from scout import run from argparse import ArgumentParser @@ -127,10 +127,10 @@ def run_batch(self): ecm_files_list = self.get_ecm_files(yml_grp) for ct, config in enumerate(yml_grp): # Set all ECMs inactive - run_setup = Utils.update_active_measures(run_setup, - to_inactive=ecm_prep_opts.ecm_files) + run_setup = ECMUtils.update_active_measures(run_setup, + to_inactive=ecm_prep_opts.ecm_files) # Set yml-specific ECMs active - run_setup = Utils.update_active_measures(run_setup, to_active=ecm_files_list[ct]) + run_setup = ECMUtils.update_active_measures(run_setup, to_active=ecm_files_list[ct]) JsonIO.dump_json(run_setup, fp.GENERATED / "run_setup.json") run_opts = self.get_run_opts(config) logger.info(f"Running run.py for {config}") diff --git a/tests/ecm_prep_test.py b/tests/ecm_prep_test.py index c105cc661..5195aac81 100644 --- a/tests/ecm_prep_test.py +++ b/tests/ecm_prep_test.py @@ -3,7 +3,7 @@ """ Tests for running the measure preparation routine """ # Import code to be tested -from scout import ecm_prep +from scout.ecm_prep import Measure, MeasurePackage, ECMUtils, ECMPrep from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles from scout.config import FilePaths as fp from scout.ecm_prep_args import ecm_args @@ -228,7 +228,7 @@ def setUpClass(cls): # Useful global variables for the sample measure object handyfiles = UsefulInputFiles(opts) handyvars = UsefulVars(base_dir, handyfiles, opts) - cls.meas = ecm_prep.Measure( + cls.meas = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) # Finalize the measure's 'technology_type' attribute (handled by the # 'fill_attr' function, which is not run as part of this test) @@ -7453,23 +7453,23 @@ def setUpClass(cls): "end_use": "cooling", "technology": "ASHP"}] cls.ok_tpmeas_fullchk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in ok_measures_in[0:5]] cls.ok_tpmeas_partchk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in ok_measures_in[5:24]] cls.ok_tpmeas_partchk_emm_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars_emm, handyfiles_emm, opts_emm_dict, **x) for x in ok_measures_in[26:28]] cls.ok_tpmeas_partchk_state_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars_state, handyfiles_state, opts_state_dict, **ok_measures_in[28])] cls.ok_mapmeas_partchk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in ok_measures_in[24:26]] ok_measures_site_energy = [ @@ -7497,7 +7497,7 @@ def setUpClass(cls): "end_use": ["heating", "cooling"], "technology": ["resistance heat", "ASHP", "GSHP", "room AC"]} ] - cls.ok_tpmeas_sitechk_in = [ecm_prep.Measure( + cls.ok_tpmeas_sitechk_in = [Measure( base_dir, handyvars, handyfiles, opts_site_energy_dict, **x) for x in ok_measures_site_energy] ok_distmeas_in = [{ @@ -7599,7 +7599,7 @@ def setUpClass(cls): # results numpy.random.seed(1234) cls.ok_distmeas_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in ok_distmeas_in] ok_partialmeas_in = [{ @@ -7646,7 +7646,7 @@ def setUpClass(cls): "general service (LED)", "external (LED)", "GSHP", "ASHP"]}] cls.ok_partialmeas_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in ok_partialmeas_in] failmeas_in = [{ @@ -7857,10 +7857,10 @@ def setUpClass(cls): "market_exit_year": None, "technology": [None, "distribution transformers"]}] cls.failmeas_inputs_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in failmeas_in[0:-1]] - cls.failmeas_missing_in = ecm_prep.Measure( + cls.failmeas_missing_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **failmeas_in[-1]) warnmeas_in = [{ @@ -8020,7 +8020,7 @@ def setUpClass(cls): "general service (LED)", "external (LED)"]}] cls.warnmeas_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in warnmeas_in] ok_hp_measures_in = [{ @@ -8116,20 +8116,20 @@ def setUpClass(cls): "end_use": ["heating", "cooling"], "technology": ["furnace (NG)", "central AC"]}] cls.ok_mapmeas_hp_chk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars_hp_rates, handyfiles_emm, opts_hp_rates_dict, **x) for x in ok_hp_measures_in[0:3]] cls.ok_mapmeas_hp_chk_in.extend([ - ecm_prep.Measure( + Measure( base_dir, handyvars_hp_norates, handyfiles_emm, opts_hp_rates_dict, **x) for x in ok_hp_measures_in[3:]]) cls.ok_tp_fmeth_chk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars_fmeth[ind], handyfiles_emm, opts_fmeth_dict[ind], **x) for ind, x in enumerate( ok_fmeth_measures_in)] cls.ok_map_frefr_chk_in = [ - ecm_prep.Measure( + Measure( base_dir, handyvars_frefr[ind], handyfiles_emm, opts_frefr_dict[ind], **x) for ind, x in enumerate( ok_frefr_measures_in)] @@ -62770,7 +62770,7 @@ def setUpClass(cls): cls.opts_tsv_features.alt_regions, \ opts_dict_tsv_features["alt_regions"] = ( "EMM" for n in range(2)) - cls.ok_tsv_measures_in_features = [ecm_prep.Measure( + cls.ok_tsv_measures_in_features = [Measure( base_dir, handyvars, handyfiles, opts_dict_tsv_features, **x) for x in sample_tsv_measures_in_features] # Sample measure to use in testing time-sensitive valuation metrics @@ -62850,7 +62850,7 @@ def setUpClass(cls): opts_dict_tsv_metrics[ind]["tsv_metrics"] = \ sample_tsv_metric_settings[ind] # Initialize TSV metric test measures using appropriate input settings - cls.ok_tsv_measures_in_metrics = [ecm_prep.Measure( + cls.ok_tsv_measures_in_metrics = [Measure( base_dir, handyvars, handyfiles, opts_dict_tsv_metrics[ind], **x) for x in sample_tsv_measure_in_metrics for ind in range(len(sample_tsv_metric_settings))] @@ -63468,25 +63468,25 @@ def setUpClass(cls): "fraction_2050": 1 }, "retro_rate": 0.02} - cls.measure_instance_fraction = ecm_prep.Measure( + cls.measure_instance_fraction = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_fraction) - cls.measure_instance_bass = ecm_prep.Measure( + cls.measure_instance_bass = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_bass) - cls.measure_instance_fraction_string = ecm_prep.Measure( + cls.measure_instance_fraction_string = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_fraction_string) - cls.measure_instance_bass_string = ecm_prep.Measure( + cls.measure_instance_bass_string = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_bass_string) - cls.measure_instance_bad_string = ecm_prep.Measure( + cls.measure_instance_bad_string = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_bad_string) - cls.measure_instance_bad_values = ecm_prep.Measure( + cls.measure_instance_bad_values = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_bad_values) - cls.measure_instance_wrong_name = ecm_prep.Measure( + cls.measure_instance_wrong_name = Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **sample_measure_wrong_name) cls.ok_diffuse_params_in = None @@ -66858,7 +66858,7 @@ def setUpClass(cls): "technology": { "primary": "all", "secondary": None}}] - cls.sample_measures_fail = [ecm_prep.Measure( + cls.sample_measures_fail = [Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in sample_measures_fail] @@ -67062,7 +67062,7 @@ def setUpClass(cls): "fuel_switch_to": None, "end_use": "heating", "technology": "all"}] - cls.sample_measures_in = [ecm_prep.Measure( + cls.sample_measures_in = [Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in sample_measures] cls.ok_primary_cpl_out = [[{ @@ -67498,7 +67498,7 @@ def setUpClass(cls): "original energy (competed and captured)": {}, "adjusted energy (total captured)": {}, "adjusted energy (competed and captured)": {}}}}} - cls.sample_measure_in = ecm_prep.Measure( + cls.sample_measure_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure) # Finalize the measure's 'technology_type' attribute (handled by the @@ -67697,7 +67697,7 @@ def setUpClass(cls): "technology": { "primary": ["resistance heat", "ASHP", "GSHP", "room AC"], "secondary": None}} - cls.sample_measure_in = ecm_prep.Measure( + cls.sample_measure_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) cls.ok_dict1_in, cls.ok_dict2_in = ({ @@ -67817,7 +67817,7 @@ def setUpClass(cls): "technology": { "primary": ["resistance heat", "ASHP", "GSHP", "room AC"], "secondary": None}} - cls.sample_measure_in = ecm_prep.Measure( + cls.sample_measure_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) cls.ok_reduce_dict = {"2009": 100, "2010": 100} @@ -67909,7 +67909,7 @@ def setUpClass(cls): "technology": { "primary": ["resistance heat", "ASHP", "GSHP", "room AC"], "secondary": None}} - cls.sample_measure_in = ecm_prep.Measure( + cls.sample_measure_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) cls.ok_reduce_num = 4 cls.ok_dict_in = { @@ -68282,7 +68282,7 @@ def setUpClass(cls): "adjusted energy (total captured)": {}, "adjusted energy (competed and captured)": {}}}}} cls.verbose = None - cls.sample_measure_in = ecm_prep.Measure( + cls.sample_measure_in = Measure( base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) cls.sample_convertdata_ok_in = { @@ -124243,10 +124243,10 @@ def test_filter_packages(self): packages = json.load(f) # Downselect via the ecm_packages argument - selected_pkgs = ecm_prep.downselect_packages(packages, opts_pkgs.ecm_packages) + selected_pkgs = ECMUtils.downselect_packages(packages, opts_pkgs.ecm_packages) # Further filter packages that do not have all contributing ECMs - valid_pkgs, invalid_pkgs = ecm_prep.filter_invalid_packages(selected_pkgs, + valid_pkgs, invalid_pkgs = ECMUtils.filter_invalid_packages(selected_pkgs, opts_pkgs.ecm_files, opts_pkgs) # Check list of valid packages @@ -124271,10 +124271,10 @@ def test_contributing_ecm_add(self): packages = json.load(f) # Downselect via the ecm_packages argument - selected_pkgs = ecm_prep.downselect_packages(packages, opts_pkg_no_ecm.ecm_packages) + selected_pkgs = ECMUtils.downselect_packages(packages, opts_pkg_no_ecm.ecm_packages) # Add ECMs that contribute to package - ecms = ecm_prep.retrieve_valid_ecms(selected_pkgs, opts_pkg_no_ecm, self.handyfiles_emm) + ecms = ECMUtils.retrieve_valid_ecms(selected_pkgs, opts_pkg_no_ecm, self.handyfiles_emm) ecms = [ecm.stem for ecm in ecms] expected_ecms = ["ENERGY STAR Res. ASHP (FS)", "Res. Air Sealing (New), IECC c. 2021", @@ -124294,7 +124294,7 @@ def test_fillmeas_ok(self): require updating and that the updates are performed correctly. """ # Check for measures using AIA baseline data - measures_out_aia = ecm_prep.prepare_measures( + measures_out_aia = ECMPrep.prepare_measures( self.aia_measures, self.convert_data, self.sample_mseg_in_aia, self.sample_cpl_in_aia, self.handyvars_aia, @@ -124308,7 +124308,7 @@ def test_fillmeas_ok(self): "Technical potential"]["master_mseg"], self.ok_out_aia[oc_aia]) # Check for measures using EMM baseline data and tsv features - measures_out_emm_features = ecm_prep.prepare_measures( + measures_out_emm_features = ECMPrep.prepare_measures( self.emm_measures_features, self.convert_data, self.sample_mseg_in_emm, self.sample_cpl_in_emm, self.handyvars_emm, @@ -124317,7 +124317,7 @@ def test_fillmeas_ok(self): ctrb_ms_pkg_prep=[], tsv_data_nonfs=None) # Check for measures using EMM baseline data and public health energy # cost adders - measures_out_health_benefits = ecm_prep.prepare_measures( + measures_out_health_benefits = ECMPrep.prepare_measures( self.health_cost_measures, self.convert_data, self.sample_mseg_in_emm, self.sample_cpl_in_emm, self.handyvars_health, @@ -124335,7 +124335,7 @@ def test_fillmeas_ok(self): for oc_emm in range(0, len(self.emm_measures_metrics)): # Check for measures using EMM baseline data and tsv metrics # or sector-level load shape options - measures_out_emm_metrics = ecm_prep.prepare_measures( + measures_out_emm_metrics = ECMPrep.prepare_measures( [self.emm_measures_metrics[oc_emm]], self.convert_data, self.sample_mseg_in_emm, self.sample_cpl_in_emm, self.handyvars_emm, @@ -127004,13 +127004,13 @@ def setUpClass(cls): } } }}] - cls.sample_measures_in_mkts = [ecm_prep.Measure( + cls.sample_measures_in_mkts = [Measure( base_dir, handyvars, handyfiles, opts_dict, **x) for x in sample_measures_in_mkts] - cls.sample_measures_in_env_costs = [ecm_prep.Measure( + cls.sample_measures_in_env_costs = [Measure( base_dir, handyvars, handyfiles, opts_env_costs_dict, **x) for x in sample_measures_in_env_costs] - cls.sample_measures_in_sect_shapes = [ecm_prep.Measure( + cls.sample_measures_in_sect_shapes = [Measure( base_dir, handyvars_sect_shapes, handyfiles, opts_sect_shapes_dict, **x) for x in sample_measures_in_sect_shapes] for ind, m in enumerate(cls.sample_measures_in_mkts): @@ -127042,7 +127042,7 @@ def setUpClass(cls): None for n in range(len(sample_package_meas_pairs_highlevel))] for pkg in range(len(sample_package_names_highlevel)): cls.sample_package_in_test1_highlevel[pkg] = \ - ecm_prep.MeasurePackage( + MeasurePackage( sample_package_meas_pairs_highlevel[pkg], sample_package_names_highlevel[pkg], benefits_test1, handyvars, handyfiles, cls.opts, @@ -127060,25 +127060,25 @@ def setUpClass(cls): cls.sample_measures_in_env_costs[0], sample_measures_in_mkts_envcosts_1, sample_measures_in_mkts_envcosts_2] - cls.sample_package_in_test1_env_costs = ecm_prep.MeasurePackage( + cls.sample_package_in_test1_env_costs = MeasurePackage( sample_package_meas_pairs_env_costs, sample_package_names_highlevel[-1], benefits_test1, handyvars, handyfiles, cls.opts_env_costs, cls.cost_convert_data) - cls.sample_package_in_test1_attr_breaks = ecm_prep.MeasurePackage( + cls.sample_package_in_test1_attr_breaks = MeasurePackage( sample_package_meas_pairs_env_costs, sample_package_names_highlevel[-1], benefits_test1, handyvars, handyfiles, cls.opts, convert_data=None) - cls.sample_package_in_sect_shapes = ecm_prep.MeasurePackage( + cls.sample_package_in_sect_shapes = MeasurePackage( cls.sample_measures_in_sect_shapes, sample_package_names_sect_shapes[0], benefits_test1, handyvars_sect_shapes, handyfiles, cls.opts_sect_shapes, convert_data=None) - cls.sample_package_in_sect_shapes_bens = ecm_prep.MeasurePackage( + cls.sample_package_in_sect_shapes_bens = MeasurePackage( cls.sample_measures_in_sect_shapes, sample_package_names_sect_shapes[0], benefits_test2, handyvars_sect_shapes, handyfiles, cls.opts_sect_shapes, convert_data=None) - cls.sample_package_in_test2 = ecm_prep.MeasurePackage( + cls.sample_package_in_test2 = MeasurePackage( cls.sample_measures_in_mkts, sample_package_names_highlevel[0], benefits_test2, handyvars, handyfiles, cls.opts, convert_data=None) @@ -127901,7 +127901,7 @@ def setUpClass(cls): }, "technology": { "primary": ["central AC"], "secondary": None}}] - cls.sample_measlist_in = [ecm_prep.Measure( + cls.sample_measlist_in = [Measure( base_dir, cls.handyvars, cls.handyfiles, opts_dict, **x) for x in sample_measindiv_dicts] cls.sample_full_dat_out = { @@ -127912,7 +127912,7 @@ def setUpClass(cls): # 'fill_mkts' function, which isn't tested here) for m in cls.sample_measlist_in: m.technology_type = {"primary": ["supply"], "secondary": None} - sample_measpackage = ecm_prep.MeasurePackage( + sample_measpackage = MeasurePackage( copy.deepcopy(cls.sample_measlist_in), "cleanup 3", benefits, cls.handyvars, cls.handyfiles, opts, convert_data=None) cls.sample_measlist_in.append(sample_measpackage) @@ -128000,8 +128000,7 @@ def test_cleanup(self): """Test 'split_clean_data' function given valid inputs.""" # Execute the function measures_comp_data, measures_summary_data, \ - measures_shape_data, measures_eff_fs_splt_data = \ - ecm_prep.split_clean_data( + measures_shape_data, measures_eff_fs_splt_data = ECMPrep.split_clean_data( self.sample_measlist_in, self.sample_full_dat_out) # Check function outputs for ind in range(0, len(self.sample_measlist_in)): @@ -128089,7 +128088,7 @@ def test_yrmap(self): # Loop across all tested year input scenarios for ind in range(0, len(self.test_tsv_data_in)): # Execute the function - out_map = ecm_prep.tsv_cost_carb_yrmap( + out_map = ECMUtils.tsv_cost_carb_yrmap( self.test_tsv_data_in[ind], self.test_aeo_years_in) # Check function outputs self.dict_check(out_map, self.test_out_map[ind]) From 46d393bbf5504968013e7d1a6a0a4080be8ca93b Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Fri, 11 Apr 2025 17:03:30 -0600 Subject: [PATCH 5/8] Pass logger to verboseprint method, style fixes --- scout/ecm_prep.py | 56 ++++++++++++++++++++++++++--------------------- scout/run.py | 2 +- scout/utils.py | 6 ++++- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index 738abd455..0cad0232a 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -142,7 +142,7 @@ def prep_error(meas_name, handyvars, handyfiles): # Add ECM to skipped list handyvars.skipped_ecms.append(meas_name) # Print error message if in verbose mode - # fmt.verboseprint(opts.verbose, err_msg, "error") + # fmt.verboseprint(opts.verbose, err_msg, "error", logger) # # Log error message to file (see ./generated) logger.error(err_msg) @@ -2220,7 +2220,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, opts.verbose, f"ECM {self.name} missing valid baseline stock/energy data for technology " f"'{str(mskeys[-2])}'; removing technology from analysis", - "warning") + "warning", + logger) # Add to the overall number of key chains that yield "stock"/ # "energy" keys (but in this case, are missing data) valid_keys += 1 @@ -2752,7 +2753,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"ECM '{self.name}' uses invalid performance units for " f"technology '{str(mskeys[-2])}' (requires " f"{str(perf_base_units)}); removing technology from analysis", - "warning") + "warning", + logger) # Continue to the next microsegment continue # Handle case where measure units do not equal baseline @@ -2777,7 +2779,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"{str(perf_base_units)}); base units changed to " f"{str(perf_units)} and base values multiplied by " f"{str(convert_fact)}", - "warning") + "warning", + logger) # Convert base performance values to values in # measure performance units perf_base = {yr: (perf_base[yr] * convert_fact) for @@ -2830,7 +2833,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, "residential heating and cooling end uses (both are divided " "by 2 when separately considered across heating and cooling " "in the raw EIA data)", - "warning") + "warning", + logger) # Adjust residential baseline lighting lifetimes to # reflect the fact that input data assume 24 h/day of # lighting use, rather than 3 h/day as assumed for @@ -2900,7 +2904,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"/lifetime data for technology '{str(mskeys[-2])}'; " "technology will remain in analysis with cost of zero; " "if lifetime data are missing, lifetime is set to 10 years", - "warning") + "warning", + logger) # In all other cases, to avoid removing any msegs, # set the baseline cost and performance to the measure @@ -2944,7 +2949,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, "technology applies to special lighting case and will " "remain in analysis at same cost/performance as ECM; if " "lifetime data are missing, lifetime is set to 10 years", - "warning") + "warning", + logger) else: # Set baseline cost and performance characteristics for any # remaining secondary microsegments to that of the measure @@ -3183,7 +3189,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"ECM '{self.name}' has baseline or measure " "performance of zero; baseline and measure " "performance set equal", - "warning") + "warning", + logger) # Ensure that the adjusted relative savings # fraction is not greater than 1 or less # than 0 if not originally specified as @@ -3236,7 +3243,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, opts.verbose, f"ECM '{self.name}' has measure performance of zero; " "baseline and measure performance set equal", - "warning") + "warning", + logger) rel_perf[yr] = 1 # Ensure that relative performance is a finite # number; if not, set to 1 @@ -3256,7 +3264,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, opts.verbose, f"ECM '{self.name}' has baseline performance of zero; " "baseline and measure performance set equal", - "warning") + "warning", + logger) rel_perf[yr] = 1 # If looping through a commercial lighting microsegment @@ -3516,7 +3525,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"ECM '{self.name}' missing valid consumer choice " f"data for end use '{str(mskeys[4])}'; using default " "choice data for refrigeration end use", - "warning") + "warning", + logger) choice_params = { "b1": { key: self.handyvars.deflt_choice[0] for @@ -3568,7 +3578,8 @@ def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, f"ECM '{self.name}' missing valid consumer choice data for " f"end use '{str(mskeys[4])}'; using default choice data for " "refrigeration end use", - "warning") + "warning", + logger) choice_params = {"rate distribution": self.handyvars.com_timeprefs[ "distributions"][ @@ -5289,7 +5300,8 @@ def apply_tsv(self, load_fact, ash_cz_wts, eplus_bldg_wts, "check that 8760 hourly savings fractions are available " "for all baseline market segments the measure applies to " f"in {self.handyfiles.tsv_shape_data}.", - "warning") + "warning", + logger) else: # Develop an adjustment from the generic @@ -5898,7 +5910,7 @@ def convert_costs(self, convert_data, bldg_sect, mskeys, cost_meas, user_message += " for building type '" + mskeys[2] + "'" # Print user message - fmt.verboseprint(verbose, user_message, "info") + fmt.verboseprint(verbose, user_message, "info", logger) # Case where cost conversion has not succeeded else: raise ValueError( @@ -6275,7 +6287,8 @@ def partition_microsegment( f"No data available to link mseg {str(mskeys)} for measure '{self.name}' " f"with {self.linked_htcl_tover_anchor_tech} " f"{self.linked_htcl_tover_anchor_eu} turnover rates; unlinking turnover", - "warning") + "warning", + logger) # In cases where no secondary heating/cooling microsegment is present, # and there are no linked stock turnover rates for primary heating and # cooling microsegments, set relevant adjustment variables to None @@ -9225,7 +9238,7 @@ def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, # Create a shorthand for baseline and efficient stock/energy/carbon/ # cost data to add to the breakout dict base_data = [add_stock_total, add_energy_total, - add_energy_cost, add_carb_total] + add_energy_cost, add_carb_total] eff_data = [add_stock_total_meas, add_energy_total_eff, add_energy_cost_eff, add_carb_total_eff] @@ -9484,7 +9497,8 @@ def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, opts.verbose, f"Baseline market key chain: '{str(mskeys)}' for ECM '{self.name}' does not map to " "output breakout categories, thus will not be reflected in output breakout data", - "warning") + "warning", + logger) class MeasurePackage(Measure): @@ -11862,14 +11876,6 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] logger.info("Measure initialization complete") - print('Initializing measures...', end="", flush=True) - # Translate user options to a dictionary for further use in Measures - opts_dict = vars(opts) - # Initialize Measure() objects based on 'measures_update' list - meas_update_objs = [Measure( - base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] - print("Complete") - # Fill in EnergyPlus-based performance information for Measure objects # with a 'From EnergyPlus' flag in their 'energy_efficiency' attribute diff --git a/scout/run.py b/scout/run.py index 21c81b818..203e7d345 100644 --- a/scout/run.py +++ b/scout/run.py @@ -5365,7 +5365,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Reset measure fuel split attribute to imported values m.eff_fs_splt = meas_eff_fs_data # Print data import message for each ECM if in verbose mode - fmt.verboseprint(opts.verbose, f"Imported ECM {m.name} competition data") + fmt.verboseprint(opts.verbose, f"Imported ECM {m.name} competition data", "info") # Import total absolute heating and cooling energy use data, used in # removing overlaps between supply-side and demand-side heating/cooling diff --git a/scout/utils.py b/scout/utils.py index b9966657a..17000b5ad 100644 --- a/scout/utils.py +++ b/scout/utils.py @@ -1,5 +1,6 @@ import json import numpy +import logging from pathlib import Path, PurePath @@ -58,15 +59,18 @@ def custom_showwarning(message, category, filename, lineno, file=None, line=None print(message) @staticmethod - def verboseprint(verbose, msg, log_type): + def verboseprint(verbose, msg, log_type, logger=None): """Print input message when the code is run in verbose mode. Args: verbose (boolean): Indicator of verbose mode msg (string): Message to print to console when in verbose mode + logger: Logger instance to use for logging """ if not verbose: return + if not logger: + logger = logging.getLogger(__name__) if log_type == "info": logger.info(msg) From ff1da713a2bbba405c8817466180961ad5151728 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Thu, 17 Apr 2025 16:54:34 -0600 Subject: [PATCH 6/8] Remove EnergyPlus mapping functions and tests. --- scout/ecm_prep.py | 277 +------------------- scout/ecm_prep_vars.py | 87 ------- tests/ecm_prep_test.py | 565 ----------------------------------------- 3 files changed, 1 insertion(+), 928 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index 0cad0232a..f702b52fd 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -20,7 +20,7 @@ from pathlib import Path import argparse from scout.ecm_prep_args import ecm_args -from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles, EPlusMapDicts +from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles from scout.utils import JsonIO, PrintFormat as fmt from scout.config import LogConfig, FilePaths as fp import traceback @@ -795,73 +795,6 @@ def __init__( "mseg_out_break"]["energy"]["efficient-captured"] = \ copy.deepcopy(self.handyvars.out_break_in) - def fill_eplus(self, msegs, eplus_dir, eplus_coltypes, - eplus_files, vintage_weights, base_cols): - """Fill in measure performance with EnergyPlus simulation results. - - Note: - Find the appropriate set of EnergyPlus simulation results for - the current measure, and use the relative savings percentages - in these results to determine the measure performance attribute. - - Args: - msegs (dict): Baseline microsegment stock/energy data to use in - validating categorization of measure performance information. - eplus_dir (string): Directory of EnergyPlus performance files. - eplus_coltypes (list): Expected EnergyPlus variable data types. - eplus_files (list): EnergyPlus performance file names. - vintage_weights (dict): Square-footage-derived weighting factors - for each EnergyPlus building vintage type. - - Returns: - Updated Measure energy_efficiency, energy_efficiency_source, and - energy_efficiency_source attribute values. - - Raises: - ValueError: If EnergyPlus file is not matched to Measure - definition or more than one EnergyPlus file matches the - Measure definition. - """ - # Instantiate useful EnergyPlus-Scout mapping dicts - handydicts = EPlusMapDicts() - # Determine the relevant EnergyPlus building type name(s) - bldg_type_names = [] - for x in self.bldg_type: - bldg_type_names.extend(handydicts.bldgtype[x].keys()) - # Find all EnergyPlus files including the relevant building type - # name(s) - eplus_perf_in = [(eplus_dir / x) for x in eplus_files if any([ - y.lower() in x for y in bldg_type_names])] - - # Import EnergyPlus input file as array and use it to fill a dict - # of measure performance data - if len(eplus_perf_in) > 0: - # Assemble the EnergyPlus data into a record array - eplus_perf_array = self.build_array(eplus_coltypes, eplus_perf_in) - # Create a measure performance dictionary, zeroed out, to - # be updated with data from EnergyPlus array - perf_dict_empty = self.create_perf_dict(msegs) - - # Update measure performance based on EnergyPlus data - # (* Note: only update efficiency information for - # secondary microsegments if applicable) - if perf_dict_empty['secondary'] is not None: - self.energy_efficiency = self.fill_perf_dict( - perf_dict_empty, eplus_perf_array, - vintage_weights, base_cols, eplus_bldg_types={}) - else: - self.energy_efficiency = self.fill_perf_dict( - perf_dict_empty['primary'], eplus_perf_array, - vintage_weights, base_cols, eplus_bldg_types={}) - # Set the energy efficiency data source for the measure to - # EnergyPlus and set to highest data quality rating - self.energy_efficiency_source = 'EnergyPlus/OpenStudio' - else: - raise ValueError( - "Failure to find relevant EPlus files for " + - "Scout building type(s) " + str(self.bldg_type) + - "in ECM '" + self.name + "'") - def fill_mkts(self, msegs, msegs_cpl, convert_data, tsv_data_init, opts, ctrb_ms_pkg_prep, tsv_data_nonfs): """Fill in a measure's market microsegments using EIA baseline data. @@ -8698,214 +8631,6 @@ def rand_list_gen(self, distrib_info, nsamples): return rand_list - def fill_perf_dict( - self, perf_dict, eplus_perf_array, vintage_weights, - base_cols, eplus_bldg_types): - """Fill an empty dict with updated measure performance information. - - Note: - Use structured array data drawn from an EnergyPlus output file - and building type/vintage weighting data to fill a dictionary of - final measure performance information. - - Args: - perf_dict (dict): Empty dictionary to fill with EnergyPlus-based - performance information broken down by climate zone, building - type/vintage, fuel type, and end use. - eplus_perf_array (numpy recarray): Structured array of EnergyPlus - energy savings information for the Measure. - vintage_weights (dict): Square-footage-derived weighting factors - for each EnergyPlus building vintage type. - eplus_bldg_types (dict): Scout-EnergyPlus building type mapping - data, including weighting factors needed to map multiple - EnergyPlus building types to a single Scout building type. - Drawn from EPlusMapDicts object's 'bldgtype' attribute. - - Returns: - A measure performance dictionary filled with relative energy - savings values from an EnergyPlus simulation output file. - - Raises: - KeyError: If an EnergyPlus category name cannot be mapped to the - input perf_dict keys. - ValueError: If weights used to map multiple EnergyPlus reference - building types to a single Scout building type do not sum to 1. - """ - # Instantiate useful EnergyPlus-Scout mapping dicts - handydicts = EPlusMapDicts() - - # Set the header of the EnergyPlus input array (used when reducing - # columns of the array to the specific performance categories being - # updated below) - eplus_header = list(eplus_perf_array.dtype.names) - - # Loop through zeroed out measure performance dictionary input and - # update the values with data from the EnergyPlus input array - for key, item in perf_dict.items(): - # If current dict item is itself a dict, reduce EnergyPlus array - # based on the current dict key and proceed further down the dict - # levels - if isinstance(item, dict): - # Microsegment type level (no update to EnergyPlus array - # required) - if key in ['primary', 'secondary']: - updated_perf_array = eplus_perf_array - # Climate zone level - elif key in handydicts.czone.keys(): - # Reduce EnergyPlus array to only rows with climate zone - # currently being updated in the performance dictionary - updated_perf_array = eplus_perf_array[numpy.where( - eplus_perf_array[ - 'climate_zone'] == handydicts.czone[key])].copy() - if len(updated_perf_array) == 0: - raise KeyError( - "EPlus climate zone name not found for ECM '" + - self.name + "'") - # Building type level - elif key in handydicts.bldgtype.keys(): - # Determine relevant EnergyPlus building types for current - # Scout building type - eplus_bldg_types = handydicts.bldgtype[key] - if sum(eplus_bldg_types.values()) != 1: - raise ValueError( - "EPlus building type weights do not sum to 1 " - "for ECM '" + self.name + "'") - # Reduce EnergyPlus array to only rows with building type - # relevant to current Scout building type - updated_perf_array = eplus_perf_array[numpy.in1d( - eplus_perf_array['building_type'], - list(eplus_bldg_types.keys()))].copy() - if len(updated_perf_array) == 0: - raise KeyError( - "EPlus building type name not found for ECM '" + - self.name + "'") - # Fuel type level - elif key in handydicts.fuel.keys(): - # Reduce EnergyPlus array to only columns with fuel type - # currently being updated in the performance dictionary, - # plus bldg. type/vintage, climate, and measure columns - colnames = base_cols + [ - x for x in eplus_header if handydicts.fuel[key] in x] - if len(colnames) == len(base_cols): - raise KeyError( - "EPlus fuel type name not found for ECM '" + - self.name + "'") - updated_perf_array = eplus_perf_array[colnames].copy() - # End use level - elif key in handydicts.enduse.keys(): - # Reduce EnergyPlus array to only columns with end use - # currently being updated in the performance dictionary, - # plus bldg. type/vintage, climate, and measure columns - colnames = base_cols + [ - x for x in eplus_header if x in handydicts.enduse[ - key]] - if len(colnames) == len(base_cols): - raise KeyError( - "EPlus end use name not found for ECM '" + - self.name + "'") - updated_perf_array = eplus_perf_array[colnames].copy() - else: - raise KeyError( - "Invalid performance dict key for ECM '" + - self.name + "'") - - # Given updated EnergyPlus array, proceed further down the - # dict level hierarchy - self.fill_perf_dict( - item, updated_perf_array, vintage_weights, - base_cols, eplus_bldg_types) - else: - # Reduce EnergyPlus array to only rows with structure type - # currently being updated in the performance dictionary - # ('new' or 'retrofit') - if key in handydicts.structure_type.keys(): - # A 'new' structure type will match only one of the - # EnergyPlus building vintage names - if key == "new": - updated_perf_array = eplus_perf_array[numpy.where( - eplus_perf_array['template'] == - handydicts.structure_type['new'])].copy() - # A 'retrofit' structure type will match multiple - # EnergyPlus building vintage names - else: - updated_perf_array = eplus_perf_array[numpy.in1d( - eplus_perf_array['template'], list( - handydicts.structure_type[ - 'retrofit'].keys()))].copy() - if len(updated_perf_array) == 0 or \ - (key == "new" and - len(numpy.unique(updated_perf_array[ - 'template'])) != 1 or key == "retrofit" and - len(numpy.unique(updated_perf_array[ - 'template'])) != len( - handydicts.structure_type["retrofit"].keys())): - raise ValueError( - "EPlus vintage name not found for ECM '" + - self.name + "'") - else: - raise KeyError( - "Invalid performance dict key for ECM '" + - self.name + "'") - - # Separate filtered array into the rows representing measure - # consumption and those representing baseline consumption - updated_perf_array_m, updated_perf_array_b = [ - updated_perf_array[updated_perf_array[ - 'measure'] != 'none'], - updated_perf_array[updated_perf_array[ - 'measure'] == 'none']] - # Ensure that a baseline consumption row exists for every - # measure consumption row retrieved - if len(updated_perf_array_m) != len(updated_perf_array_b): - raise ValueError( - "Lengths of ECM and baseline EPlus data arrays " - "are unequal for ECM '" + self.name + "'") - # Initialize total measure and baseline consumption values - val_m, val_b = (0 for n in range(2)) - - # Weight and combine the measure/baseline consumption values - # left in the EnergyPlus arrays; subtract total measure - # consumption from baseline consumption and divide by baseline - # consumption to reach relative savings value for the current - # dictionary branch - for ind in range(0, len(updated_perf_array_m)): - row_m, row_b = [ - updated_perf_array_m[ind], updated_perf_array_b[ind]] - # Loop through remaining columns with consumption data - for n in eplus_header: - if row_m[n].dtype.char != 'S' and \ - row_m[n].dtype.char != 'U': - # Find appropriate building type to weight - # consumption data points by - eplus_bldg_type_wt_row_m, \ - eplus_bldg_type_wt_row_b = [ - eplus_bldg_types[row_m['building_type']], - eplus_bldg_types[row_b['building_type']]] - # Weight consumption data points by factors for - # appropriate building type and vintage - row_m_val, row_b_val = [( - row_m[n] * eplus_bldg_type_wt_row_m * - vintage_weights[row_m['template'].copy()]), - (row_b[n] * eplus_bldg_type_wt_row_b * - vintage_weights[row_b['template'].copy()])] - # Add weighted measure consumption data point to - # total measure consumption - val_m += row_m_val - # Add weighted baseline consumption data point to - # total base consumption - val_b += row_b_val - # Find relative savings if total baseline use != zero - if val_b != 0: - end_key_val = (val_b - val_m) / val_b - else: - end_key_val = 0 - - # Update the current dictionary branch value to the final - # measure relative savings value derived above - perf_dict[key] = round(end_key_val, 3) - - return perf_dict - def create_perf_dict(self, msegs): """Create dict to fill with updated measure performance information. diff --git a/scout/ecm_prep_vars.py b/scout/ecm_prep_vars.py index 590e95fdc..9b0638250 100644 --- a/scout/ecm_prep_vars.py +++ b/scout/ecm_prep_vars.py @@ -1702,90 +1702,3 @@ def get_suffix(arg): f"tsv_carbon-{opts.alt_regions.lower()}-MidCase.json") self.ss_data_nonfs, self.tsv_cost_data_nonfs, \ self.tsv_carbon_data_nonfs = (None for n in range(3)) - - -class EPlusMapDicts(object): - """Class of dicts used to map Scout measure definitions to EnergyPlus. - - Attributes: - czone (dict): Scout-EnergyPlus climate zone mapping. - bldgtype (dict): Scout-EnergyPlus building type mapping. Shown are - the EnergyPlus commercial reference building names that correspond - to each AEO commercial building type, and the weights needed in - some cases to map multiple EnergyPlus reference building types to - a single AEO type. See 'convert_data' JSON for more details. - fuel (dict): Scout-EnergyPlus fuel type mapping. - enduse (dict): Scout-EnergyPlus end use mapping. - structure_type (dict): Scout-EnergyPlus structure type mapping. - """ - - def __init__(self): - self.czone = { - "sub arctic": "BA-SubArctic", - "very cold": "BA-VeryCold", - "cold": "BA-Cold", - "marine": "BA-Marine", - "mixed humid": "BA-MixedHumid", - "mixed dry": "BA-MixedDry", - "hot dry": "BA-HotDry", - "hot humid": "BA-HotHumid"} - self.bldgtype = { - "assembly": { - "Hospital": 1}, - "education": { - "PrimarySchool": 0.26, - "SecondarySchool": 0.74}, - "food sales": { - "Supermarket": 1}, - "food service": { - "QuickServiceRestaurant": 0.31, - "FullServiceRestaurant": 0.69}, - "health care": None, - "lodging": { - "SmallHotel": 0.26, - "LargeHotel": 0.74}, - "large office": { - "LargeOfficeDetailed": 0.9, - "MediumOfficeDetailed": 0.1}, - "small office": { - "SmallOffice": 0.12, - "OutpatientHealthcare": 0.88}, - "mercantile/service": { - "RetailStandalone": 0.53, - "RetailStripmall": 0.47}, - "warehouse": { - "Warehouse": 1}, - "other": None, - "unspecified": None} - self.fuel = { - 'electricity': 'electricity', - 'natural gas': 'gas', - 'distillate': 'other_fuel'} - self.enduse = { - 'heating': [ - 'heating_electricity', 'heat_recovery_electricity', - 'humidification_electricity', 'pump_electricity', - 'heating_gas', 'heating_other_fuel'], - 'cooling': [ - 'cooling_electricity', 'pump_electricity', - 'heat_rejection_electricity'], - 'water heating': [ - 'service_water_heating_electricity', - 'service_water_heating_gas', - 'service_water_heating_other_fuel'], - 'ventilation': ['fan_electricity'], - 'cooking': [ - 'interior_equipment_gas', 'interior_equipment_other_fuel'], - 'lighting': ['interior_lighting_electricity'], - 'refrigeration': ['refrigeration_electricity'], - 'PCs': ['interior_equipment_electricity'], - 'non-PC office equipment': ['interior_equipment_electricity'], - 'MELs': ['interior_equipment_electricity']} - # Note: assumed year range for each structure vintage shown in lists - self.structure_type = { - "new": '90.1-2013', - "retrofit": { - '90.1-2004': [2004, 2009], - '90.1-2010': [2010, 2012], - 'DOE Ref 1980-2004': [1980, 2003], - 'DOE Ref Pre-1980': [0, 1979]}} diff --git a/tests/ecm_prep_test.py b/tests/ecm_prep_test.py index 5195aac81..333305c55 100644 --- a/tests/ecm_prep_test.py +++ b/tests/ecm_prep_test.py @@ -12,7 +12,6 @@ import unittest import numpy import os -from collections import OrderedDict import warnings import copy import json @@ -143,570 +142,6 @@ def __init__(self): self.opts_dict = vars(self.opts) -class EPlusUpdateTest(unittest.TestCase, CommonMethods): - """Test the 'fill_eplus' function and its supporting functions. - - Ensure that the 'build_array' function properly assembles a set of input - CSVs into a structured array and that the 'create_perf_dict' and - 'fill_perf_dict' functions properly initialize and fill a measure - performance dictionary with results from an EnergyPlus simulation output - file. - - Attributes: - meas (object): Measure object instantiated based on sample_measure_in - attributes. - eplus_dir (string): EnergyPlus simulation output file directory. - eplus_coltypes (list): List of expected EnergyPlus output data types. - eplus_basecols (list): Variable columns that should never be removed. - mseg_in (dict): Sample baseline microsegment stock/energy data. - ok_eplus_vintagewts (dict): Sample EnergyPlus vintage weights. - ok_eplusfiles_in (list): List of all EnergyPlus simulation file names. - ok_perfarray_in (numpy recarray): Valid structured array of - EnergyPlus-based relative savings data. - fail_perfarray_in (numpy recarray): Invalid structured array of - EnergyPlus-based relative savings data (missing certain climate - zones, building types, and building vintages). - fail_perfdictempty_in (dict): Invalid empty dictionary to fill with - EnergyPlus-based performance information broken down by climate - zone, building type/vintage, fuel type, and end use (dictionary - includes invalid climate zone key). - ok_array_type_out (string): The array type that should be yielded by - 'convert_to_array' given valid input. - ok_array_length_out (int): The array length that should be yielded by - 'convert_to_array' given valid input. - ok_array_names_out (tuple): Tuple of column names for the recarray that - should be yielded by 'convert_to_array' given valid input. - ok_perfdictempty_out (dict): The empty dictionary that should be - yielded by 'create_perf_dict' given valid inputs. - ok_perfdictfill_out (dict): The dictionary filled with EnergyPlus-based - measure performance information that should be yielded by - 'fill_perf_dict' and 'fill_eplus' given valid inputs. - - Raises: - AssertionError: If function yields unexpected results or does not - raise a KeyError when it should. - """ - - @classmethod - def setUpClass(cls): - """Define variables and objects for use across all class functions.""" - # Sample measure attributes to use in instantiating Measure object. - sample_measure_in = OrderedDict([ - ("name", "eplus sample measure 1"), - ("status", OrderedDict([ - ("active", 1), ("updated", 1)])), - ("installed_cost", 25), - ("cost_units", "2014$/unit"), - ("energy_efficiency", OrderedDict([ - ("EnergyPlus file", "eplus_sample_measure")])), - ("energy_efficiency_units", OrderedDict([ - ("primary", "relative savings (constant)"), - ("secondary", "relative savings (constant)")])), - ("energy_efficiency_source", None), - ("market_entry_year", None), - ("market_exit_year", None), - ("product_lifetime", 10), - ("structure_type", ["new", "retrofit"]), - ("bldg_type", ["assembly", "education"]), - ("climate_zone", ["hot dry", "mixed humid"]), - ("fuel_type", OrderedDict([ - ("primary", ["electricity"]), - ("secondary", [ - "electricity", "natural gas", "distillate"])])), - ("fuel_switch_to", None), - ("end_use", OrderedDict([ - ("primary", ["lighting"]), - ("secondary", ["heating", "cooling"])])), - ("technology", OrderedDict([ - ("primary", [ - "technology A", "technology B", "technology C"]), - ("secondary", ["windows conduction", "windows solar"])]))]) - # Base directory - base_dir = os.getcwd() - # Null user options/options dict - opts, opts_dict = (NullOpts().opts, NullOpts().opts_dict) - # Useful global variables for the sample measure object - handyfiles = UsefulInputFiles(opts) - handyvars = UsefulVars(base_dir, handyfiles, opts) - cls.meas = Measure( - base_dir, handyvars, handyfiles, opts_dict, **sample_measure_in) - # Finalize the measure's 'technology_type' attribute (handled by the - # 'fill_attr' function, which is not run as part of this test) - cls.meas.technology_type = {"primary": "supply", "secondary": "demand"} - cls.eplus_dir = fp.ECM_DEF / "energyplus_data" / "energyplus_test_ok" - cls.eplus_coltypes = [ - ('building_type', ' Date: Mon, 14 Jul 2025 10:44:46 -0600 Subject: [PATCH 7/8] Rename ECMUtils, move method into it --- scout/ecm_prep.py | 276 +++++++++++++++++++++-------------------- scout/run_batch.py | 11 +- tests/ecm_prep_test.py | 23 ++-- 3 files changed, 159 insertions(+), 151 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index f702b52fd..fbdeeca1f 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) -class ECMUtils: +class ECMPrepHelper: """Shared methods used throughout ecm_prep.py""" @staticmethod @@ -73,8 +73,8 @@ def initialize_run_setup(input_files: UsefulInputFiles) -> dict: f"Error reading in '{input_files.run_setup}': {str(e)}") from None am.close() # Initialize all measures as inactive - run_setup = ECMUtils.update_active_measures(run_setup, - to_inactive=run_setup["active"]) + run_setup = ECMPrepHelper.update_active_measures(run_setup, + to_inactive=run_setup["active"]) except FileNotFoundError: run_setup = {"active": [], "inactive": [], "skipped": []} @@ -247,6 +247,118 @@ def tsv_cost_carb_yrmap(tsv_data, aeo_years): return tsv_yr_map + @staticmethod + def split_clean_data(meas_prepped_objs, full_dat_out): + """Reorganize and remove data from input Measure objects. + + Note: + The input Measure objects have updated data, which must + be reorganized/condensed for the purposes of writing out + to JSON files. + + Args: + meas_prepped_objs (object): Measure objects with data to + be split in to separate dicts or removed. + full_dat_out (dict): Flag that limits technical potential (TP) data + prep/reporting when TP is not in user-specified adoption schemes. + + Returns: + Three to four lists of dicts, one containing competition data for + each updated measure, one containing high level summary + data for each updated measure, another containing sector shape + data for each measure (if applicable), and a final one containing + efficient fuel split data, as applicable to fuel switching measures + when the user has required fuel splits. + """ + # Initialize lists of measure competition/summary data + meas_prepped_compete = [] + meas_prepped_summary = [] + meas_prepped_shapes = [] + meas_eff_fs_splt = [] + # Loop through all Measure objects and reorganize/remove the + # needed data. + for m in meas_prepped_objs: + # Initialize a reorganized measure competition data dict and efficient + # fuel split data dict + comp_data_dict, fs_splits_dict, shapes_dict = ({} for n in range(3)) + # Retrieve measure contributing microsegment data that are relevant to + # markets competition in the analysis engine, then remove these data + # from measure object + for adopt_scheme in m.handyvars.adopt_schemes_prep: + # Delete contributing microsegment data that are + # not relevant to competition in the analysis engine + del m.markets[adopt_scheme]["mseg_adjust"][ + "secondary mseg adjustments"]["sub-market"] + del m.markets[adopt_scheme]["mseg_adjust"][ + "secondary mseg adjustments"]["stock-and-flow"] + # If individual measure, delete markets data used to linked + # heating/cooling turnover and switching rates across msegs (these + # data are not prepared for packages) + if not isinstance(m, MeasurePackage): + del m.markets[adopt_scheme]["mseg_adjust"][ + "paired heat/cool mseg adjustments"] + # Add remaining contributing microsegment data to + # competition data dict, if the adoption scenario will be competed + # in the run.py module, then delete from measure + if full_dat_out[adopt_scheme]: + comp_data_dict[adopt_scheme] = \ + m.markets[adopt_scheme]["mseg_adjust"] + # If applicable, add efficient fuel split data to fuel split + # data dict + if len(m.eff_fs_splt[adopt_scheme].keys()) != 0: + fs_splits_dict[adopt_scheme] = \ + m.eff_fs_splt[adopt_scheme] + # If applicable, add sector shape data + if m.sector_shapes is not None and len( + m.sector_shapes[adopt_scheme].keys()) != 0: + shapes_dict["name"] = m.name + shapes_dict[adopt_scheme] = \ + m.sector_shapes[adopt_scheme] + else: + # If adoption scenario will not be competed in the run.py + # module, remove detailed mseg breakouts + del m.markets[adopt_scheme]["mseg_out_break"] + del m.markets[adopt_scheme]["mseg_adjust"] + # Delete info. about efficient fuel splits for fuel switch measures + del m.eff_fs_splt + # Delete info. about sector shapes + del m.sector_shapes + + # Append updated competition data from measure to + # list of competition data across all measures + meas_prepped_compete.append(comp_data_dict) + # Append fuel switching split information, if applicable + meas_eff_fs_splt.append(fs_splits_dict) + # Append sector shape information, if applicable + meas_prepped_shapes.append(shapes_dict) + # Delete 'handyvars' measure attribute (not relevant to + # analysis engine) + del m.handyvars + # Delete 'tsv_features' measure attributes + # (not relevant) for individual measures + if not isinstance(m, MeasurePackage): + del m.tsv_features + # Delete individual measure attributes used to link heating/ + # cooling microsegment turnover and switching rates + del m.linked_htcl_tover + del m.linked_htcl_tover_anchor_eu + del m.linked_htcl_tover_anchor_tech + # For measure packages, replace 'contributing_ECMs' + # objects list with a list of these measures' names and remove + # unnecessary heating/cooling equip/env overlap data + if isinstance(m, MeasurePackage): + m.contributing_ECMs = [ + x.name for x in m.contributing_ECMs] + del m.htcl_overlaps + del m.contributing_ECMs_eqp + del m.contributing_ECMs_env + # Append updated measure __dict__ attribute to list of + # summary data across all measures + meas_prepped_summary.append(m.__dict__) + + return meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ + meas_eff_fs_splt + class Measure(object): """Set up a class representing efficiency measures as objects. @@ -11627,7 +11739,7 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, # are valid before attempting to retrieve data on this baseline market m.check_meas_inputs() except Exception: - ECMUtils.prep_error(m.name, handyvars, handyfiles) + ECMPrepHelper.prep_error(m.name, handyvars, handyfiles) # Add measure index to removal list remove_inds.append(m_ind) @@ -11639,7 +11751,7 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, msegs, msegs_cpl, convert_data, tsv_data, opts, ctrb_ms_pkg_prep, tsv_data_nonfs) except Exception: - ECMUtils.prep_error(m.name, handyvars, handyfiles) + ECMPrepHelper.prep_error(m.name, handyvars, handyfiles) # Add measure index to removal list remove_inds.append(m_ind) @@ -11775,122 +11887,10 @@ def prepare_packages(packages, meas_update_objs, meas_summary, if packaged_measure is not False: meas_update_objs.append(packaged_measure) except Exception: - ECMUtils.prep_error(p["name"], handyvars, handyfiles) + ECMPrepHelper.prep_error(p["name"], handyvars, handyfiles) return meas_update_objs - @staticmethod - def split_clean_data(meas_prepped_objs, full_dat_out): - """Reorganize and remove data from input Measure objects. - - Note: - The input Measure objects have updated data, which must - be reorganized/condensed for the purposes of writing out - to JSON files. - - Args: - meas_prepped_objs (object): Measure objects with data to - be split in to separate dicts or removed. - full_dat_out (dict): Flag that limits technical potential (TP) data - prep/reporting when TP is not in user-specified adoption schemes. - - Returns: - Three to four lists of dicts, one containing competition data for - each updated measure, one containing high level summary - data for each updated measure, another containing sector shape - data for each measure (if applicable), and a final one containing - efficient fuel split data, as applicable to fuel switching measures - when the user has required fuel splits. - """ - # Initialize lists of measure competition/summary data - meas_prepped_compete = [] - meas_prepped_summary = [] - meas_prepped_shapes = [] - meas_eff_fs_splt = [] - # Loop through all Measure objects and reorganize/remove the - # needed data. - for m in meas_prepped_objs: - # Initialize a reorganized measure competition data dict and efficient - # fuel split data dict - comp_data_dict, fs_splits_dict, shapes_dict = ({} for n in range(3)) - # Retrieve measure contributing microsegment data that are relevant to - # markets competition in the analysis engine, then remove these data - # from measure object - for adopt_scheme in m.handyvars.adopt_schemes_prep: - # Delete contributing microsegment data that are - # not relevant to competition in the analysis engine - del m.markets[adopt_scheme]["mseg_adjust"][ - "secondary mseg adjustments"]["sub-market"] - del m.markets[adopt_scheme]["mseg_adjust"][ - "secondary mseg adjustments"]["stock-and-flow"] - # If individual measure, delete markets data used to linked - # heating/cooling turnover and switching rates across msegs (these - # data are not prepared for packages) - if not isinstance(m, MeasurePackage): - del m.markets[adopt_scheme]["mseg_adjust"][ - "paired heat/cool mseg adjustments"] - # Add remaining contributing microsegment data to - # competition data dict, if the adoption scenario will be competed - # in the run.py module, then delete from measure - if full_dat_out[adopt_scheme]: - comp_data_dict[adopt_scheme] = \ - m.markets[adopt_scheme]["mseg_adjust"] - # If applicable, add efficient fuel split data to fuel split - # data dict - if len(m.eff_fs_splt[adopt_scheme].keys()) != 0: - fs_splits_dict[adopt_scheme] = \ - m.eff_fs_splt[adopt_scheme] - # If applicable, add sector shape data - if m.sector_shapes is not None and len( - m.sector_shapes[adopt_scheme].keys()) != 0: - shapes_dict["name"] = m.name - shapes_dict[adopt_scheme] = \ - m.sector_shapes[adopt_scheme] - else: - # If adoption scenario will not be competed in the run.py - # module, remove detailed mseg breakouts - del m.markets[adopt_scheme]["mseg_out_break"] - del m.markets[adopt_scheme]["mseg_adjust"] - # Delete info. about efficient fuel splits for fuel switch measures - del m.eff_fs_splt - # Delete info. about sector shapes - del m.sector_shapes - - # Append updated competition data from measure to - # list of competition data across all measures - meas_prepped_compete.append(comp_data_dict) - # Append fuel switching split information, if applicable - meas_eff_fs_splt.append(fs_splits_dict) - # Append sector shape information, if applicable - meas_prepped_shapes.append(shapes_dict) - # Delete 'handyvars' measure attribute (not relevant to - # analysis engine) - del m.handyvars - # Delete 'tsv_features' measure attributes - # (not relevant) for individual measures - if not isinstance(m, MeasurePackage): - del m.tsv_features - # Delete individual measure attributes used to link heating/ - # cooling microsegment turnover and switching rates - del m.linked_htcl_tover - del m.linked_htcl_tover_anchor_eu - del m.linked_htcl_tover_anchor_tech - # For measure packages, replace 'contributing_ECMs' - # objects list with a list of these measures' names and remove - # unnecessary heating/cooling equip/env overlap data - if isinstance(m, MeasurePackage): - m.contributing_ECMs = [ - x.name for x in m.contributing_ECMs] - del m.htcl_overlaps - del m.contributing_ECMs_eqp - del m.contributing_ECMs_env - # Append updated measure __dict__ attribute to list of - # summary data across all measures - meas_prepped_summary.append(m.__dict__) - - return meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ - meas_eff_fs_splt - def main(opts: argparse.NameSpace): # noqa: F821 """Import and prepare measure attributes for analysis engine. @@ -11906,7 +11906,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 """ # Configure logger specific to ecm_prep - ECMUtils.configure_ecm_prep_logger() + ECMPrepHelper.configure_ecm_prep_logger() # Set current working directory base_dir = getcwd() @@ -11936,8 +11936,8 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Import packages JSON, filter as needed meas_toprep_package_init = JsonIO.load_json(handyfiles.ecm_packages) - meas_toprep_package_init = ECMUtils.downselect_packages(meas_toprep_package_init, - opts.ecm_packages) + meas_toprep_package_init = ECMPrepHelper.downselect_packages(meas_toprep_package_init, + opts.ecm_packages) # If applicable, import file to write prepared measure sector shapes to # (if file does not exist, provide empty list as substitute, since file @@ -11954,9 +11954,9 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_shapes = [] # Determine full list of individual measure JSON names - meas_toprep_indiv_names = ECMUtils.retrieve_valid_ecms(meas_toprep_package_init, - opts, - handyfiles) + meas_toprep_indiv_names = ECMPrepHelper.retrieve_valid_ecms(meas_toprep_package_init, + opts, + handyfiles) # Initialize list of all individual measures that require updates meas_toprep_indiv = [] @@ -12292,7 +12292,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_prepped_pkgs = [mpkg for mpkg in meas_summary if "contributing_ECMs" in mpkg.keys()] # Identify and filter packages whose ECMs are not all present in ECM list ecm_names = [meas.stem for meas in meas_toprep_indiv_names] - meas_toprep_package_init, pkgs_skipped = ECMUtils.filter_invalid_packages( + meas_toprep_package_init, pkgs_skipped = ECMPrepHelper.filter_invalid_packages( meas_toprep_package_init, ecm_names, opts @@ -12300,15 +12300,15 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Write initial data for run_setup.json # Import analysis engine setup file - run_setup = ECMUtils.initialize_run_setup(handyfiles) + run_setup = ECMPrepHelper.initialize_run_setup(handyfiles) # Set contributing ECMs as inactive in run_setup and throw warning, set all others as active ctrb_ms = [ecm for pkg in meas_toprep_package_init for ecm in pkg["contributing_ECMs"]] non_ctrb_ms = [ecm for ecm in opts.ecm_files if ecm not in ctrb_ms] excluded_ind_ecms = [ecm for ecm in opts.ecm_files_user if ecm in ctrb_ms] - run_setup = ECMUtils.update_active_measures(run_setup, - to_active=non_ctrb_ms, - to_inactive=excluded_ind_ecms) + run_setup = ECMPrepHelper.update_active_measures(run_setup, + to_active=non_ctrb_ms, + to_inactive=excluded_ind_ecms) if excluded_ind_ecms: excluded_ind_ecms_txt = fmt.format_console_list(excluded_ind_ecms) warnings.warn("The following ECMs were selected to be prepared, but due to their" @@ -12318,7 +12318,7 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Set packages to active in run_setup valid_packages = [pkg["name"] for pkg in meas_toprep_package_init] - run_setup = ECMUtils.update_active_measures(run_setup, to_active=valid_packages) + run_setup = ECMPrepHelper.update_active_measures(run_setup, to_active=valid_packages) # Loop through each package dict in the current list and determine which # of these package measures require further preparation @@ -12486,10 +12486,10 @@ def main(opts: argparse.NameSpace): # noqa: F821 tsv_carbon_nonfs_data = None # Map years available in 8760 TSV cost/carbon data to AEO yrs. - tsv_cost_yrmap = ECMUtils.tsv_cost_carb_yrmap( + tsv_cost_yrmap = ECMPrepHelper.tsv_cost_carb_yrmap( tsv_cost_data["electricity price shapes"], handyvars.aeo_years) - tsv_carbon_yrmap = ECMUtils.tsv_cost_carb_yrmap( + tsv_carbon_yrmap = ECMPrepHelper.tsv_cost_carb_yrmap( tsv_carbon_data["average carbon emissions rates"], handyvars.aeo_years) # Stitch together load shape, cost, emissions, and year @@ -12529,13 +12529,13 @@ def main(opts: argparse.NameSpace): # noqa: F821 meas_check_list = [mo.name for mo in meas_prepped_objs] # User is warned later, after being warned that ECMs have been skipped if len(handyvars.skipped_ecms) != 0: - meas_toprep_package, pkgs_skipped = ECMUtils.filter_invalid_packages( + meas_toprep_package, pkgs_skipped = ECMPrepHelper.filter_invalid_packages( meas_toprep_package, meas_check_list, opts ) # Move package name to skipped list - run_setup = ECMUtils.update_active_measures(run_setup, to_skipped=pkgs_skipped) + run_setup = ECMPrepHelper.update_active_measures(run_setup, to_skipped=pkgs_skipped) # Prepare measure packages for use in analysis engine (if needed) if meas_toprep_package: @@ -12562,14 +12562,15 @@ def main(opts: argparse.NameSpace): # noqa: F821 "corresponding timestamp in ./generated for details.") # Add names of skipped measures to run setup list if not already there - run_setup = ECMUtils.update_active_measures(run_setup, to_skipped=handyvars.skipped_ecms) + run_setup = ECMPrepHelper.update_active_measures(run_setup, + to_skipped=handyvars.skipped_ecms) logger.info("All ECM updates complete; finalizing data...") # Split prepared measure data into subsets needed to set high-level # measure attributes information and to execute measure competition # in the analysis engine meas_prepped_compete, meas_prepped_summary, meas_prepped_shapes, \ - meas_eff_fs_splt = ECMPrep.split_clean_data( + meas_eff_fs_splt = ECMPrepHelper.split_clean_data( meas_prepped_objs, handyvars.full_dat_out) # Add all prepared high-level measure information to existing @@ -12609,7 +12610,8 @@ def main(opts: argparse.NameSpace): # noqa: F821 # Remove measures from active list; when public health costs are assumed, only # the "high" health costs versions of prepared measures remain active if opts.health_costs is True and "PHC-EE (high)" not in m["name"]: - run_setup = ECMUtils.update_active_measures(run_setup, to_inactive=[m["name"]]) + run_setup = ECMPrepHelper.update_active_measures(run_setup, + to_inactive=[m["name"]]) # Measure serves as counterfactual for isolating envelope impacts # within packages; append data to separate list, which will # be written to a separate ecm_prep file diff --git a/scout/run_batch.py b/scout/run_batch.py index 72b7bff2e..ded410076 100644 --- a/scout/run_batch.py +++ b/scout/run_batch.py @@ -2,7 +2,7 @@ from pathlib import Path from scout.config import LogConfig, Config, FilePaths as fp from scout.ecm_prep_args import ecm_args -from scout.ecm_prep import ECMUtils, main as ecm_prep_main +from scout.ecm_prep import ECMPrepHelper, main as ecm_prep_main from scout.utils import JsonIO from scout import run from argparse import ArgumentParser @@ -127,10 +127,13 @@ def run_batch(self): ecm_files_list = self.get_ecm_files(yml_grp) for ct, config in enumerate(yml_grp): # Set all ECMs inactive - run_setup = ECMUtils.update_active_measures(run_setup, - to_inactive=ecm_prep_opts.ecm_files) + run_setup = ECMPrepHelper.update_active_measures( + run_setup, + to_inactive=ecm_prep_opts.ecm_files + ) # Set yml-specific ECMs active - run_setup = ECMUtils.update_active_measures(run_setup, to_active=ecm_files_list[ct]) + run_setup = ECMPrepHelper.update_active_measures(run_setup, + to_active=ecm_files_list[ct]) JsonIO.dump_json(run_setup, fp.GENERATED / "run_setup.json") run_opts = self.get_run_opts(config) logger.info(f"Running run.py for {config}") diff --git a/tests/ecm_prep_test.py b/tests/ecm_prep_test.py index 333305c55..5b0770c0e 100644 --- a/tests/ecm_prep_test.py +++ b/tests/ecm_prep_test.py @@ -3,7 +3,7 @@ """ Tests for running the measure preparation routine """ # Import code to be tested -from scout.ecm_prep import Measure, MeasurePackage, ECMUtils, ECMPrep +from scout.ecm_prep import Measure, MeasurePackage, ECMPrepHelper, ECMPrep from scout.ecm_prep_vars import UsefulVars, UsefulInputFiles from scout.config import FilePaths as fp from scout.ecm_prep_args import ecm_args @@ -123678,12 +123678,12 @@ def test_filter_packages(self): packages = json.load(f) # Downselect via the ecm_packages argument - selected_pkgs = ECMUtils.downselect_packages(packages, opts_pkgs.ecm_packages) + selected_pkgs = ECMPrepHelper.downselect_packages(packages, opts_pkgs.ecm_packages) # Further filter packages that do not have all contributing ECMs - valid_pkgs, invalid_pkgs = ECMUtils.filter_invalid_packages(selected_pkgs, - opts_pkgs.ecm_files, - opts_pkgs) + valid_pkgs, invalid_pkgs = ECMPrepHelper.filter_invalid_packages(selected_pkgs, + opts_pkgs.ecm_files, + opts_pkgs) # Check list of valid packages valid_pkg_names = [pkg["name"] for pkg in valid_pkgs] expected_pkgs = ["ENERGY STAR Res. ASHP (FS) + Env.", @@ -123706,10 +123706,12 @@ def test_contributing_ecm_add(self): packages = json.load(f) # Downselect via the ecm_packages argument - selected_pkgs = ECMUtils.downselect_packages(packages, opts_pkg_no_ecm.ecm_packages) + selected_pkgs = ECMPrepHelper.downselect_packages(packages, opts_pkg_no_ecm.ecm_packages) # Add ECMs that contribute to package - ecms = ECMUtils.retrieve_valid_ecms(selected_pkgs, opts_pkg_no_ecm, self.handyfiles_emm) + ecms = ECMPrepHelper.retrieve_valid_ecms(selected_pkgs, + opts_pkg_no_ecm, + self.handyfiles_emm) ecms = [ecm.stem for ecm in ecms] expected_ecms = ["ENERGY STAR Res. ASHP (FS)", "Res. Air Sealing (New), IECC c. 2021", @@ -127435,7 +127437,7 @@ def test_cleanup(self): """Test 'split_clean_data' function given valid inputs.""" # Execute the function measures_comp_data, measures_summary_data, \ - measures_shape_data, measures_eff_fs_splt_data = ECMPrep.split_clean_data( + measures_shape_data, measures_eff_fs_splt_data = ECMPrepHelper.split_clean_data( self.sample_measlist_in, self.sample_full_dat_out) # Check function outputs for ind in range(0, len(self.sample_measlist_in)): @@ -127523,8 +127525,9 @@ def test_yrmap(self): # Loop across all tested year input scenarios for ind in range(0, len(self.test_tsv_data_in)): # Execute the function - out_map = ECMUtils.tsv_cost_carb_yrmap( - self.test_tsv_data_in[ind], self.test_aeo_years_in) + out_map = ECMPrepHelper.tsv_cost_carb_yrmap( + self.test_tsv_data_in[ind], + self.test_aeo_years_in) # Check function outputs self.dict_check(out_map, self.test_out_map[ind]) From 4b4900af88d38f22b4552470e9071ef65a407ed4 Mon Sep 17 00:00:00 2001 From: aspeake1 Date: Mon, 11 Aug 2025 16:06:59 -0600 Subject: [PATCH 8/8] Delete unused methods --- scout/ecm_prep.py | 156 ---------------------------------------------- 1 file changed, 156 deletions(-) diff --git a/scout/ecm_prep.py b/scout/ecm_prep.py index fbdeeca1f..5ad23992b 100644 --- a/scout/ecm_prep.py +++ b/scout/ecm_prep.py @@ -8743,159 +8743,6 @@ def rand_list_gen(self, distrib_info, nsamples): return rand_list - def create_perf_dict(self, msegs): - """Create dict to fill with updated measure performance information. - - Note: - Given a measure's applicable climate zone, building type, - structure type, fuel type, and end use, create a dict of zeros - with a hierarchy that is defined by these measure properties. - - Args: - msegs (dict): Baseline microsegment stock and energy use - information to use in validating categorization of - measure performance information. - - Returns: - Empty dictionary to fill with EnergyPlus-based performance - information broken down by climate zone, building type/vintage, - fuel type, and end use. - """ - # Initialize performance dict - perf_dict_empty = {"primary": None, "secondary": None} - # Create primary dict structure from baseline market properties - perf_dict_empty["primary"] = self.create_nested_dict( - msegs, "primary") - - # Create secondary dict structure from baseline market properties - # (if needed) - if isinstance(self.end_use, dict): - perf_dict_empty["secondary"] = self.create_nested_dict( - msegs, "secondary") - - return perf_dict_empty - - def create_nested_dict(self, msegs, mseg_type): - """Create a nested dictionary based on a pre-defined branch structure. - - Note: - Create a nested dictionary with a structure that is defined by a - measure's applicable baseline market, with end leaf node values set - to zero. - - Args: - msegs (dict): Baseline microsegment stock and energy use - information to use in validating categorization of - measure performance information. - mseg_type (string): Primary or secondary microsegment type flag. - - Returns: - Nested dictionary of zeros with desired branch structure. - """ - # Initialize output dictionary - output_dict = {} - # Establish levels of the dictionary key hierarchy from measure's - # applicable baseline market information - keylevels = [ - self.climate_zone, self.bldg_type, self.fuel_type[mseg_type], - self.end_use[mseg_type], self.structure_type] - # Find all possible dictionary key chains from the above key level - # info. - dict_keys = list(itertools.product(*keylevels)) - # Remove all natural gas cooling key chains (EnergyPlus output - # files do not include a column for natural gas cooling) - dict_keys = [x for x in dict_keys if not ( - 'natural gas' in x and 'cooling' in x)] - - # Use an input dictionary with valid baseline microsegment information - # to check that each of the microsegment key chains generated above is - # valid for the current measure; if not, remove each invalid key chain - # from further operations - - # Initialize a list of valid baseline microsegment key chains for the - # measure - dict_keys_fin = [] - # Loop through the initial set of candidate key chains generated above - for kc in dict_keys: - # Copy the input dictionary containing valid key chains - dict_check = copy.deepcopy(msegs) - # Loop through all keys in the candidate key chain and move down - # successive levels of the input dict until either the end of the - # key chain is reached or a key is not found in the list of valid - # keys for the current input dict level. In the former case, the - # resultant dict will point to all technologies associated with the - # current key chain (e.g., ASHP, LFL, etc.) If none of these - # technologies are found in the list of technologies covered by the - # measure, the key chain is deemed invalid - for ind, key in enumerate(kc): - # If key is found in the list of valid keys for the current - # input microsegment dict level, move on to next level in the - # dict; otherwise, break the current loop - if key in dict_check.keys(): - dict_check = dict_check[key] - # In the case of heating or cooling end uses, an additional - # 'technology type' key must be accounted for ('supply' or - # 'demand') - if key in ['heating', 'cooling']: - dict_check = \ - dict_check[self.technology_type[mseg_type]] - else: - break - - # If any of the technology types listed in the measure definition - # are found in the keys of the dictionary yielded by the above - # loop, add the key chain to the list that is used to define the - # final nested dictionary output (e.g., the key chain is valid) - if any([x in self.technology[mseg_type] - for x in dict_check.keys()]): - dict_keys_fin.append(kc) - - # Loop through each of the valid key chains and create an - # associated path in the dictionary, terminating with a zero value - # to be updated in a subsequent routine with EnergyPlus output data - for kc in dict_keys_fin: - current_level = output_dict - for ind, elem in enumerate(kc): - if elem not in current_level and (ind + 1) != len(kc): - current_level[elem] = {} - elif elem not in current_level and (ind + 1) == len(kc): - current_level[elem] = 0 - current_level = current_level[elem] - - return output_dict - - def build_array(self, eplus_coltyp, files_to_build): - """Assemble EnergyPlus data from one or more CSVs into a record array. - - Args: - eplus_coltypes (list): Expected EnergyPlus variable data types. - files_to_build (CSV objects): CSV files of EnergyPlus energy - consumption information under measure and baseline cases. - - Returns: - Structured array of EnergyPlus energy savings information for the - Measure. - """ - # Loop through CSV file objects and import/add to record array - for ind, f in enumerate(files_to_build): - # Read in CSV file to array - eplus_file = numpy.genfromtxt(f, names=True, dtype=eplus_coltyp, - delimiter=",", missing_values='') - # Find only those rows in the array that represent - # completed simulation runs for the measure of interest - eplus_file = eplus_file[(eplus_file[ - 'measure'] == self.energy_efficiency['EnergyPlus file']) | - (eplus_file['measure'] == 'none') & - (eplus_file['status'] == 'completed normal')] - # Initialize or add to a master array that covers all CSV data - if ind == 0: - eplus_perf_array = eplus_file - else: - eplus_perf_array = \ - numpy.concatenate((eplus_perf_array, eplus_file)) - - return eplus_perf_array - def breakout_mseg(self, mskeys, contrib_mseg_key, adopt_scheme, opts, add_stock_total, add_energy_total, add_energy_cost, add_carb_total, add_stock_total_meas, add_energy_total_eff, @@ -11713,9 +11560,6 @@ def prepare_measures(measures, convert_data, msegs, msegs_cpl, handyvars, base_dir, handyvars, handyfiles, opts_dict, **m) for m in measures] logger.info("Measure initialization complete") - # Fill in EnergyPlus-based performance information for Measure objects - # with a 'From EnergyPlus' flag in their 'energy_efficiency' attribute - # Handle a superfluous 'undefined' key in the ECM performance field that is # generated by the 'Add ECM' web form in certain cases *** NOTE: WILL # FIX IN FUTURE UI VERSION ***