diff --git a/python/calib/pyproject.toml b/python/calib/pyproject.toml index 7b95937..851cea8 100644 --- a/python/calib/pyproject.toml +++ b/python/calib/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["setuptools>=64", "wheel"] build-backend = "setuptools.build_meta" + +[tool.ruff] +exclude = ["src/"] diff --git a/python/calib/src/calib/calibration_set.py b/python/calib/src/calib/calibration_set.py index c7155cc..30dc16b 100644 --- a/python/calib/src/calib/calibration_set.py +++ b/python/calib/src/calib/calibration_set.py @@ -75,6 +75,13 @@ def __init__( if os.path.exists(obsflow_file): logger.info(f"Read observed streamflow from: {obsflow_file}") obs = pd.read_csv(obsflow_file) + + # if obs is empty, raise an error + if obs.empty: + msg = f"Streamflow observation file is empty: {obsflow_file}" + logger.error(msg) + raise ValueError(msg) + cols = obs.columns.str.lower() obs.columns = cols @@ -217,8 +224,10 @@ def output(self) -> "DataFrame": except Exception as e: raise e - if hydrograph is None: - logger.info("Output could not be read after multiple attempts.") + if hydrograph is None or hydrograph.empty: + msg = "Simulated hydrograph is unavailable or empty." + logger.error(msg) + raise ValueError(msg) return hydrograph diff --git a/python/calib/src/calib/configuration.py b/python/calib/src/calib/configuration.py index d0d9332..0dacfe9 100644 --- a/python/calib/src/calib/configuration.py +++ b/python/calib/src/calib/configuration.py @@ -246,6 +246,11 @@ def get_sim_df(self, _output_file, _wb_lst) -> "DataFrame": except Exception as e: raise (e) + if hydrograph is None or hydrograph.empty: + msg = "Simulated hydrograph is unavailable or empty." + logger.error(msg) + raise ValueError(msg) + return hydrograph def postprocess_single_calibration_output(self, agent): @@ -295,6 +300,13 @@ def postprocess_single_calibration_output(self, agent): # Load observed data obs_df = pd.read_csv(self.obsflow, parse_dates=["value_date"]) + + # if obs is empty, raise an error + if obs_df.empty: + msg = f"Streamflow observation file is empty: {self.obsflow}" + logger.error(msg) + raise ValueError(msg) + obs_df = obs_df.rename( columns={"value_date": "Time", obs_df.columns[1]: obs_flow_col} ).set_index("Time") @@ -316,8 +328,6 @@ def postprocess_single_calibration_output(self, agent): metrics_df = pd.DataFrame([metrics]) metrics_df.insert(0, "iteration", 0, True) - # IS THIS CORRECT??????? - # metrics_df["objFunVal"] = metrics_df[self.eval_params.objective.upper()] metrics_best_path = workdir / f"{basin_id}_metrics_iteration.csv" metrics_df.to_csv(metrics_best_path, index=False) diff --git a/python/calib/src/calib/search.py b/python/calib/src/calib/search.py index b6d1861..8194cd8 100644 --- a/python/calib/src/calib/search.py +++ b/python/calib/src/calib/search.py @@ -130,14 +130,35 @@ def _calc_metrics( simulated_hydrograph, observed_hydrograph, left_index=True, right_index=True ) if df.empty: - _logger().warning("Cannot compute objective function, do time indicies align?") + msg = "No overlapping time period between simulated and observed streamflow. Metrics cannot be calculated. Exit." + _logger().error(msg) + raise ValueError(msg) + + # If eval_range is provided, filter the dataframe to only include data within that range if eval_range: df = df.loc[eval_range[0] : eval_range[1]] df.reset_index(inplace=True) + # treat the data by removing negative values, NaN values, and replacing zero values with a small positive value df = treat_values(df, remove_neg=True, remove_na=True, replace_zero=True) + # if df is empty, log an error and raise an exception + if df.empty: + if eval_range: + eval_range_str = ( + f" within the evaluation datetime range " + f"{eval_range[0].strftime('%Y-%m-%d %H:%M')} " + f"to {eval_range[1].strftime('%Y-%m-%d %H:%M')}" + ) + else: + eval_range_str = "" + + msg = f"There are no valid observed or simulated streamflow data{eval_range_str}. Metrics cannot be calculated. Exit." + + _logger().error(msg) + raise ValueError(msg) + # reset the time index (needed for calculation of event-based metrics) df.set_index(df.columns[0], inplace=True) diff --git a/python/calib/src/calib/validation_run.py b/python/calib/src/calib/validation_run.py index 573f7ac..2c42a2f 100644 --- a/python/calib/src/calib/validation_run.py +++ b/python/calib/src/calib/validation_run.py @@ -92,15 +92,14 @@ def run_valid_ctrl_best(agent): outputs = [primary_set.output] runs = [agent.run_name] - if agent.run_name != "valid_control": - if agent.nwmflow is not None: - outputs.append(agent.nwmflow) - runs.append("nwm_retro") + if agent.run_name != "valid_control" and agent.nwmflow is not None: + outputs.append(agent.nwmflow) + runs.append("nwm_retro") for out1, run1 in zip(outputs, runs): metrics = pd.DataFrame() - # _logger().info(f"Computing metrics for out1 : {out1}, run1: {run1}") for key, value in time_period.items(): + _logger().info(f"Computing metrics for {run1} and time period: {key}") result = _calc_metrics( out1, primary_set.observed,