diff --git a/src/troute-network/troute/HYFeaturesNetwork.py b/src/troute-network/troute/HYFeaturesNetwork.py index fb8a19225..65393b2e9 100644 --- a/src/troute-network/troute/HYFeaturesNetwork.py +++ b/src/troute-network/troute/HYFeaturesNetwork.py @@ -88,18 +88,28 @@ def read_layer(layer_name): flowpaths_df = table_dict.get('flowpaths', pd.DataFrame()) flowpath_attributes_df = table_dict.get('flowpath_attributes', pd.DataFrame()) - # Check if 'link' column exists and rename it to 'id' + # Check if 'link' column exists; drop existing 'id' col; rename 'link' to 'id' if 'link' in flowpath_attributes_df.columns: + # In HF 2.2, a 'link' field was introduced. The field is identical to + # previous version's 'id' field, but it preferred moving forwards. + flowpath_attributes_df.drop(columns=['id'], errors='ignore', inplace=True) flowpath_attributes_df.rename(columns={'link': 'id'}, inplace=True) - + + # NOTE: aaraney: `flowpaths_df` and `flowpath_attributes_df` can share + # column names but this is not accounted for elsewhere. im not sure if it + # is okay to assume that if the left and right df have the same `id` field + # that the other shared columns will match and thus it is safe to set, for + # example, the left suffix to "". # Merge flowpaths and flowpath_attributes flowpaths = pd.merge( flowpaths_df, flowpath_attributes_df, on='id', - how='inner' + how='inner', + # NOTE: aaraney: not sure if this is safe + suffixes=("", "_flowpath_attributes"), ) - + lakes = table_dict.get('lakes', pd.DataFrame()) network = table_dict.get('network', pd.DataFrame()) nexus = table_dict.get('nexus', pd.DataFrame()) @@ -601,7 +611,7 @@ def preprocess_waterbodies(self, lakes, nexus): self._duplicate_ids_df = pd.DataFrame() self._gl_climatology_df = pd.DataFrame() - self._dataframe = self.dataframe.drop('waterbody', axis=1).drop_duplicates() + self._dataframe = self.dataframe.drop('waterbody', axis=1, errors='ignore').drop_duplicates() def preprocess_data_assimilation(self, network): if not network.empty: @@ -726,7 +736,16 @@ def build_qlateral_array(self, run,): # This capability should be here, but we need to think through how to handle all of this # data in memory for large domains and many timesteps... - shorvath, Feb 28, 2024 qlat_file_pattern_filter = self.forcing_parameters.get("qlat_file_pattern_filter", None) - if qlat_file_pattern_filter=="nex-*": + + if qlat_file_pattern_filter=="*.CATOUT.csv": + for f in qlat_files: + df = pd.read_csv(f) + df = df.set_index('feature_id') + dfs.append(df) + + qlats_df = pd.concat(dfs, axis=1) + qlats_df = qlats_df[qlats_df.index.isin(self.segment_index)] + elif qlat_file_pattern_filter=="nex-*": for f in qlat_files: df = pd.read_csv(f, names=['timestamp', 'qlat'], index_col=[0]) df['timestamp'] = pd.to_datetime(df['timestamp']).dt.strftime('%Y%m%d%H%M') @@ -752,10 +771,11 @@ def build_qlateral_array(self, run,): # lateral flows [m^3/s] are stored at NEXUS points with NEXUS ids nexuses_lateralflows_df = pd.concat(dfs, axis=1) - # Take flowpath ids entering NEXUS and replace NEXUS ids by the upstream flowpath ids - qlats_df = nexuses_lateralflows_df.rename(index=self.downstream_flowpath_dict) - qlats_df = qlats_df[qlats_df.index.isin(self.segment_index)] - + if qlat_file_pattern_filter!="*.CATOUT.csv": + # Take flowpath ids entering NEXUS and replace NEXUS ids by the upstream flowpath ids + qlats_df = nexuses_lateralflows_df.rename(index=self.downstream_flowpath_dict) + qlats_df = qlats_df[qlats_df.index.isin(self.segment_index)] + ''' #For a terminal nexus, we want to include the lateral flow from the catchment contributing to that nexus #one way to do that is to cheat and put that lateral flow at the upstream...this is probably the simplest way diff --git a/src/troute-network/troute/nhd_io.py b/src/troute-network/troute/nhd_io.py index 381810c84..5b09ddf11 100644 --- a/src/troute-network/troute/nhd_io.py +++ b/src/troute-network/troute/nhd_io.py @@ -2337,6 +2337,8 @@ def optimized_agg(merge_flowveldepth_reset): # Set the new 'Type' column as an index all_nex_data = all_nex_data.set_index('Type', append=True) + # Reindex to (n, f), (n, v), (n, d), ... + all_nex_data = all_nex_data.reindex(flowveldepth.columns, axis=1) else: all_nex_data = pd.DataFrame()