tag_to_var_map arguments and tests (#109)

* adding tag_to_var_map argument when calculate_values is called in tag.py

* Adding missing tag_to_var_map argument within process_binary_ops

* Adding test for use of tag_to_var_map

* Simplifying logic to pass in tag_to_var_map in test_tag.py

* Reformatting with black and addressing flake8 issues

Tests passing

* Reformatting with black and addressing flake8 issues

Tests passing

* Reverting unneeded changes in test_calculate_values

* Adding 2 more tag_to_var_map arguments in process_unary_ops and process_binary_ops, respectively

* temp

* Adding additional test coverage for tag_to_var_map

* Consolidating when TypeError is raised for unary_helper (for codecov)

Adding test case for empty unary operations (for codecov)

Removing files generated in tests

* Delete pype_schema/tests/data/EPANET_Net_3_temp.inp

* Delete pype_schema/tests/data/L-TOWN_temp.inp

* Delete pype_schema/tests/data/valve_temp.inp

* Adding test for invalid unary data

* Removing type check from unary_helper since type is already checked in process_unary_ops

* Adding back simplified TypeError raising in unary_helper

Adding tests for:
- TypeError from unary_helper
- tag_to_var_map with List data
- invalid mode in VirtualTag class

Author: dalyw

pype_schema/operations.py

@@ -198,6 +198,7 @@ def unary_helper(data, un_op):
         numpy array of dataset trannsformed by unary operation
     """
     # allow for multiple unary operations to be performed sequentially
+    result = None
     if isinstance(un_op, list):
         result = data.copy()
         for op in un_op:
@@ -212,15 +213,11 @@ def unary_helper(data, un_op):
             result = [-x for x in data]
         elif isinstance(data, (np.ndarray, pd.Series)):
             result = -data
-        else:
-            raise TypeError("Data must be either a list, array, or Series")
     elif un_op == "~":
         if isinstance(data, list):
-            result = result = [not bool(x) for x in data]
+            result = [not bool(x) for x in data]
         elif isinstance(data, (np.ndarray, pd.Series)):
             result = data == 0
-        else:
-            raise TypeError("Data must be either a list, array, or Series")
     else:
         if isinstance(data, list):
             result = data.copy()
@@ -247,7 +244,8 @@ def unary_helper(data, un_op):
                 result = data.shift(-1)
             elif un_op == ">>":
                 result = data.shift(1)
-        else:
-            raise TypeError("Data must be either a list, array, or Series")
+
+    if result is None:
+        raise TypeError("Data must be either a list, array, or Series")
 
     return result

pype_schema/tag.py

@@ -918,16 +918,32 @@ def process_unary_ops(self, data, tag_to_var_map={}):
                     else:  # must be a DataFrame
                         relevant_data = pd.Series([tag_obj.value] * len(data))
                 elif isinstance(tag_obj, self.__class__):
-                    relevant_data = tag_obj.calculate_values(data)
+                    relevant_data = tag_obj.calculate_values(data, tag_to_var_map)
                 elif tag_to_var_map:
                     relevant_data = result[tag_to_var_map[tag_obj.id]]
                 else:
                     relevant_data = result[tag_obj.id]
 
-                relevant_data = unary_helper(  # noqa: F405
+                is_series = isinstance(relevant_data, pd.Series)
+                if is_series:  # store info, then convert to np array
+                    original_index = relevant_data.index
+                    original_name = relevant_data.name
+                    relevant_data = relevant_data.values
+
+                processed_relevant_data = unary_helper(  # noqa: F405
                     relevant_data, self.unary_operations[i]
                 )
 
+                # Convert back if original was Series
+                if is_series:
+                    relevant_data = pd.Series(
+                        processed_relevant_data,
+                        index=original_index,
+                        name=original_name,
+                    )
+                else:
+                    relevant_data = processed_relevant_data
+
                 if tag_to_var_map:
                     result[tag_to_var_map[tag_obj.id]] = relevant_data
                 else:
@@ -1015,7 +1031,7 @@ def process_binary_ops(self, data, tag_to_var_map={}):
                     else:
                         relevant_data = data[tag_obj.id].copy()
                 elif isinstance(tag_obj, self.__class__):
-                    relevant_data = tag_obj.calculate_values(data)
+                    relevant_data = tag_obj.calculate_values(data, tag_to_var_map)
                 elif tag_to_var_map:
                     relevant_data = data[tag_to_var_map[tag_obj.id]].copy()
                 else:
@@ -1077,7 +1093,7 @@ def process_binary_ops(self, data, tag_to_var_map={}):
                     else:
                         relevant_data = data[tag_obj.id].copy()
                 elif isinstance(tag_obj, self.__class__):
-                    relevant_data = tag_obj.calculate_values(data)
+                    relevant_data = tag_obj.calculate_values(data, tag_to_var_map)
                 elif tag_to_var_map:
                     relevant_data = data[tag_to_var_map[tag_obj.id]].copy()
                 else:
@@ -1155,7 +1171,7 @@ def process_custom_ops(self, data, tag_to_var_map={}):
                 varname = tag_to_var_map[tag_obj.id] if tag_to_var_map else tag_obj.id
                 varnames.append(varname)
                 if isinstance(tag_obj, self.__class__):
-                    data[varname] = tag_obj.calculate_values(data)
+                    data[varname] = tag_obj.calculate_values(data, tag_to_var_map)
             result = func_(*[data[varname] for varname in varnames])
             if isinstance(result, Series):
                 result.rename(self.id, inplace=True)
@@ -1193,7 +1209,10 @@ def calculate_values(self, data, tag_to_var_map={}):
                 data = self.process_binary_ops(data, tag_to_var_map=tag_to_var_map)
             elif isinstance(data, (dict, DataFrame)):
                 # if no binary ops, get appropriate column from unary ops and rename
-                data = data[self.tags[0].id].rename(self.id)
+                if isinstance(data, dict):
+                    data = pd.Series(data[self.tags[0].id], name=self.id)
+                else:
+                    data = data[self.tags[0].id].rename(self.id)
             elif isinstance(data, ndarray):
                 # flatten array since binary operations do that automatically
                 data = data[:, 0]
@@ -1202,7 +1221,10 @@ def calculate_values(self, data, tag_to_var_map={}):
                 data = self.process_custom_ops(data, tag_to_var_map=tag_to_var_map)
             elif isinstance(data, (dict, DataFrame)):
                 # if custom_operations is empty, get appropriate column and rename
-                data = data[self.tags[0].id].rename(self.id)
+                if isinstance(data, dict):
+                    data = pd.Series(data[self.tags[0].id], name=self.id)
+                else:
+                    data = data[self.tags[0].id].rename(self.id)
             elif isinstance(data, ndarray):
                 # flatten array since operations do that automatically
                 data = data[:, 0]

pype_schema/tests/__init__.py

@@ -3,7 +3,7 @@
 """Unit test package for pype_schema."""
 
 import pickle
-from pype_schema.tag import VirtualTag
+from pype_schema.tag import VirtualTag, Tag
 from pype_schema.node import Boiler, Cogeneration, Pump, Network
 
 
@@ -48,3 +48,37 @@ def pickle_without_functions(network, outpath):
     # export pickled object
     with open(outpath, "wb") as pickle_file:
         pickle.dump(network, pickle_file)
+
+
+def generate_tag_to_var_map(network):
+    """Simple example of generating a mapping from
+    rom tag IDs to variable names, renaming variables linking
+    objects in the network
+    (used to generate mapping json file for tests)
+
+    Parameters
+    ----------
+    network : Network
+        The network to generate the mapping for
+
+    Returns
+    -------
+    dict
+        Dictionary mapping tag IDs to variable names
+    """
+    tag_to_var_map = {}
+    for tag in network.get_all_tags(recurse=True):
+        if isinstance(tag, VirtualTag):  # keep network name for virtual tags
+            tag_to_var_map[tag.id] = tag.id
+        elif isinstance(tag, Tag):  # rename for other tags
+            parent = network.get_parent_from_tag(tag)
+            if hasattr(parent, "id"):
+                source_id = parent.id
+            else:
+                source_id = "Unknown"
+
+            contents_type = tag.contents.name if tag.contents is not None else "None"
+            variable_type = tag.tag_type.name
+
+            tag_to_var_map[tag.id] = f"{source_id}_{contents_type}_{variable_type}"
+    return tag_to_var_map

pype_schema/tests/data/connection_less_than.json

@@ -64,6 +64,14 @@
         "type": "Flow",
         "parent_id": "Cogenerator",
         "contents": "Electricity"
+      },
+      "TestEmptyCustom": {
+        "tags": ["ElectricityGeneration"],
+        "custom_operations": "",
+        "type": "Flow",
+        "parent_id": "Cogenerator",
+        "contents": "Electricity",
+        "units": "kWh"
       }
     },
     "DrinkingWaterFacility": {

pype_schema/tests/data/sample_data_mapped.csv

@@ -0,0 +1,97 @@
+GasToBoiler_NaturalGas_Flow,GasToCogen_NaturalGas_Flow,ConditionerToCogen_Biogas_Flow,CogenElecToFacility_Electricity_Flow,Constant1
+0,1264.064032,6361.096418,919,1
+0,1304.840291,6361.096418,863,1
+0,1304.840291,6034.886346,882,1
+0,1264.064032,5994.110087,945,1
+0,1223.287773,6075.662605,914,1
+0,1223.287773,6605.753973,912,1
+0,1264.064032,5831.00505,867,1
+0,1345.61655,6157.215123,926,1
+0,1223.287773,5790.228791,890,1
+0,1264.064032,6483.425196,911,1
+0,1264.064032,6687.306491,901,1
+0,1223.287773,6157.215123,926,1
+0,1223.287773,6361.096418,916,1
+0,1304.840291,6809.635268,925,1
+0,1386.392809,6850.411528,890,1
+0,1223.287773,6809.635268,935,1
+0,1264.064032,6687.306491,913,1
+0,1304.840291,6646.530232,905,1
+0,1264.064032,6646.530232,927,1
+0,1264.064032,6687.306491,921,1
+0,1223.287773,6972.740305,903,1
+0,1223.287773,6564.977714,886,1
+0,1304.840291,5953.333828,911,1
+0,1264.064032,6157.215123,896,1
+0,1304.840291,5667.900014,879,1
+0,1264.064032,5260.137423,915,1
+0,1304.840291,5423.242459,915,1
+0,1304.840291,5341.689941,882,1
+0,1304.840291,5137.808646,930,1
+0,1264.064032,4811.598573,889,1
+0,1264.064032,5056.256127,890,1
+0,1264.064032,5056.256127,897,1
+17.488486,1182.511514,5097.032387,867,1
+17.488486,1182.511514,5504.794977,878,1
+302.9223,897.0777,4730.046055,939,1
+343.6985591,856.3014409,4566.941018,853,1
+1200,0,4730.046055,862,1
+1200,0,5056.256127,911,1
+1200,0,5219.361164,816,1
+1200,0,5545.571237,879,1
+1200,0,5341.689941,905,1
+1200,0,5545.571237,961,1
+1200,0,5178.584905,919,1
+1200,0,5871.781309,903,1
+1200,0,5953.333828,944,1
+1200,0,5871.781309,909,1
+1200,0,5790.228791,927,1
+1200,0,5464.018718,910,1
+1200,0,5545.571237,903,1
+1200,0,5667.900014,917,1
+1200,0,5382.4662,886,1
+1200,0,5871.781309,907,1
+1200,0,5708.676273,897,1
+1200,0,5667.900014,966,1
+1200,0,5627.123755,893,1
+1200,0,5300.913682,893,1
+1200,0,5586.347496,953,1
+1200,0,5056.256127,924,1
+1200,0,5300.913682,908,1
+1200,0,5015.479868,935,1
+1200,0,5464.018718,976,1
+1200,0,5464.018718,923,1
+1200,0,5464.018718,895,1
+1200,0,5464.018718,910,1
+1200,0,5341.689941,953,1
+1200,0,4770.822314,595,1
+1200,0,5178.584905,946,1
+1200,0,5341.689941,926,1
+1200,0,5831.00505,959,1
+1200,0,5219.361164,916,1
+1200,0,5382.4662,887,1
+1200,0,5137.808646,840,1
+1200,0,5382.4662,863,1
+1200,0,4485.3885,881,1
+1200,0,5341.689941,863,1
+1200,0,5097.032387,799,1
+1200,0,4770.822314,819,1
+0,1304.840291,5097.032387,840,1
+0,1264.064032,4811.598573,877,1
+0,1304.840291,4893.151091,918,1
+0,1264.064032,5260.137423,939,1
+0,1223.287773,5260.137423,956,1
+0,1264.064032,4240.730946,933,1
+0,1264.064032,4974.703609,889,1
+17.488486,1182.511514,5382.4662,921,1
+0,1223.287773,4933.92735,938,1
+17.488486,1182.511514,4974.703609,896,1
+0,1223.287773,4566.941018,852,1
+0,1223.287773,4526.164759,925,1
+0,1264.064032,4811.598573,914,1
+0,1223.287773,4770.822314,892,1
+0,1223.287773,4363.059723,903,1
+0,1223.287773,4811.598573,909,1
+0,1223.287773,5178.584905,950,1
+0,1223.287773,4648.493537,926,1
+17.488486,1182.511514,5178.584905,919,1

pype_schema/tests/data/tag_to_var_map.json

@@ -0,0 +1,23 @@
+{
+  "ElectricityGeneration_LShift1": "ElectricityGeneration_LShift1",
+  "Digester3GasFlow": "ConditionerToCogen_Biogas_Flow",
+  "Digester2GasFlow": "ConditionerToCogen_Biogas_Flow",
+  "BoilerGasPurchases": "GasToBoiler_NaturalGas_Flow",
+  "TotalizedFlaredGas": "ConditionerToFlare_Biogas_Flow",
+  "NoGasPurchases": "NoGasPurchases",
+  "GrossGasProduction": "GrossGasProduction",
+  "ElectricityProductionByGasVolume": "ElectricityProductionByGasVolume",
+  "ElectricityGeneration_RShift2_List": "ElectricityGeneration_RShift2_List",
+  "Digester1GasFlow": "ConditionerToCogen_Biogas_Flow",
+  "CogenGasPurchases": "GasToCogen_NaturalGas_Flow",
+  "CombinedDigesterGasFlow": "ConditionerToCogen_Biogas_Flow",
+  "ElectricityGeneration": "CogenElecToFacility_Electricity_Flow",
+  "TankLevel": "FOGTank_FatOilGrease_Level",
+  "InfluentFlow": "SewerIntake_UntreatedSewage_Flow",
+  "NoGasPurchasesList": "NoGasPurchasesList",
+  "ElectricityGeneration_LShift1_List": "ElectricityGeneration_LShift1_List",
+  "ElectricityGeneration_RShift2": "ElectricityGeneration_RShift2",
+  "TankVolume": "FOGTank_FatOilGrease_Volume",
+  "GrossGasProductionList": "GrossGasProductionList",
+  "ElectricityGenDelta": "ElectricityGenDelta"
+}