Fix inconsistent reading of agent share names (#586)

* Remove to_agent_share conversion

* Remove inline comment

* Fix error in trade model

* Update documentation

* Update reader tests

* More helpful error message

* More helpful error message

Author: tsmbland

docs/inputs/technodata.rst

@@ -136,33 +136,32 @@ InterestRate
    is the technology interest rate (called hurdle rates in other models).
    This is used for the interest used in the discount rate and corresponds to the interest built when borrowing money.
 
-Agent_0, ..., Agent_N
-   represent the allocation of the initial capacity to the each agent.
-   The column heading refers each retrofit agent "AgentShare" as defined in the agents' definition (see :ref:`inputs-agents`).
+Agent0, ..., AgentN
+   represent the proportion of initial capacity allocated to each agent.
+   Must match AgentShare names specified in the :ref:`inputs-agents` file.
+   All agents must be represented in the table.
+   If using "New" and "Retrofit" agents, you should create a column with the name of each "Retrofit" agent share.
+   If only using "New" agents, you should create a column with the name of each "New" agent share.
    The value corresponds to the ownership of the initial stock, as defined in the :ref:`inputs-existing-capacity` for the starting year of the simulation.
-   For example, if an initial boiler stock of 10 PJ is available, this is allocated to each agent according to the "AgentShare".
 
-   In a one-agent simulation, assuming that the *AgentShare* equals to *Agent_2* for the retrofit agent, the technodata should indicate the stock ownership as follows.
-   The modelled agent would own the total 10 PJ of the initial stock of boilers.
+   For example, in a one-agent simulation, you should specify the following to indicate full ownership of existing capacity by the agent (assuming an agent share name of "Agent1"):
 
    .. csv-table:: Techno-data: AgentShare - 1 agent
-      :header: ProcessName, RegionName, Time, ..., Agent_2
+      :header: ProcessName, RegionName, Time, ..., Agent1
 
       resBoilerElectric, region1, 2010, ..., 1
       resBoilerElectric, region1, 2030, ..., 1
 
-   In a two-agent simulation, a new column needs to be added for each retrofit agent belonging to the new-retrofit agent pair.
-   The column heading refers each retrofit agent "AgentShare" as defined in the agents' definition (see :ref:`inputs-agents`).
-   Assuming a split of the initial capacity into 30 \% and 70 \% for each retrofit agent, the model table would be setup as follows.
-   The values of the "AgentShare" needs to reflect the demand split represented by the "Quantity" attribute (see :ref:`inputs-agents`),
-   to make sure that the initial demand is fulfilled with the initial stock.
+   In a two-agent simulation, assuming a 30\% / 70\% split of initial capacity between the two agents, the table would be as follows:
 
    .. csv-table:: Techno-data: AgentShare - 2 agents
-      :header: ProcessName, RegionName, Time, ..., Agent_2, Agent_4
+      :header: ProcessName, RegionName, Time, ..., Agent1, Agent2
 
       resBoilerElectric, region1, 2010, ..., 0.3, 0.7
       resBoilerElectric, region1, 2030, ..., 0.3, 0.7
 
+   Values must sum to 1 for each row of the table.
+
 The input data has to be provided for the base year. Additional years within the time
 framework of the overall simulation can be defined. In this case, MUSE would interpolate
 the values between the provided periods and assume a constant value afterwards. The additional

src/muse/agents/factories.py

@@ -8,7 +8,7 @@
 
 from muse.agents.agent import Agent, InvestingAgent
 from muse.defaults import DEFAULT_SECTORS_DIRECTORY
-from muse.errors import RetrofitAgentNotDefined, TechnologyNotDefined
+from muse.errors import AgentShareNotDefined, TechnologyNotDefined
 
 
 def create_standard_agent(
@@ -342,7 +342,7 @@ def _shared_capacity(
     try:
         shares = technologies[share]
     except KeyError:
-        raise RetrofitAgentNotDefined
+        raise AgentShareNotDefined
 
     try:
         shares = shares.sel(technology=capacity.technology)

src/muse/data/example/trade/technodata/Agents.csv

@@ -1,3 +1,3 @@
 AgentShare,Name,RegionName,Objective,ObjData,Objsort,SearchRule,DecisionMethod,MaturityThreshold,SpendLimit,Type
-agent_share,A1,R1,ALCOE,1,TRUE,from_assets->compress->reduce_assets,singleObj,-1,inf,default
-agent_share,A1,R2,ALCOE,1,TRUE,from_assets->compress->reduce_assets,singleObj,-1,inf,default
+Agent1,A1,R1,ALCOE,1,TRUE,from_assets->compress->reduce_assets,singleObj,-1,inf,default
+Agent1,A1,R2,ALCOE,1,TRUE,from_assets->compress->reduce_assets,singleObj,-1,inf,default

src/muse/data/example/trade/technodata/gas/Technodata.csv

@@ -1,4 +1,4 @@
-ProcessName,RegionName,Time,var_par,var_exp,TechnicalLife,UtilizationFactor,ScalingSize,efficiency,InterestRate,Type,Fuel,EndUse,AgentShare
-Unit,-,Year,MUS$2010/PJ,-,Years,-,PJ,%,-,-,-,-,-
+ProcessName,RegionName,Time,var_par,var_exp,TechnicalLife,UtilizationFactor,ScalingSize,efficiency,InterestRate,Type,Fuel,EndUse,Agent1
+Unit,-,Year,MUS$2010/PJ,-,Years,-,PJ,%,-,-,-,-,New
 gassupply1,R1,2010,3,1,10,0.9,0.00000189,86,0.1,energy,gas,gas,1
 gassupply1,R2,2010,3,1,10,0.9,0.00000189,86,0.1,energy,gas,gas,1

src/muse/data/example/trade/technodata/power/Technodata.csv

@@ -1,5 +1,5 @@
-ProcessName,RegionName,Time,cap_exp,fix_exp,var_par,var_exp,TechnicalLife,UtilizationFactor,ScalingSize,efficiency,InterestRate,Type,Fuel,EndUse,AgentShare
-Unit,-,Year,-,-,MUS$2010/PJ,-,Years,-,PJ,%,-,-,-,-,
+ProcessName,RegionName,Time,cap_exp,fix_exp,var_par,var_exp,TechnicalLife,UtilizationFactor,ScalingSize,efficiency,InterestRate,Type,Fuel,EndUse,Agent1
+Unit,-,Year,-,-,MUS$2010/PJ,-,Years,-,PJ,%,-,-,-,-,New
 gasCCGT,R1,2010,1,1,0,1,35,0.9,0.00000189,86,0.1,energy,gas,electricity,1
 windturbine,R1,2010,1,1,0,1,25,0.4,0.00000189,86,0.1,energy,wind,electricity,1
 gasCCGT,R2,2010,1,1,0,1,35,0.9,0.00000189,86,0.1,energy,gas,electricity,1

src/muse/data/example/trade/technodata/residential/Agents.csv

@@ -1,5 +1,5 @@
 AgentShare,Name,RegionName,Objective1,Objective2,Objective3,ObjData1,ObjData2,ObjData3,Objsort1,Objsort2,Objsort3,SearchRule,DecisionMethod,Quantity,MaturityThreshold,Budget,Type
-agent_share_1,A1,R1,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,New
-agent_share_2,A1,R1,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,Retrofit
-agent_share_1,A1,R2,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,New
-agent_share_2,A1,R2,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,Retrofit
+Agent1,A1,R1,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,New
+Agent2,A1,R1,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,Retrofit
+Agent1,A1,R2,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,New
+Agent2,A1,R2,LCOE,,,1,,,TRUE,,,all,singleObj,1,-1,inf,Retrofit

src/muse/errors.py

@@ -1,13 +1,12 @@
-class RetrofitAgentNotDefined(Exception):
-    """Indicates that the retrofit agent has not been defined."""
-
-    msg = """The retrofit agent has not been defined. This might be because it actually
-has not been defined in the agents file or specified in the technodata file or it cannot
-be found because there is a typo in its name in either the agents and or technodata
-files. When a retrofit agent is defined in the agent file, the same name should be
-reported as an additional column in the technodata, assigning the fraction of the
-initial stock assigned to each retrofit agent. In presence of multiple retrofit agents,
-a new column needs to be added per agent."""
+class AgentShareNotDefined(Exception):
+    """Indicates that an agent share is missing from a technodata file."""
+
+    msg = """All agents must be represented in the technodata file.
+If using "New" and "Retrofit" agents, you need a column with the name of each "Retrofit"
+agent share. If only using "New" agents, you need a column with the name of each "New"
+agent share. Please check that all agents are represented in the technodata file, and
+that the agent share names match those specified in your agents file.
+"""
 
     def __str__(self):
         return self.msg

src/muse/readers/csv.py

@@ -74,19 +74,12 @@ def read_technodictionary(filename: Union[str, Path]) -> xr.Dataset:
 
     There are three axes: technologies, regions, and year.
     """
-    from re import sub
-
     from muse.readers import camel_to_snake
 
-    def to_agent_share(name):
-        return sub(r"agent(\d)", r"agent_share_\1", name)
-
     csv = pd.read_csv(filename, float_precision="high", low_memory=False)
     csv.drop(csv.filter(regex="Unname"), axis=1, inplace=True)
-    csv = (
-        csv.rename(columns=camel_to_snake)
-        .rename(columns=to_agent_share)
-        .rename(columns={"end_use": "enduse", "availabiliy year": "availability"})
+    csv = csv.rename(columns=camel_to_snake).rename(
+        columns={"end_use": "enduse", "availabiliy year": "availability"}
     )
     data = csv[csv.process_name != "Unit"]
 
@@ -533,7 +526,7 @@ def read_csv_agent_parameters(filename) -> list:
     Returns a list of dictionaries, where each dictionary can be used to instantiate an
     agent in :py:func:`muse.agents.factories.factory`.
     """
-    from re import sub
+    from muse.readers import camel_to_snake
 
     if (
         isinstance(filename, str)
@@ -599,8 +592,7 @@ def read_csv_agent_parameters(filename) -> list:
             data["maturity_threshold"] = row.MaturityThreshold
         if hasattr(row, "SpendLimit"):
             data["spend_limit"] = row.SpendLimit
-        # if agent_type != "newcapa":
-        data["share"] = sub(r"Agent(\d)", r"agent_share_\1", row.AgentShare)
+        data["share"] = camel_to_snake(row.AgentShare)
         if agent_type == "retrofit" and data["decision"] == "lexo":
             data["decision"] = "retro_lexo"
         result.append(data)

tests/test_readers.py

@@ -455,7 +455,7 @@ def test_read_technodictionary(default_model):
         type=np.dtype("O"),
         fuel=np.dtype("<U11"),
         enduse=np.dtype("<U4"),
-        agent_share_1=np.dtype("int64"),
+        agent1=np.dtype("int64"),
         tech_type=np.dtype("<U6"),
         efficiency=np.dtype("int64"),
         max_capacity_addition=np.dtype("int64"),
@@ -602,7 +602,7 @@ def test_read_csv_agent_parameters(default_model):
             "quantity": 1,
             "maturity_threshold": -1,
             "spend_limit": np.inf,
-            "share": "agent_share_1",
+            "share": "agent1",
         },
     ]