Features/status2023

CHANGELOG.rst

@@ -252,6 +252,14 @@ Added
   `PR #903 <https://github.com/openego/eGon-data/pull/903>`_
 * Add low flex scenario 'eGon2035_lowflex'
   `#822 <https://github.com/openego/eGon-data/issues/822>`_
+* Add Status2023 update heat demand
+  `#199 <https://github.com/openego/powerd-data/pull/199>`_
+* Add status2023 electrical neighbours
+  `#182 <https://github.com/openego/powerd-data/pull/182>`_
+* Add Status2023 to gas nodes
+  `#205 <https://github.com/openego/powerd-data/pull/205>`_
+* Update osm for status2023
+  `#169 <https://github.com/openego/powerd-data/pull/169>`_
 * Add wrapped_partial to dynamise task generation
   `#207 <https://github.com/openego/powerd-data/pull/207>`_
 
@@ -489,6 +497,8 @@ Changed
   `#1026 <https://github.com/openego/eGon-data/issues/1026>`_
 * Change hgv data source to use database
   `#1086 <https://github.com/openego/eGon-data/issues/1086>`_
+* Add eMob MIT for SQ2023 scenario
+  `#176 <https://github.com/openego/powerd-data/issues/176>`_
 
 .. _#799: https://github.com/openego/eGon-data/issues/799
 
@@ -732,9 +742,6 @@ Bug Fixes
   `#1098 <https://github.com/openego/eGon-data/issues/1098>`_
 * Fix conversion factor for CH4 loads abroad in eGon2035
   `#1104 <https://github.com/openego/eGon-data/issues/1104>`_
-* Fix URLs of MaStR datasets
-* Fix CRS in ERA5 transformation
-  `#179 <https://github.com/openego/powerd-data/issues/179>`_
 
 .. _PR #692: https://github.com/openego/eGon-data/pull/692
 .. _#343: https://github.com/openego/eGon-data/issues/343

README.rst

@@ -139,11 +139,14 @@ packages are required too. Right now these are:
 
 * To download generation and demand data from entsoe you need to register in
   the `entsoe platform <https://transparency.entsoe.eu/>`_. Once the user is
-  created, a personal token can be generated to access the available data. This
-  token must be saved in a text file called *.entsoe-token* in the home directory.
+  created, send an email to [email protected] with “Restful API access” in
+  the subject line. Indicate the email address you entered during registration
+  in the email body. After authorisation a personal token can be generated in
+  'My Account Settings'to access the available data. This token must be saved
+  in a text file called *.entsoe-token* in the home directory.
   In Ubuntu you can go to the home directory by typing :code:`cd ~` in a
   terminal. To create the file execute :code:`nano ".entsoe-token"` and then
-  paste your personal token (36 characters). Finally press :code:`CRL + x` to 
+  paste your personal token (36 characters). Finally press :code:`CRL + x` to
   save and exit.
 
 * Make sure you have enough free disk space (~350 GB) in your working

setup.py

@@ -87,7 +87,7 @@ def read(*names, **kwargs):
         "atlite==0.2.11",
         "cdsapi",
         "click<8.1",
-        "entsoe-py >=0.5.5",
+        "entsoe-py >=0.6.2",
         "Flask-Session<0.6.0",
         "GeoAlchemy2",
         "geopandas>=0.10.0",

src/egon/data/__init__.py

@@ -2,14 +2,14 @@
 
 from loguru import logger
 import click
-
+import shutil
 __version__ = "0.0.0"
 
 
 def echo(message):
     prefix, message = message.split(" - ")
     lines = message.split("\n")
-    width = min(72, click.get_terminal_size()[0])
+    width = min(72, shutil.get_terminal_size()[0])
     wraps = ["\n".join(wrap(line, width)) for line in lines]
     message = "\n".join([prefix] + wraps)
     click.echo(message, err=True)

src/egon/data/airflow/dags/pipeline_status_quo.py

@@ -35,7 +35,7 @@
 from egon.data.datasets.etrago_setup import EtragoSetup
 from egon.data.datasets.fill_etrago_gen import Egon_etrago_gen
 from egon.data.datasets.fix_ehv_subnetworks import FixEhvSubnetworks
-from egon.data.datasets.gas_areas import GasAreasstatus2019
+from egon.data.datasets.gas_areas import GasAreasStatusQuo
 from egon.data.datasets.gas_grid import GasNodesAndPipes
 from egon.data.datasets.gas_neighbours import GasNeighbours
 from egon.data.datasets.heat_demand import HeatDemandImport
@@ -44,7 +44,7 @@
 from egon.data.datasets.heat_etrago import HeatEtrago
 from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable
 from egon.data.datasets.heat_supply import HeatSupply
-from egon.data.datasets.heat_supply.individual_heating import HeatPumps2019
+from egon.data.datasets.heat_supply.individual_heating import HeatPumpsStatusQuo
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
 from egon.data.datasets.industry import IndustrialDemandCurves
 from egon.data.datasets.loadarea import LoadArea, OsmLanduse
@@ -284,6 +284,7 @@
             cts_electricity_demand_annual,
             demand_curves_industry,
             hh_demand_buildings_setup,
+            household_electricity_demand_annual,
         ]
     )
 
@@ -349,8 +350,8 @@
             tasks["etrago_setup.create-tables"],
         ]
     )
-    # Create gas voronoi status2019
-    create_gas_polygons_status2019 = GasAreasstatus2019(
+    # Create gas voronoi status quo
+    create_gas_polygons_statusquo = GasAreasStatusQuo(
         dependencies=[setup_etrago, vg250, gas_grid_insert_data, substation_voronoi]
     )
 
@@ -360,24 +361,24 @@
             gas_grid_insert_data,
             run_pypsaeursec,
             foreign_lines,
-            create_gas_polygons_status2019,
+            create_gas_polygons_statusquo,
         ]
     )
 
     # Import gas production
     gas_production_insert_data = CH4Production(
-        dependencies=[create_gas_polygons_status2019]
+        dependencies=[create_gas_polygons_statusquo]
     )
 
     # Import CH4 storages
     insert_data_ch4_storages = CH4Storages(
-        dependencies=[create_gas_polygons_status2019]
+        dependencies=[create_gas_polygons_statusquo]
     )
 
     # CHP locations
     chp = Chp(
         dependencies=[
-            create_gas_polygons_status2019,
+            create_gas_polygons_statusquo,
             demand_curves_industry,
             district_heating_areas,
             industrial_sites,
@@ -406,7 +407,7 @@
     )
 
     create_ocgt = OpenCycleGasTurbineEtrago(
-        dependencies=[create_gas_polygons_status2019, power_plants]
+        dependencies=[create_gas_polygons_statusquo, power_plants]
     )
 
     # Fill eTraGo generators tables
@@ -424,17 +425,18 @@
             scenario_capacities,
         ]
     )
-    
-    
-    # Pumped hydro units
-    pumped_hydro = Storages(
+
+
+    # Pumped hydro and home storage units
+    storage_units = Storages(
         dependencies=[
             mastr_data,
             mv_grid_districts,
             power_plants,
             scenario_parameters,
             setup,
             vg250_mv_grid_districts,
+            fill_etrago_generators,
         ]
     )
 
@@ -452,8 +454,8 @@
     # CHP to eTraGo
     chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago])
 
-    # Heat pump disaggregation for status2019
-    heat_pumps_2019 = HeatPumps2019(
+    # Heat pump disaggregation for status quo
+    heat_pumps_sq = HeatPumpsStatusQuo(
         dependencies=[
             cts_demand_buildings,
             DistrictHeatingAreas,
@@ -470,13 +472,13 @@
             heat_etrago,
             heat_time_series,
             mv_grid_districts,
-            heat_pumps_2019,
+            heat_pumps_sq,
         ]
     )
 
     # Storages to eTraGo
     storage_etrago = StorageEtrago(
-        dependencies=[pumped_hydro, scenario_parameters, setup_etrago]
+        dependencies=[storage_units, scenario_parameters, setup_etrago]
     )
 
     # eMobility: motorized individual travel TODO: adjust for SQ

src/egon/data/cli.py

@@ -99,6 +99,15 @@
     ),
     show_default=True,
 )
+@click.option(
+    "--household-demand-source",
+    type=click.Choice(["IEE", "demand-regio"]),
+    default="demand-regio",
+    help=(
+        "Choose the source to calculate and allocate household demands."
+    ),
+    show_default=True,
+)
 @click.option(
     "--jobs",
     default=1,
@@ -166,7 +175,7 @@
 
 @click.option(
     "--scenarios",
-    default=["status2019", "eGon2035"],
+    default=["status2023"],
     metavar="SCENARIOS",
     help=(
         "List of scenario names for which a data model shall be created."

src/egon/data/config.py

@@ -73,7 +73,7 @@ def settings() -> dict[str, dict[str, str]]:
                 "--jobs": 1,
                 "--random-seed": 42,
                 "--processes-per-task": 1,
-                "--scenarios": ["status2019", "eGon2035"],
+                "--scenarios": "status2023",
             }
         }
     with open(files[0]) as f:

src/egon/data/datasets.yml

@@ -1,13 +1,13 @@
 openstreetmap:
   original_data:
     source:
-      url: "https://download.geofabrik.de/europe/germany-200101.osm.pbf"
-      url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-200101.osm.pbf"
+      url: "https://download.geofabrik.de/europe/germany-240101.osm.pbf"
+      url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-240101.osm.pbf"
       stylefile: "oedb.style"
     target:
       table_prefix: "osm"
-      file: "germany-200101.osm.pbf"
-      file_testmode: "schleswig-holstein-200101.osm.pbf"
+      file: "germany-240101.osm.pbf"
+      file_testmode: "schleswig-holstein-240101.osm.pbf"
   processed:
     schema: "openstreetmap"
     tables:
@@ -94,6 +94,17 @@ demandregio_installation:
   targets:
     path: 'demandregio-disaggregator'
 
+demandregio_workaround:
+  source:
+    cache:
+      url: "https://tubcloud.tu-berlin.de/s/dKqF6wKJnLRyDws/download/cache.zip"
+    dbdump:
+      url: "https://tubcloud.tu-berlin.de/s/ktaxyo8kSTK8w3f/download/status2019-egon-demandregio-cts-ind.zip"
+  targets:
+    cache:
+      path: 'demandregio-disaggregator/disaggregator/disaggregator/data_in/'
+    dbdump:
+      path: "demandregio_dbdump"
 demandregio_society:
   sources:
     disaggregator:
@@ -305,10 +316,12 @@ mastr_new:
     - "biomass"
     - "combustion"
     - "nuclear"
+    - "storage"
   file_basename: "bnetza_mastr"
-  deposit_id: 10480958
+  deposit_id: 10491882
   egon2021_date_max: "2021-12-31 23:59:00"
   status2019_date_max: "2019-12-31 23:59:00"
+  status2023_date_max: "2023-12-31 23:59:00"
 
 re_potential_areas:
   target:
@@ -365,8 +378,8 @@ power_plants:
       mastr_hydro: "bnetza_mastr_hydro_cleaned.csv"
       mastr_location: "location_elec_generation_raw.csv"
       mastr_combustion_without_chp: "supply.egon_mastr_conventional_without_chp"
-      mastr_combustion: "bnetza_mastr/dump_2022-11-17/bnetza_mastr_combustion_cleaned.csv"
-      mastr_nuclear: "bnetza_mastr/dump_2022-11-17/bnetza_mastr_nuclear_cleaned.csv"
+      mastr_combustion: "bnetza_mastr/dump_2024-01-08/bnetza_mastr_combustion_cleaned.csv"
+      mastr_nuclear: "bnetza_mastr/dump_2024-01-08/bnetza_mastr_nuclear_cleaned.csv"
       mastr_storage: "bnetza_mastr_storage_cleaned.csv"
       mastr_gsgk: "bnetza_mastr_gsgk_cleaned.csv"
       capacities: "supply.egon_scenario_capacities"
@@ -1118,16 +1131,19 @@ emobility_mit:
         file_processed: "regiostar-referenzdateien_preprocessed.csv"
         sheet: "ReferenzGebietsstand2020"
       KBA:
-        url: "https://www.kba.de/SharedDocs/Downloads/DE/Statistik/Fahrzeuge/FZ1/fz1_2020_xlsx.xlsx?__blob=publicationFile&v=2"
-        file: "fz1_2020_xlsx.xlsx"
-        file_processed: "fz1_2020_preprocessed.csv"
+        url: "https://www.kba.de/SharedDocs/Downloads/DE/Statistik/Fahrzeuge/FZ1/fz1_2021.xlsx?__blob=publicationFile&v=2"
+        file: "fz1_2021.xlsx"
+        file_processed: "fz1_2021_preprocessed.csv"
         sheet: "FZ1.1"
         columns: "D, J:N"
         skiprows: 8
       trips:
         status2019:
           file:  "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz"
           file_metadata: "metadata_simbev_run.json"
+        status2023:
+          file:  "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz"
+          file_metadata: "metadata_simbev_run.json"
         eGon2035:
           file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz"
           file_metadata: "metadata_simbev_run.json"
@@ -1138,6 +1154,7 @@ emobility_mit:
     # used scenario variation (available scenarios see parameters.py)
     variation:
       status2019: "status2019"
+      status2023: "status2023"
       eGon2035: "NEP C 2035"
       eGon100RE: "Reference 2050"
     # name of low-flex scenario

src/egon/data/datasets/__init__.py

@@ -9,7 +9,7 @@
 import re
 
 from airflow.models.baseoperator import BaseOperator as Operator
-from airflow.operators.python_operator import PythonOperator
+from airflow.operators.python import PythonOperator
 from sqlalchemy import Column, ForeignKey, Integer, String, Table, orm, tuple_
 from sqlalchemy.ext.declarative import declarative_base
 

src/egon/data/datasets/ch4_prod.py

@@ -45,7 +45,7 @@ class CH4Production(Dataset):
     name: str = "CH4Production"
     #:
 
-    version: str = "0.0.8"
+    version: str = "0.0.9"
 
     def __init__(self, dependencies):
         super().__init__(
@@ -368,13 +368,13 @@ def import_gas_generators():
                 .reset_index(drop=False)
             )
 
-        elif scn_name == "status2019":
+        elif "status" in scn_name:
             # Add one large CH4 generator at each CH4 bus
             CH4_generators_list = db.select_dataframe(
-                """
+                f"""
                 SELECT bus_id as bus, scn_name, carrier
                 FROM grid.egon_gas_voronoi
-                WHERE scn_name = 'status2019'
+                WHERE scn_name = '{scn_name}'
                 AND carrier = 'CH4'
                 """
             )

src/egon/data/datasets/ch4_storages.py

@@ -47,17 +47,22 @@ class CH4Storages(Dataset):
     #:
     name: str = "CH4Storages"
     #:
-    version: str = "0.0.2"
+    version: str = "0.0.3"
 
     def __init__(self, dependencies):
         super().__init__(
             name=self.name,
             version=self.version,
             dependencies=dependencies,
-            tasks=(insert_ch4_storages),
+            # tasks=(insert_ch4_storages),
+            tasks=(notasks),
         )
 
 
+def notasks():
+    return None
+
+
 def import_installed_ch4_storages(scn_name):
     """
     Define list of CH4 stores from the SciGRID_gas data