diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 33d0a1ef4..a569c6208 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -252,6 +252,14 @@ Added `PR #903 `_ * Add low flex scenario 'eGon2035_lowflex' `#822 `_ +* Add Status2023 update heat demand + `#199 `_ +* Add status2023 electrical neighbours + `#182 `_ +* Add Status2023 to gas nodes + `#205 `_ +* Update osm for status2023 + `#169 `_ * Add wrapped_partial to dynamise task generation `#207 `_ @@ -489,6 +497,8 @@ Changed `#1026 `_ * Change hgv data source to use database `#1086 `_ +* Add eMob MIT for SQ2023 scenario + `#176 `_ .. _#799: https://github.com/openego/eGon-data/issues/799 @@ -732,9 +742,6 @@ Bug Fixes `#1098 `_ * Fix conversion factor for CH4 loads abroad in eGon2035 `#1104 `_ -* Fix URLs of MaStR datasets -* Fix CRS in ERA5 transformation - `#179 `_ .. _PR #692: https://github.com/openego/eGon-data/pull/692 .. _#343: https://github.com/openego/eGon-data/issues/343 diff --git a/README.rst b/README.rst index 38cb528af..7c5e4aa77 100644 --- a/README.rst +++ b/README.rst @@ -139,11 +139,14 @@ packages are required too. Right now these are: * To download generation and demand data from entsoe you need to register in the `entsoe platform `_. Once the user is - created, a personal token can be generated to access the available data. This - token must be saved in a text file called *.entsoe-token* in the home directory. + created, send an email to transparency@entsoe.eu with “Restful API access” in + the subject line. Indicate the email address you entered during registration + in the email body. After authorisation a personal token can be generated in + 'My Account Settings'to access the available data. This token must be saved + in a text file called *.entsoe-token* in the home directory. In Ubuntu you can go to the home directory by typing :code:`cd ~` in a terminal. To create the file execute :code:`nano ".entsoe-token"` and then - paste your personal token (36 characters). Finally press :code:`CRL + x` to + paste your personal token (36 characters). Finally press :code:`CRL + x` to save and exit. * Make sure you have enough free disk space (~350 GB) in your working diff --git a/setup.py b/setup.py index 7956076fd..f7b870fc2 100755 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ def read(*names, **kwargs): "atlite==0.2.11", "cdsapi", "click<8.1", - "entsoe-py >=0.5.5", + "entsoe-py >=0.6.2", "Flask-Session<0.6.0", "GeoAlchemy2", "geopandas>=0.10.0", diff --git a/src/egon/data/__init__.py b/src/egon/data/__init__.py index 22de0bf2b..f0b261789 100644 --- a/src/egon/data/__init__.py +++ b/src/egon/data/__init__.py @@ -2,14 +2,14 @@ from loguru import logger import click - +import shutil __version__ = "0.0.0" def echo(message): prefix, message = message.split(" - ") lines = message.split("\n") - width = min(72, click.get_terminal_size()[0]) + width = min(72, shutil.get_terminal_size()[0]) wraps = ["\n".join(wrap(line, width)) for line in lines] message = "\n".join([prefix] + wraps) click.echo(message, err=True) diff --git a/src/egon/data/airflow/dags/pipeline_status_quo.py b/src/egon/data/airflow/dags/pipeline_status_quo.py index fd93b7399..4400248c9 100755 --- a/src/egon/data/airflow/dags/pipeline_status_quo.py +++ b/src/egon/data/airflow/dags/pipeline_status_quo.py @@ -35,7 +35,7 @@ from egon.data.datasets.etrago_setup import EtragoSetup from egon.data.datasets.fill_etrago_gen import Egon_etrago_gen from egon.data.datasets.fix_ehv_subnetworks import FixEhvSubnetworks -from egon.data.datasets.gas_areas import GasAreasstatus2019 +from egon.data.datasets.gas_areas import GasAreasStatusQuo from egon.data.datasets.gas_grid import GasNodesAndPipes from egon.data.datasets.gas_neighbours import GasNeighbours from egon.data.datasets.heat_demand import HeatDemandImport @@ -44,7 +44,7 @@ from egon.data.datasets.heat_etrago import HeatEtrago from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable from egon.data.datasets.heat_supply import HeatSupply -from egon.data.datasets.heat_supply.individual_heating import HeatPumps2019 +from egon.data.datasets.heat_supply.individual_heating import HeatPumpsStatusQuo from egon.data.datasets.industrial_sites import MergeIndustrialSites from egon.data.datasets.industry import IndustrialDemandCurves from egon.data.datasets.loadarea import LoadArea, OsmLanduse @@ -284,6 +284,7 @@ cts_electricity_demand_annual, demand_curves_industry, hh_demand_buildings_setup, + household_electricity_demand_annual, ] ) @@ -349,8 +350,8 @@ tasks["etrago_setup.create-tables"], ] ) - # Create gas voronoi status2019 - create_gas_polygons_status2019 = GasAreasstatus2019( + # Create gas voronoi status quo + create_gas_polygons_statusquo = GasAreasStatusQuo( dependencies=[setup_etrago, vg250, gas_grid_insert_data, substation_voronoi] ) @@ -360,24 +361,24 @@ gas_grid_insert_data, run_pypsaeursec, foreign_lines, - create_gas_polygons_status2019, + create_gas_polygons_statusquo, ] ) # Import gas production gas_production_insert_data = CH4Production( - dependencies=[create_gas_polygons_status2019] + dependencies=[create_gas_polygons_statusquo] ) # Import CH4 storages insert_data_ch4_storages = CH4Storages( - dependencies=[create_gas_polygons_status2019] + dependencies=[create_gas_polygons_statusquo] ) # CHP locations chp = Chp( dependencies=[ - create_gas_polygons_status2019, + create_gas_polygons_statusquo, demand_curves_industry, district_heating_areas, industrial_sites, @@ -406,7 +407,7 @@ ) create_ocgt = OpenCycleGasTurbineEtrago( - dependencies=[create_gas_polygons_status2019, power_plants] + dependencies=[create_gas_polygons_statusquo, power_plants] ) # Fill eTraGo generators tables @@ -424,10 +425,10 @@ scenario_capacities, ] ) - - - # Pumped hydro units - pumped_hydro = Storages( + + + # Pumped hydro and home storage units + storage_units = Storages( dependencies=[ mastr_data, mv_grid_districts, @@ -435,6 +436,7 @@ scenario_parameters, setup, vg250_mv_grid_districts, + fill_etrago_generators, ] ) @@ -452,8 +454,8 @@ # CHP to eTraGo chp_etrago = ChpEtrago(dependencies=[chp, heat_etrago]) - # Heat pump disaggregation for status2019 - heat_pumps_2019 = HeatPumps2019( + # Heat pump disaggregation for status quo + heat_pumps_sq = HeatPumpsStatusQuo( dependencies=[ cts_demand_buildings, DistrictHeatingAreas, @@ -470,13 +472,13 @@ heat_etrago, heat_time_series, mv_grid_districts, - heat_pumps_2019, + heat_pumps_sq, ] ) # Storages to eTraGo storage_etrago = StorageEtrago( - dependencies=[pumped_hydro, scenario_parameters, setup_etrago] + dependencies=[storage_units, scenario_parameters, setup_etrago] ) # eMobility: motorized individual travel TODO: adjust for SQ diff --git a/src/egon/data/cli.py b/src/egon/data/cli.py index ba66f006a..598b6015c 100644 --- a/src/egon/data/cli.py +++ b/src/egon/data/cli.py @@ -99,6 +99,15 @@ ), show_default=True, ) +@click.option( + "--household-demand-source", + type=click.Choice(["IEE", "demand-regio"]), + default="demand-regio", + help=( + "Choose the source to calculate and allocate household demands." + ), + show_default=True, +) @click.option( "--jobs", default=1, @@ -166,7 +175,7 @@ @click.option( "--scenarios", - default=["status2019", "eGon2035"], + default=["status2023"], metavar="SCENARIOS", help=( "List of scenario names for which a data model shall be created." diff --git a/src/egon/data/config.py b/src/egon/data/config.py index 5dad8693f..e26e87f41 100644 --- a/src/egon/data/config.py +++ b/src/egon/data/config.py @@ -73,7 +73,7 @@ def settings() -> dict[str, dict[str, str]]: "--jobs": 1, "--random-seed": 42, "--processes-per-task": 1, - "--scenarios": ["status2019", "eGon2035"], + "--scenarios": "status2023", } } with open(files[0]) as f: diff --git a/src/egon/data/datasets.yml b/src/egon/data/datasets.yml index 7f6a584a7..befc3fecb 100644 --- a/src/egon/data/datasets.yml +++ b/src/egon/data/datasets.yml @@ -1,13 +1,13 @@ openstreetmap: original_data: source: - url: "https://download.geofabrik.de/europe/germany-200101.osm.pbf" - url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-200101.osm.pbf" + url: "https://download.geofabrik.de/europe/germany-240101.osm.pbf" + url_testmode: "https://download.geofabrik.de/europe/germany/schleswig-holstein-240101.osm.pbf" stylefile: "oedb.style" target: table_prefix: "osm" - file: "germany-200101.osm.pbf" - file_testmode: "schleswig-holstein-200101.osm.pbf" + file: "germany-240101.osm.pbf" + file_testmode: "schleswig-holstein-240101.osm.pbf" processed: schema: "openstreetmap" tables: @@ -94,6 +94,17 @@ demandregio_installation: targets: path: 'demandregio-disaggregator' +demandregio_workaround: + source: + cache: + url: "https://tubcloud.tu-berlin.de/s/dKqF6wKJnLRyDws/download/cache.zip" + dbdump: + url: "https://tubcloud.tu-berlin.de/s/ktaxyo8kSTK8w3f/download/status2019-egon-demandregio-cts-ind.zip" + targets: + cache: + path: 'demandregio-disaggregator/disaggregator/disaggregator/data_in/' + dbdump: + path: "demandregio_dbdump" demandregio_society: sources: disaggregator: @@ -305,10 +316,12 @@ mastr_new: - "biomass" - "combustion" - "nuclear" + - "storage" file_basename: "bnetza_mastr" - deposit_id: 10480958 + deposit_id: 10491882 egon2021_date_max: "2021-12-31 23:59:00" status2019_date_max: "2019-12-31 23:59:00" + status2023_date_max: "2023-12-31 23:59:00" re_potential_areas: target: @@ -365,8 +378,8 @@ power_plants: mastr_hydro: "bnetza_mastr_hydro_cleaned.csv" mastr_location: "location_elec_generation_raw.csv" mastr_combustion_without_chp: "supply.egon_mastr_conventional_without_chp" - mastr_combustion: "bnetza_mastr/dump_2022-11-17/bnetza_mastr_combustion_cleaned.csv" - mastr_nuclear: "bnetza_mastr/dump_2022-11-17/bnetza_mastr_nuclear_cleaned.csv" + mastr_combustion: "bnetza_mastr/dump_2024-01-08/bnetza_mastr_combustion_cleaned.csv" + mastr_nuclear: "bnetza_mastr/dump_2024-01-08/bnetza_mastr_nuclear_cleaned.csv" mastr_storage: "bnetza_mastr_storage_cleaned.csv" mastr_gsgk: "bnetza_mastr_gsgk_cleaned.csv" capacities: "supply.egon_scenario_capacities" @@ -1118,9 +1131,9 @@ emobility_mit: file_processed: "regiostar-referenzdateien_preprocessed.csv" sheet: "ReferenzGebietsstand2020" KBA: - url: "https://www.kba.de/SharedDocs/Downloads/DE/Statistik/Fahrzeuge/FZ1/fz1_2020_xlsx.xlsx?__blob=publicationFile&v=2" - file: "fz1_2020_xlsx.xlsx" - file_processed: "fz1_2020_preprocessed.csv" + url: "https://www.kba.de/SharedDocs/Downloads/DE/Statistik/Fahrzeuge/FZ1/fz1_2021.xlsx?__blob=publicationFile&v=2" + file: "fz1_2021.xlsx" + file_processed: "fz1_2021_preprocessed.csv" sheet: "FZ1.1" columns: "D, J:N" skiprows: 8 @@ -1128,6 +1141,9 @@ emobility_mit: status2019: file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" file_metadata: "metadata_simbev_run.json" + status2023: + file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" + file_metadata: "metadata_simbev_run.json" eGon2035: file: "eGon2035_RS7_min2k_2022-06-01_175429_simbev_run.tar.gz" file_metadata: "metadata_simbev_run.json" @@ -1138,6 +1154,7 @@ emobility_mit: # used scenario variation (available scenarios see parameters.py) variation: status2019: "status2019" + status2023: "status2023" eGon2035: "NEP C 2035" eGon100RE: "Reference 2050" # name of low-flex scenario diff --git a/src/egon/data/datasets/__init__.py b/src/egon/data/datasets/__init__.py index 34254bb38..521123c43 100644 --- a/src/egon/data/datasets/__init__.py +++ b/src/egon/data/datasets/__init__.py @@ -9,7 +9,7 @@ import re from airflow.models.baseoperator import BaseOperator as Operator -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from sqlalchemy import Column, ForeignKey, Integer, String, Table, orm, tuple_ from sqlalchemy.ext.declarative import declarative_base diff --git a/src/egon/data/datasets/ch4_prod.py b/src/egon/data/datasets/ch4_prod.py index f4f317a71..71790dd89 100755 --- a/src/egon/data/datasets/ch4_prod.py +++ b/src/egon/data/datasets/ch4_prod.py @@ -45,7 +45,7 @@ class CH4Production(Dataset): name: str = "CH4Production" #: - version: str = "0.0.8" + version: str = "0.0.9" def __init__(self, dependencies): super().__init__( @@ -368,13 +368,13 @@ def import_gas_generators(): .reset_index(drop=False) ) - elif scn_name == "status2019": + elif "status" in scn_name: # Add one large CH4 generator at each CH4 bus CH4_generators_list = db.select_dataframe( - """ + f""" SELECT bus_id as bus, scn_name, carrier FROM grid.egon_gas_voronoi - WHERE scn_name = 'status2019' + WHERE scn_name = '{scn_name}' AND carrier = 'CH4' """ ) diff --git a/src/egon/data/datasets/ch4_storages.py b/src/egon/data/datasets/ch4_storages.py index ff84af686..6f0843b0f 100755 --- a/src/egon/data/datasets/ch4_storages.py +++ b/src/egon/data/datasets/ch4_storages.py @@ -47,17 +47,22 @@ class CH4Storages(Dataset): #: name: str = "CH4Storages" #: - version: str = "0.0.2" + version: str = "0.0.3" def __init__(self, dependencies): super().__init__( name=self.name, version=self.version, dependencies=dependencies, - tasks=(insert_ch4_storages), + # tasks=(insert_ch4_storages), + tasks=(notasks), ) +def notasks(): + return None + + def import_installed_ch4_storages(scn_name): """ Define list of CH4 stores from the SciGRID_gas data diff --git a/src/egon/data/datasets/chp/__init__.py b/src/egon/data/datasets/chp/__init__.py index 1a5329717..a5f5ada80 100644 --- a/src/egon/data/datasets/chp/__init__.py +++ b/src/egon/data/datasets/chp/__init__.py @@ -1,692 +1,706 @@ -""" -The central module containing all code dealing with combined heat and power -(CHP) plants. -""" - -from pathlib import Path - -from geoalchemy2 import Geometry -from shapely.ops import nearest_points -from sqlalchemy import Boolean, Column, Float, Integer, Sequence, String -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker -import geopandas as gpd -import pandas as pd -import pypsa - -from egon.data import config, db -from egon.data.datasets import Dataset -from egon.data.datasets.chp.match_nep import insert_large_chp, map_carrier -from egon.data.datasets.chp.small_chp import ( - assign_use_case, - existing_chp_smaller_10mw, - extension_per_federal_state, - extension_to_areas, - select_target, -) -from egon.data.datasets.mastr import ( - WORKING_DIR_MASTR_OLD, - WORKING_DIR_MASTR_NEW, -) -from egon.data.datasets.power_plants import ( - assign_bus_id, - assign_voltage_level, - filter_mastr_geometry, - scale_prox2now, -) - -Base = declarative_base() - - -class EgonChp(Base): - __tablename__ = "egon_chp_plants" - __table_args__ = {"schema": "supply"} - id = Column(Integer, Sequence("chp_seq"), primary_key=True) - sources = Column(JSONB) - source_id = Column(JSONB) - carrier = Column(String) - district_heating = Column(Boolean) - el_capacity = Column(Float) - th_capacity = Column(Float) - electrical_bus_id = Column(Integer) - district_heating_area_id = Column(Integer) - ch4_bus_id = Column(Integer) - voltage_level = Column(Integer) - scenario = Column(String) - geom = Column(Geometry("POINT", 4326)) - - -class EgonMaStRConventinalWithoutChp(Base): - __tablename__ = "egon_mastr_conventional_without_chp" - __table_args__ = {"schema": "supply"} - id = Column(Integer, Sequence("mastr_conventional_seq"), primary_key=True) - EinheitMastrNummer = Column(String) - carrier = Column(String) - el_capacity = Column(Float) - plz = Column(Integer) - city = Column(String) - federal_state = Column(String) - geometry = Column(Geometry("POINT", 4326)) - - -def create_tables(): - """Create tables for chp data - Returns - ------- - None. - """ - - db.execute_sql("CREATE SCHEMA IF NOT EXISTS supply;") - engine = db.engine() - EgonChp.__table__.drop(bind=engine, checkfirst=True) - EgonChp.__table__.create(bind=engine, checkfirst=True) - EgonMaStRConventinalWithoutChp.__table__.drop(bind=engine, checkfirst=True) - EgonMaStRConventinalWithoutChp.__table__.create( - bind=engine, checkfirst=True - ) - - -def nearest( - row, - df, - centroid=False, - row_geom_col="geometry", - df_geom_col="geometry", - src_column=None, -): - """ - Finds the nearest point and returns the specified column values - - Parameters - ---------- - row : pandas.Series - Data to which the nearest data of df is assigned. - df : pandas.DataFrame - Data which includes all options for the nearest neighbor alogrithm. - centroid : boolean - Use centroid geoemtry. The default is False. - row_geom_col : str, optional - Name of row's geometry column. The default is 'geometry'. - df_geom_col : str, optional - Name of df's geometry column. The default is 'geometry'. - src_column : str, optional - Name of returned df column. The default is None. - - Returns - ------- - value : pandas.Series - Values of specified column of df - - """ - - if centroid: - unary_union = df.centroid.unary_union - else: - unary_union = df[df_geom_col].unary_union - # Find the geometry that is closest - nearest = ( - df[df_geom_col] == nearest_points(row[row_geom_col], unary_union)[1] - ) - - # Get the corresponding value from df (matching is based on the geometry) - value = df[nearest][src_column].values[0] - - return value - - -def assign_heat_bus(): - """Selects heat_bus for chps used in district heating. - - Parameters - ---------- - scenario : str, optional - Name of the corresponding scenario. The default is 'eGon2035'. - - Returns - ------- - None. - - """ - sources = config.datasets()["chp_location"]["sources"] - target = config.datasets()["chp_location"]["targets"]["chp_table"] - - for scenario in config.settings()["egon-data"]["--scenarios"]: - # Select CHP with use_case = 'district_heating' - chp = db.select_geodataframe( - f""" - SELECT * FROM - {target['schema']}.{target['table']} - WHERE scenario = '{scenario}' - AND district_heating = True - """, - index_col="id", - epsg=4326, - ) - - # Select district heating areas and their centroid - district_heating = db.select_geodataframe( - f""" - SELECT area_id, ST_Centroid(geom_polygon) as geom - FROM - {sources['district_heating_areas']['schema']}. - {sources['district_heating_areas']['table']} - WHERE scenario = '{scenario}' - """, - epsg=4326, - ) - - # Assign district heating area_id to district_heating_chp - # According to nearest centroid of district heating area - chp["district_heating_area_id"] = chp.apply( - nearest, - df=district_heating, - row_geom_col="geom", - df_geom_col="geom", - centroid=True, - src_column="area_id", - axis=1, - ) - - # Drop district heating CHP without heat_bus_id - db.execute_sql( - f""" - DELETE FROM {target['schema']}.{target['table']} - WHERE scenario = '{scenario}' - AND district_heating = True - """ - ) - - # Insert district heating CHP with heat_bus_id - session = sessionmaker(bind=db.engine())() - for i, row in chp.iterrows(): - if row.carrier != "biomass": - entry = EgonChp( - id=i, - sources=row.sources, - source_id=row.source_id, - carrier=row.carrier, - el_capacity=row.el_capacity, - th_capacity=row.th_capacity, - electrical_bus_id=row.electrical_bus_id, - ch4_bus_id=row.ch4_bus_id, - district_heating_area_id=row.district_heating_area_id, - district_heating=row.district_heating, - voltage_level=row.voltage_level, - scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", - ) - else: - entry = EgonChp( - id=i, - sources=row.sources, - source_id=row.source_id, - carrier=row.carrier, - el_capacity=row.el_capacity, - th_capacity=row.th_capacity, - electrical_bus_id=row.electrical_bus_id, - district_heating_area_id=row.district_heating_area_id, - district_heating=row.district_heating, - voltage_level=row.voltage_level, - scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", - ) - session.add(entry) - session.commit() - - -def insert_biomass_chp(scenario): - """Insert biomass chp plants of future scenario - - Parameters - ---------- - scenario : str - Name of scenario. - - Returns - ------- - None. - - """ - cfg = config.datasets()["chp_location"] - - # import target values from NEP 2021, scneario C 2035 - target = select_target("biomass", scenario) - - # import data for MaStR - mastr = pd.read_csv( - WORKING_DIR_MASTR_OLD / cfg["sources"]["mastr_biomass"] - ).query("EinheitBetriebsstatus=='InBetrieb'") - - # Drop entries without federal state or 'AusschließlichWirtschaftszone' - mastr = mastr[ - mastr.Bundesland.isin( - pd.read_sql( - f"""SELECT DISTINCT ON (gen) - REPLACE(REPLACE(gen, '-', ''), 'ü', 'ue') as states - FROM {cfg['sources']['vg250_lan']['schema']}. - {cfg['sources']['vg250_lan']['table']}""", - con=db.engine(), - ).states.values - ) - ] - - # Scaling will be done per federal state in case of eGon2035 scenario. - if scenario == "eGon2035": - level = "federal_state" - else: - level = "country" - # Choose only entries with valid geometries inside DE/test mode - mastr_loc = filter_mastr_geometry(mastr).set_geometry("geometry") - - # Scale capacities to meet target values - mastr_loc = scale_prox2now(mastr_loc, target, level=level) - - # Assign bus_id - if len(mastr_loc) > 0: - mastr_loc["voltage_level"] = assign_voltage_level( - mastr_loc, cfg, WORKING_DIR_MASTR_OLD - ) - mastr_loc = assign_bus_id(mastr_loc, cfg) - mastr_loc = assign_use_case(mastr_loc, cfg["sources"], scenario) - - # Insert entries with location - session = sessionmaker(bind=db.engine())() - for i, row in mastr_loc.iterrows(): - if row.ThermischeNutzleistung > 0: - entry = EgonChp( - sources={ - "chp": "MaStR", - "el_capacity": "MaStR scaled with NEP 2021", - "th_capacity": "MaStR", - }, - source_id={"MastrNummer": row.EinheitMastrNummer}, - carrier="biomass", - el_capacity=row.Nettonennleistung, - th_capacity=row.ThermischeNutzleistung / 1000, - scenario=scenario, - district_heating=row.district_heating, - electrical_bus_id=row.bus_id, - voltage_level=row.voltage_level, - geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", - ) - session.add(entry) - session.commit() - - -def insert_chp_statusquo(): - cfg = config.datasets()["chp_location"] - - # import data for MaStR - mastr = pd.read_csv( - WORKING_DIR_MASTR_NEW / "bnetza_mastr_combustion_cleaned.csv" - ) - - mastr = mastr.loc[mastr.ThermischeNutzleistung > 0] - - mastr = mastr.loc[ - mastr.Energietraeger.isin( - [ - "Erdgas", - "Mineralölprodukte", - "andere Gase", - "nicht biogener Abfall", - "Braunkohle", - "Steinkohle", - ] - ) - ] - - mastr.Inbetriebnahmedatum = pd.to_datetime(mastr.Inbetriebnahmedatum) - mastr.DatumEndgueltigeStilllegung = pd.to_datetime( - mastr.DatumEndgueltigeStilllegung - ) - mastr = mastr.loc[ - mastr.Inbetriebnahmedatum - <= config.datasets()["mastr_new"]["status2019_date_max"] - ] - - mastr = mastr.loc[ - ( - mastr.DatumEndgueltigeStilllegung - >= config.datasets()["mastr_new"]["status2019_date_max"] - ) - | (mastr.DatumEndgueltigeStilllegung.isnull()) - ] - - mastr.groupby("Energietraeger").Nettonennleistung.sum().mul(1e-6) - - geom_municipalities = db.select_geodataframe( - """ - SELECT gen, ST_UNION(geometry) as geom - FROM boundaries.vg250_gem - GROUP BY gen - """ - ).set_index("gen") - - # Assing Laengengrad and Breitengrad to chps without location data - # based on the centroid of the municipaltiy - idx_no_location = mastr[ - (mastr.Laengengrad.isnull()) - & (mastr.Gemeinde.isin(geom_municipalities.index)) - ].index - - mastr.loc[idx_no_location, "Laengengrad"] = ( - geom_municipalities.to_crs(epsg="4326").centroid.x.loc[ - mastr.Gemeinde[idx_no_location] - ] - ).values - - mastr.loc[idx_no_location, "Breitengrad"] = ( - geom_municipalities.to_crs(epsg="4326").centroid.y.loc[ - mastr.Gemeinde[idx_no_location] - ] - ).values - - if ( - config.settings()["egon-data"]["--dataset-boundary"] - == "Schleswig-Holstein" - ): - dropped_capacity = mastr[ - (mastr.Laengengrad.isnull()) - & (mastr.Bundesland == "SchleswigHolstein") - ].Nettonennleistung.sum() - - else: - dropped_capacity = mastr[ - (mastr.Laengengrad.isnull()) - ].Nettonennleistung.sum() - - print( - f""" - CHPs with a total installed electrical capacity of {dropped_capacity} kW are dropped - because of missing or wrong location data - """ - ) - - mastr = mastr[~mastr.Laengengrad.isnull()] - mastr = filter_mastr_geometry(mastr).set_geometry("geometry") - - # Assign bus_id - if len(mastr) > 0: - mastr["voltage_level"] = assign_voltage_level( - mastr, cfg, WORKING_DIR_MASTR_NEW - ) - - gas_bus_id = db.assign_gas_bus_id(mastr, "status2019", "CH4").bus - - mastr = assign_bus_id(mastr, cfg, drop_missing=True) - - mastr["gas_bus_id"] = gas_bus_id - - mastr = assign_use_case(mastr, cfg["sources"], "status2019") - - # Insert entries with location - session = sessionmaker(bind=db.engine())() - for i, row in mastr.iterrows(): - if row.ThermischeNutzleistung > 0: - entry = EgonChp( - sources={ - "chp": "MaStR", - "el_capacity": "MaStR", - "th_capacity": "MaStR", - }, - source_id={"MastrNummer": row.EinheitMastrNummer}, - carrier=map_carrier().loc[row.Energietraeger], - el_capacity=row.Nettonennleistung / 1000, - th_capacity=row.ThermischeNutzleistung / 1000, - scenario="status2019", - district_heating=row.district_heating, - electrical_bus_id=row.bus_id, - ch4_bus_id=row.gas_bus_id, - voltage_level=row.voltage_level, - geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", - ) - session.add(entry) - session.commit() - - -def insert_chp_egon2035(): - """Insert CHP plants for eGon2035 considering NEP and MaStR data - - Returns - ------- - None. - - """ - - sources = config.datasets()["chp_location"]["sources"] - - targets = config.datasets()["chp_location"]["targets"] - - insert_biomass_chp("eGon2035") - - # Insert large CHPs based on NEP's list of conventional power plants - MaStR_konv = insert_large_chp(sources, targets["chp_table"], EgonChp) - - # Insert smaller CHPs (< 10MW) based on existing locations from MaStR - existing_chp_smaller_10mw(sources, MaStR_konv, EgonChp) - - gpd.GeoDataFrame( - MaStR_konv[ - [ - "EinheitMastrNummer", - "el_capacity", - "geometry", - "carrier", - "plz", - "city", - "federal_state", - ] - ] - ).to_postgis( - targets["mastr_conventional_without_chp"]["table"], - schema=targets["mastr_conventional_without_chp"]["schema"], - con=db.engine(), - if_exists="replace", - ) - - -def extension_BW(): - extension_per_federal_state("BadenWuerttemberg", EgonChp) - - -def extension_BY(): - extension_per_federal_state("Bayern", EgonChp) - - -def extension_HB(): - extension_per_federal_state("Bremen", EgonChp) - - -def extension_BB(): - extension_per_federal_state("Brandenburg", EgonChp) - - -def extension_HH(): - extension_per_federal_state("Hamburg", EgonChp) - - -def extension_HE(): - extension_per_federal_state("Hessen", EgonChp) - - -def extension_MV(): - extension_per_federal_state("MecklenburgVorpommern", EgonChp) - - -def extension_NS(): - extension_per_federal_state("Niedersachsen", EgonChp) - - -def extension_NW(): - extension_per_federal_state("NordrheinWestfalen", EgonChp) - - -def extension_SN(): - extension_per_federal_state("Sachsen", EgonChp) - - -def extension_TH(): - extension_per_federal_state("Thueringen", EgonChp) - - -def extension_SL(): - extension_per_federal_state("Saarland", EgonChp) - - -def extension_ST(): - extension_per_federal_state("SachsenAnhalt", EgonChp) - - -def extension_RP(): - extension_per_federal_state("RheinlandPfalz", EgonChp) - - -def extension_BE(): - extension_per_federal_state("Berlin", EgonChp) - - -def extension_SH(): - extension_per_federal_state("SchleswigHolstein", EgonChp) - - -def insert_chp_egon100re(): - """Insert CHP plants for eGon100RE considering results from pypsa-eur-sec - - Returns - ------- - None. - - """ - - sources = config.datasets()["chp_location"]["sources"] - - db.execute_sql( - f""" - DELETE FROM {EgonChp.__table__.schema}.{EgonChp.__table__.name} - WHERE scenario = 'eGon100RE' - """ - ) - - # select target values from pypsa-eur-sec - additional_capacity = db.select_dataframe( - """ - SELECT capacity - FROM supply.egon_scenario_capacities - WHERE scenario_name = 'eGon100RE' - AND carrier = 'urban_central_gas_CHP' - """ - ).capacity[0] - - if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": - additional_capacity /= 16 - target_file = ( - Path(".") - / "data_bundle_egon_data" - / "pypsa_eur_sec" - / "2022-07-26-egondata-integration" - / "postnetworks" - / "elec_s_37_lv2.0__Co2L0-1H-T-H-B-I-dist1_2050.nc" - ) - - network = pypsa.Network(str(target_file)) - chp_index = "DE0 0 urban central gas CHP" - - standard_chp_th = 10 - standard_chp_el = ( - standard_chp_th - * network.links.loc[chp_index, "efficiency"] - / network.links.loc[chp_index, "efficiency2"] - ) - - areas = db.select_geodataframe( - f""" - SELECT - residential_and_service_demand as demand, area_id, - ST_Transform(ST_PointOnSurface(geom_polygon), 4326) as geom - FROM - {sources['district_heating_areas']['schema']}. - {sources['district_heating_areas']['table']} - WHERE scenario = 'eGon100RE' - """ - ) - - existing_chp = pd.DataFrame( - data={ - "el_capacity": standard_chp_el, - "th_capacity": standard_chp_th, - "voltage_level": 5, - }, - index=range(1), - ) - - flh = ( - network.links_t.p0[chp_index].sum() - / network.links.p_nom_opt[chp_index] - ) - - extension_to_areas( - areas, - additional_capacity, - existing_chp, - flh, - EgonChp, - district_heating=True, - scenario="eGon100RE", - ) - - -tasks = (create_tables,) - -insert_per_scenario = set() - -if "status2019" in config.settings()["egon-data"]["--scenarios"]: - insert_per_scenario.add(insert_chp_statusquo) - -if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: - insert_per_scenario.add(insert_chp_egon2035) - -if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: - insert_per_scenario.add(insert_chp_egon100re) - -tasks = tasks + (insert_per_scenario, assign_heat_bus) - -extension = set() - -if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: - # Add one task per federal state for small CHP extension - if ( - config.settings()["egon-data"]["--dataset-boundary"] - == "Schleswig-Holstein" - ): - extension = extension_SH - else: - extension = { - extension_BW, - extension_BY, - extension_HB, - extension_BB, - extension_HE, - extension_MV, - extension_NS, - extension_NW, - extension_SH, - extension_HH, - extension_RP, - extension_SL, - extension_SN, - extension_ST, - extension_TH, - extension_BE, - } - -if extension != set(): - tasks = tasks + (extension,) - - -class Chp(Dataset): - def __init__(self, dependencies): - super().__init__( - name="Chp", version="0.0.8", dependencies=dependencies, tasks=tasks - ) +""" +The central module containing all code dealing with combined heat and power +(CHP) plants. +""" + +from pathlib import Path + +from geoalchemy2 import Geometry +from shapely.ops import nearest_points +from sqlalchemy import Boolean, Column, Float, Integer, Sequence, String +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +import geopandas as gpd +import pandas as pd +import pypsa + +from egon.data import config, db +from egon.data.datasets import Dataset, wrapped_partial +from egon.data.datasets.chp.match_nep import insert_large_chp, map_carrier +from egon.data.datasets.chp.small_chp import ( + assign_use_case, + existing_chp_smaller_10mw, + extension_per_federal_state, + extension_to_areas, + select_target, +) +from egon.data.datasets.mastr import ( + WORKING_DIR_MASTR_OLD, + WORKING_DIR_MASTR_NEW, +) +from egon.data.datasets.power_plants import ( + assign_bus_id, + assign_voltage_level, + filter_mastr_geometry, + scale_prox2now, +) + +Base = declarative_base() + + +class EgonChp(Base): + __tablename__ = "egon_chp_plants" + __table_args__ = {"schema": "supply"} + id = Column(Integer, Sequence("chp_seq"), primary_key=True) + sources = Column(JSONB) + source_id = Column(JSONB) + carrier = Column(String) + district_heating = Column(Boolean) + el_capacity = Column(Float) + th_capacity = Column(Float) + electrical_bus_id = Column(Integer) + district_heating_area_id = Column(Integer) + ch4_bus_id = Column(Integer) + voltage_level = Column(Integer) + scenario = Column(String) + geom = Column(Geometry("POINT", 4326)) + + +class EgonMaStRConventinalWithoutChp(Base): + __tablename__ = "egon_mastr_conventional_without_chp" + __table_args__ = {"schema": "supply"} + id = Column(Integer, Sequence("mastr_conventional_seq"), primary_key=True) + EinheitMastrNummer = Column(String) + carrier = Column(String) + el_capacity = Column(Float) + plz = Column(Integer) + city = Column(String) + federal_state = Column(String) + geometry = Column(Geometry("POINT", 4326)) + + +def create_tables(): + """Create tables for chp data + Returns + ------- + None. + """ + + db.execute_sql("CREATE SCHEMA IF NOT EXISTS supply;") + engine = db.engine() + EgonChp.__table__.drop(bind=engine, checkfirst=True) + EgonChp.__table__.create(bind=engine, checkfirst=True) + EgonMaStRConventinalWithoutChp.__table__.drop(bind=engine, checkfirst=True) + EgonMaStRConventinalWithoutChp.__table__.create( + bind=engine, checkfirst=True + ) + + +def nearest( + row, + df, + centroid=False, + row_geom_col="geometry", + df_geom_col="geometry", + src_column=None, +): + """ + Finds the nearest point and returns the specified column values + + Parameters + ---------- + row : pandas.Series + Data to which the nearest data of df is assigned. + df : pandas.DataFrame + Data which includes all options for the nearest neighbor alogrithm. + centroid : boolean + Use centroid geoemtry. The default is False. + row_geom_col : str, optional + Name of row's geometry column. The default is 'geometry'. + df_geom_col : str, optional + Name of df's geometry column. The default is 'geometry'. + src_column : str, optional + Name of returned df column. The default is None. + + Returns + ------- + value : pandas.Series + Values of specified column of df + + """ + + if centroid: + unary_union = df.centroid.unary_union + else: + unary_union = df[df_geom_col].unary_union + # Find the geometry that is closest + nearest = ( + df[df_geom_col] == nearest_points(row[row_geom_col], unary_union)[1] + ) + + # Get the corresponding value from df (matching is based on the geometry) + value = df[nearest][src_column].values[0] + + return value + + +def assign_heat_bus(): + """Selects heat_bus for chps used in district heating. + + Parameters + ---------- + scenario : str, optional + Name of the corresponding scenario. The default is 'eGon2035'. + + Returns + ------- + None. + + """ + sources = config.datasets()["chp_location"]["sources"] + target = config.datasets()["chp_location"]["targets"]["chp_table"] + + for scenario in config.settings()["egon-data"]["--scenarios"]: + # Select CHP with use_case = 'district_heating' + chp = db.select_geodataframe( + f""" + SELECT * FROM + {target['schema']}.{target['table']} + WHERE scenario = '{scenario}' + AND district_heating = True + """, + index_col="id", + epsg=4326, + ) + + # Select district heating areas and their centroid + district_heating = db.select_geodataframe( + f""" + SELECT area_id, ST_Centroid(geom_polygon) as geom + FROM + {sources['district_heating_areas']['schema']}. + {sources['district_heating_areas']['table']} + WHERE scenario = '{scenario}' + """, + epsg=4326, + ) + + # Assign district heating area_id to district_heating_chp + # According to nearest centroid of district heating area + chp["district_heating_area_id"] = chp.apply( + nearest, + df=district_heating, + row_geom_col="geom", + df_geom_col="geom", + centroid=True, + src_column="area_id", + axis=1, + ) + + # Drop district heating CHP without heat_bus_id + db.execute_sql( + f""" + DELETE FROM {target['schema']}.{target['table']} + WHERE scenario = '{scenario}' + AND district_heating = True + """ + ) + + # Insert district heating CHP with heat_bus_id + session = sessionmaker(bind=db.engine())() + for i, row in chp.iterrows(): + if row.carrier != "biomass": + entry = EgonChp( + id=i, + sources=row.sources, + source_id=row.source_id, + carrier=row.carrier, + el_capacity=row.el_capacity, + th_capacity=row.th_capacity, + electrical_bus_id=row.electrical_bus_id, + ch4_bus_id=row.ch4_bus_id, + district_heating_area_id=row.district_heating_area_id, + district_heating=row.district_heating, + voltage_level=row.voltage_level, + scenario=scenario, + geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + ) + else: + entry = EgonChp( + id=i, + sources=row.sources, + source_id=row.source_id, + carrier=row.carrier, + el_capacity=row.el_capacity, + th_capacity=row.th_capacity, + electrical_bus_id=row.electrical_bus_id, + district_heating_area_id=row.district_heating_area_id, + district_heating=row.district_heating, + voltage_level=row.voltage_level, + scenario=scenario, + geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + ) + session.add(entry) + session.commit() + + +def insert_biomass_chp(scenario): + """Insert biomass chp plants of future scenario + + Parameters + ---------- + scenario : str + Name of scenario. + + Returns + ------- + None. + + """ + cfg = config.datasets()["chp_location"] + + # import target values from NEP 2021, scneario C 2035 + target = select_target("biomass", scenario) + + # import data for MaStR + mastr = pd.read_csv( + WORKING_DIR_MASTR_OLD / cfg["sources"]["mastr_biomass"] + ).query("EinheitBetriebsstatus=='InBetrieb'") + + # Drop entries without federal state or 'AusschließlichWirtschaftszone' + mastr = mastr[ + mastr.Bundesland.isin( + pd.read_sql( + f"""SELECT DISTINCT ON (gen) + REPLACE(REPLACE(gen, '-', ''), 'ü', 'ue') as states + FROM {cfg['sources']['vg250_lan']['schema']}. + {cfg['sources']['vg250_lan']['table']}""", + con=db.engine(), + ).states.values + ) + ] + + # Scaling will be done per federal state in case of eGon2035 scenario. + if scenario == "eGon2035": + level = "federal_state" + else: + level = "country" + # Choose only entries with valid geometries inside DE/test mode + mastr_loc = filter_mastr_geometry(mastr).set_geometry("geometry") + + # Scale capacities to meet target values + mastr_loc = scale_prox2now(mastr_loc, target, level=level) + + # Assign bus_id + if len(mastr_loc) > 0: + mastr_loc["voltage_level"] = assign_voltage_level( + mastr_loc, cfg, WORKING_DIR_MASTR_OLD + ) + mastr_loc = assign_bus_id(mastr_loc, cfg) + mastr_loc = assign_use_case(mastr_loc, cfg["sources"], scenario) + + # Insert entries with location + session = sessionmaker(bind=db.engine())() + for i, row in mastr_loc.iterrows(): + if row.ThermischeNutzleistung > 0: + entry = EgonChp( + sources={ + "chp": "MaStR", + "el_capacity": "MaStR scaled with NEP 2021", + "th_capacity": "MaStR", + }, + source_id={"MastrNummer": row.EinheitMastrNummer}, + carrier="biomass", + el_capacity=row.Nettonennleistung, + th_capacity=row.ThermischeNutzleistung / 1000, + scenario=scenario, + district_heating=row.district_heating, + electrical_bus_id=row.bus_id, + voltage_level=row.voltage_level, + geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", + ) + session.add(entry) + session.commit() + + +def insert_chp_statusquo(scn="status2019"): + cfg = config.datasets()["chp_location"] + + # import data for MaStR + mastr = pd.read_csv( + WORKING_DIR_MASTR_NEW / "bnetza_mastr_combustion_cleaned.csv" + ) + + mastr_biomass = pd.read_csv( + WORKING_DIR_MASTR_NEW / "bnetza_mastr_biomass_cleaned.csv" + ) + + mastr = pd.concat([mastr, mastr_biomass]).reset_index(drop=True) + + mastr = mastr.loc[mastr.ThermischeNutzleistung > 0] + + mastr = mastr.loc[ + mastr.Energietraeger.isin( + [ + "Erdgas", + "Mineralölprodukte", + "andere Gase", + "nicht biogener Abfall", + "Braunkohle", + "Steinkohle", + "Biomasse", + ] + ) + ] + + mastr.Inbetriebnahmedatum = pd.to_datetime(mastr.Inbetriebnahmedatum) + mastr.DatumEndgueltigeStilllegung = pd.to_datetime( + mastr.DatumEndgueltigeStilllegung + ) + mastr = mastr.loc[ + mastr.Inbetriebnahmedatum + <= config.datasets()["mastr_new"][f"{scn}_date_max"] + ] + + mastr = mastr.loc[ + ( + mastr.DatumEndgueltigeStilllegung + >= config.datasets()["mastr_new"][f"{scn}_date_max"] + ) + | (mastr.DatumEndgueltigeStilllegung.isnull()) + ] + + mastr.groupby("Energietraeger").Nettonennleistung.sum().mul(1e-6) + + geom_municipalities = db.select_geodataframe( + """ + SELECT gen, ST_UNION(geometry) as geom + FROM boundaries.vg250_gem + GROUP BY gen + """ + ).set_index("gen") + + # Assing Laengengrad and Breitengrad to chps without location data + # based on the centroid of the municipaltiy + idx_no_location = mastr[ + (mastr.Laengengrad.isnull()) + & (mastr.Gemeinde.isin(geom_municipalities.index)) + ].index + + mastr.loc[idx_no_location, "Laengengrad"] = ( + geom_municipalities.to_crs(epsg="4326").centroid.x.loc[ + mastr.Gemeinde[idx_no_location] + ] + ).values + + mastr.loc[idx_no_location, "Breitengrad"] = ( + geom_municipalities.to_crs(epsg="4326").centroid.y.loc[ + mastr.Gemeinde[idx_no_location] + ] + ).values + + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + dropped_capacity = mastr[ + (mastr.Laengengrad.isnull()) + & (mastr.Bundesland == "SchleswigHolstein") + ].Nettonennleistung.sum() + + else: + dropped_capacity = mastr[ + (mastr.Laengengrad.isnull()) + ].Nettonennleistung.sum() + + print( + f""" + CHPs with a total installed electrical capacity of {dropped_capacity} kW are dropped + because of missing or wrong location data + """ + ) + + mastr = mastr[~mastr.Laengengrad.isnull()] + mastr = filter_mastr_geometry(mastr).set_geometry("geometry") + + # Assign bus_id + if len(mastr) > 0: + mastr["voltage_level"] = assign_voltage_level( + mastr, cfg, WORKING_DIR_MASTR_NEW + ) + + gas_bus_id = db.assign_gas_bus_id(mastr, scn, "CH4").bus + + mastr = assign_bus_id(mastr, cfg, drop_missing=True) + + mastr["gas_bus_id"] = gas_bus_id + + mastr = assign_use_case(mastr, cfg["sources"], scn) + + # Insert entries with location + session = sessionmaker(bind=db.engine())() + for i, row in mastr.iterrows(): + if row.ThermischeNutzleistung > 0: + entry = EgonChp( + sources={ + "chp": "MaStR", + "el_capacity": "MaStR", + "th_capacity": "MaStR", + }, + source_id={"MastrNummer": row.EinheitMastrNummer}, + carrier=map_carrier().loc[row.Energietraeger], + el_capacity=row.Nettonennleistung / 1000, + th_capacity=row.ThermischeNutzleistung / 1000, + scenario=scn, + district_heating=row.district_heating, + electrical_bus_id=row.bus_id, + ch4_bus_id=row.gas_bus_id, + voltage_level=row.voltage_level, + geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", + ) + session.add(entry) + session.commit() + + +def insert_chp_egon2035(): + """Insert CHP plants for eGon2035 considering NEP and MaStR data + + Returns + ------- + None. + + """ + + sources = config.datasets()["chp_location"]["sources"] + + targets = config.datasets()["chp_location"]["targets"] + + insert_biomass_chp("eGon2035") + + # Insert large CHPs based on NEP's list of conventional power plants + MaStR_konv = insert_large_chp(sources, targets["chp_table"], EgonChp) + + # Insert smaller CHPs (< 10MW) based on existing locations from MaStR + existing_chp_smaller_10mw(sources, MaStR_konv, EgonChp) + + gpd.GeoDataFrame( + MaStR_konv[ + [ + "EinheitMastrNummer", + "el_capacity", + "geometry", + "carrier", + "plz", + "city", + "federal_state", + ] + ] + ).to_postgis( + targets["mastr_conventional_without_chp"]["table"], + schema=targets["mastr_conventional_without_chp"]["schema"], + con=db.engine(), + if_exists="replace", + ) + + +def extension_BW(): + extension_per_federal_state("BadenWuerttemberg", EgonChp) + + +def extension_BY(): + extension_per_federal_state("Bayern", EgonChp) + + +def extension_HB(): + extension_per_federal_state("Bremen", EgonChp) + + +def extension_BB(): + extension_per_federal_state("Brandenburg", EgonChp) + + +def extension_HH(): + extension_per_federal_state("Hamburg", EgonChp) + + +def extension_HE(): + extension_per_federal_state("Hessen", EgonChp) + + +def extension_MV(): + extension_per_federal_state("MecklenburgVorpommern", EgonChp) + + +def extension_NS(): + extension_per_federal_state("Niedersachsen", EgonChp) + + +def extension_NW(): + extension_per_federal_state("NordrheinWestfalen", EgonChp) + + +def extension_SN(): + extension_per_federal_state("Sachsen", EgonChp) + + +def extension_TH(): + extension_per_federal_state("Thueringen", EgonChp) + + +def extension_SL(): + extension_per_federal_state("Saarland", EgonChp) + + +def extension_ST(): + extension_per_federal_state("SachsenAnhalt", EgonChp) + + +def extension_RP(): + extension_per_federal_state("RheinlandPfalz", EgonChp) + + +def extension_BE(): + extension_per_federal_state("Berlin", EgonChp) + + +def extension_SH(): + extension_per_federal_state("SchleswigHolstein", EgonChp) + + +def insert_chp_egon100re(): + """Insert CHP plants for eGon100RE considering results from pypsa-eur-sec + + Returns + ------- + None. + + """ + + sources = config.datasets()["chp_location"]["sources"] + + db.execute_sql( + f""" + DELETE FROM {EgonChp.__table__.schema}.{EgonChp.__table__.name} + WHERE scenario = 'eGon100RE' + """ + ) + + # select target values from pypsa-eur-sec + additional_capacity = db.select_dataframe( + """ + SELECT capacity + FROM supply.egon_scenario_capacities + WHERE scenario_name = 'eGon100RE' + AND carrier = 'urban_central_gas_CHP' + """ + ).capacity[0] + + if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": + additional_capacity /= 16 + target_file = ( + Path(".") + / "data_bundle_egon_data" + / "pypsa_eur_sec" + / "2022-07-26-egondata-integration" + / "postnetworks" + / "elec_s_37_lv2.0__Co2L0-1H-T-H-B-I-dist1_2050.nc" + ) + + network = pypsa.Network(str(target_file)) + chp_index = "DE0 0 urban central gas CHP" + + standard_chp_th = 10 + standard_chp_el = ( + standard_chp_th + * network.links.loc[chp_index, "efficiency"] + / network.links.loc[chp_index, "efficiency2"] + ) + + areas = db.select_geodataframe( + f""" + SELECT + residential_and_service_demand as demand, area_id, + ST_Transform(ST_PointOnSurface(geom_polygon), 4326) as geom + FROM + {sources['district_heating_areas']['schema']}. + {sources['district_heating_areas']['table']} + WHERE scenario = 'eGon100RE' + """ + ) + + existing_chp = pd.DataFrame( + data={ + "el_capacity": standard_chp_el, + "th_capacity": standard_chp_th, + "voltage_level": 5, + }, + index=range(1), + ) + + flh = ( + network.links_t.p0[chp_index].sum() + / network.links.p_nom_opt[chp_index] + ) + + extension_to_areas( + areas, + additional_capacity, + existing_chp, + flh, + EgonChp, + district_heating=True, + scenario="eGon100RE", + ) + + +tasks = (create_tables,) + +insert_per_scenario = set() + +if "status2019" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add( + wrapped_partial(insert_chp_statusquo, scn="status2019", postfix="_2019") + ) + +if "status2023" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add( + wrapped_partial(insert_chp_statusquo, scn="status2023", postfix="_2023") + ) + +if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add(insert_chp_egon2035) + +if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: + insert_per_scenario.add(insert_chp_egon100re) + +tasks = tasks + (insert_per_scenario, assign_heat_bus) + +extension = set() + +if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + # Add one task per federal state for small CHP extension + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + extension = extension_SH + else: + extension = { + extension_BW, + extension_BY, + extension_HB, + extension_BB, + extension_HE, + extension_MV, + extension_NS, + extension_NW, + extension_SH, + extension_HH, + extension_RP, + extension_SL, + extension_SN, + extension_ST, + extension_TH, + extension_BE, + } + +if extension != set(): + tasks = tasks + (extension,) + + +class Chp(Dataset): + def __init__(self, dependencies): + super().__init__( + name="Chp", version="0.0.8", dependencies=dependencies, tasks=tasks + ) diff --git a/src/egon/data/datasets/data_bundle/__init__.py b/src/egon/data/datasets/data_bundle/__init__.py index ce16f3fdc..b49f64327 100644 --- a/src/egon/data/datasets/data_bundle/__init__.py +++ b/src/egon/data/datasets/data_bundle/__init__.py @@ -1,4 +1,4 @@ -"""The central module containing all code dealing with small scale inpu-data +"""The central module containing all code dealing with small scale input-data """ @@ -13,7 +13,7 @@ def download(): """ - Download small scale imput data from Zenodo + Download small scale input data from Zenodo Parameters ---------- @@ -27,8 +27,10 @@ def download(): url = f"""https://zenodo.org/record/{sources['deposit_id']}/files/data_bundle_egon_data.zip""" target_file = egon.data.config.datasets()["data-bundle"]["targets"]["file"] - # Retrieve files - urlretrieve(url, target_file) + # check if file exists + if not Path(target_file).exists(): + # Retrieve files + urlretrieve(url, target_file) with zipfile.ZipFile(target_file, "r") as zip_ref: zip_ref.extractall(".") @@ -41,7 +43,7 @@ def __init__(self, dependencies): ]["deposit_id"] super().__init__( name="DataBundle", - version=str(deposit_id) + "-0.0.2", + version=str(deposit_id) + "-0.0.3", dependencies=dependencies, tasks=(download), ) diff --git a/src/egon/data/datasets/demandregio/__init__.py b/src/egon/data/datasets/demandregio/__init__.py index ef8b384d2..a2a837ddc 100644 --- a/src/egon/data/datasets/demandregio/__init__.py +++ b/src/egon/data/datasets/demandregio/__init__.py @@ -3,14 +3,17 @@ """ from pathlib import Path +import subprocess +import os +import zipfile from sqlalchemy import ARRAY, Column, Float, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base import numpy as np import pandas as pd -from egon.data import db -from egon.data.datasets import Dataset +from egon.data import db, logger +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.demandregio.install_disaggregator import ( clone_and_install, ) @@ -18,11 +21,12 @@ EgonScenario, get_sector_parameters, ) +from egon.data.datasets.zensus import download_and_check import egon.data.config import egon.data.datasets.scenario_parameters.parameters as scenario_parameters try: - from disaggregator import config, data, spatial + from disaggregator import config, data, spatial, temporal except ImportError as e: pass @@ -35,10 +39,11 @@ class DemandRegio(Dataset): def __init__(self, dependencies): super().__init__( name="DemandRegio", - version="0.0.6", + version="0.0.9", dependencies=dependencies, tasks=( clone_and_install, + get_cached_tables, # adhoc workaround #180 create_tables, { insert_household_demand, @@ -48,6 +53,14 @@ def __init__(self, dependencies): ), ) +class DemandRegioLoadProfiles(Base): + __tablename__ = "demandregio_household_load_profiles" + __table_args__ = {"schema": "demand"} + + id = Column(Integer, primary_key=True) + year = Column(Integer) + nuts3 = Column(String) + load_in_mwh = Column(ARRAY(Float())) class EgonDemandRegioHH(Base): __tablename__ = "egon_demandregio_hh" @@ -423,18 +436,40 @@ def disagg_households_power( pd.DataFrame or pd.Series """ # source: survey of energieAgenturNRW + # with/without direct water heating (DHW), and weighted average + # https://1-stromvergleich.com/wp-content/uploads/erhebung_wo_bleibt_der_strom.pdf demand_per_hh_size = pd.DataFrame( index=range(1, 7), data={ - "weighted DWH": [2290, 3202, 4193, 4955, 5928, 5928], - "without DHW": [1714, 2812, 3704, 4432, 5317, 5317], + # "weighted DWH": [2290, 3202, 4193, 4955, 5928, 5928], + # "without DHW": [1714, 2812, 3704, 4432, 5317, 5317], + "with_DHW": [2181, 3843, 5151, 6189, 7494, 8465], + "without_DHW": [1798, 2850, 3733, 4480, 5311, 5816], + "weighted": [2256, 3248, 4246, 5009, 5969, 6579], }, ) + if scenario == "eGon100RE": + # chose demand per household size from survey without DHW + power_per_HH = ( + demand_per_hh_size["without_DHW"] / 1e3 + ) # TODO why without? + + # calculate demand per nuts3 in 2011 + df_2011 = data.households_per_size(year=2011) * power_per_HH + + # scale demand per hh-size to meet demand without heat + # according to JRC in 2011 (136.6-(20.14+9.41) TWh) + # TODO check source and method + power_per_HH *= (136.6 - (20.14 + 9.41)) * 1e6 / df_2011.sum().sum() + + # calculate demand per nuts3 in 2050 + df = data.households_per_size(year=year) * power_per_HH + # Bottom-Up: Power demand by household sizes in [MWh/a] for each scenario - if scenario in ["status2019", "eGon2021", "eGon2035"]: + elif scenario in ["status2019", "status2023", "eGon2021", "eGon2035"]: # chose demand per household size from survey including weighted DHW - power_per_HH = demand_per_hh_size["weighted DWH"] / 1e3 + power_per_HH = demand_per_hh_size["weighted"] / 1e3 # calculate demand per nuts3 df = ( @@ -444,21 +479,16 @@ def disagg_households_power( if scenario == "eGon2035": # scale to fit demand of NEP 2021 scebario C 2035 (119TWh) - df *= 119000000 / df.sum().sum() + df *= 119 * 1e6 / df.sum().sum() - elif scenario == "eGon100RE": - # chose demand per household size from survey without DHW - power_per_HH = demand_per_hh_size["without DHW"] / 1e3 + if scenario == "status2023": + # scale to fit demand of BDEW 2023 (130.48 TWh) see issue #180 + df *= 130.48 * 1e6 / df.sum().sum() - # calculate demand per nuts3 in 2011 - df_2011 = data.households_per_size(year=2011) * power_per_HH - - # scale demand per hh-size to meet demand without heat - # according to JRC in 2011 (136.6-(20.14+9.41) TWh) - power_per_HH *= (136.6 - (20.14 + 9.41)) * 1e6 / df_2011.sum().sum() - - # calculate demand per nuts3 in 2050 - df = data.households_per_size(year=year) * power_per_HH + # if scenario == "status2021": # TODO status2021 + # # scale to fit demand of AGEB 2021 (138.6 TWh) + # # https://ag-energiebilanzen.de/wp-content/uploads/2023/01/AGEB_22p2_rev-1.pdf#page=10 + # df *= 138.6 * 1e6 / df.sum().sum() else: print( @@ -471,6 +501,56 @@ def disagg_households_power( return df +def write_demandregio_hh_profiles_to_db(hh_profiles): + """Write HH demand profiles from demand regio into db. One row per + year and nuts3. The annual load profile timeseries is an array. + + schema: demand + tablename: demandregio_household_load_profiles + + + + Parameters + ---------- + hh_profiles: pd.DataFrame + + Returns + ------- + """ + years = hh_profiles.index.year.unique().values + df_to_db = pd.DataFrame(columns=["id", "year", "nuts3", "load_in_mwh"]).set_index("id") + dataset = egon.data.config.settings()["egon-data"]["--dataset-boundary"] + + if dataset == "Schleswig-Holstein": + hh_profiles = hh_profiles.loc[ + :, hh_profiles.columns.str.contains("DEF0")] + + id = 0 + for year in years: + df = hh_profiles[hh_profiles.index.year == year] + for nuts3 in hh_profiles.columns: + id+=1 + df_to_db.at[id, "year"] = year + df_to_db.at[id, "nuts3"] = nuts3 + df_to_db.at[id, "load_in_mwh"] = df[nuts3].to_list() + + df_to_db["year"] = df_to_db["year"].apply(int) + df_to_db["nuts3"] = df_to_db["nuts3"].astype(str) + df_to_db["load_in_mwh"] = df_to_db["load_in_mwh"].apply(list) + df_to_db = df_to_db.reset_index() + + DemandRegioLoadProfiles.__table__.drop(bind=db.engine(), checkfirst=True) + DemandRegioLoadProfiles.__table__.create(bind=db.engine()) + + df_to_db.to_sql( + name=DemandRegioLoadProfiles.__table__.name, + schema=DemandRegioLoadProfiles.__table__.schema, + con=db.engine(), + if_exists="append", + index=-False, + ) + + return def insert_hh_demand(scenario, year, engine): """Calculates electrical demands of private households using demandregio's @@ -479,7 +559,7 @@ def insert_hh_demand(scenario, year, engine): Parameters ---------- scenario : str - Name of the corresponing scenario. + Name of the corresponding scenario. year : int The number of households per region is taken from this year. @@ -500,7 +580,9 @@ def insert_hh_demand(scenario, year, engine): # insert into database for hh_size in ec_hh.columns: df = pd.DataFrame(ec_hh[hh_size]) - df["year"] = year + df["year"] = 2023 if scenario == "status2023" else year # TODO status2023 + # adhoc fix until ffeopendata servers are up and population_year can be set + df["scenario"] = scenario df["hh_size"] = hh_size df = df.rename({hh_size: "demand"}, axis="columns") @@ -511,6 +593,37 @@ def insert_hh_demand(scenario, year, engine): if_exists="append", ) + # insert housholds demand timeseries + try: + hh_load_timeseries = ( + temporal.disagg_temporal_power_housholds_slp( + use_nuts3code=True, + by="households", + weight_by_income=False, + year=year, + ) + .resample("h") + .sum() + ) + hh_load_timeseries.rename( + columns={"DEB16": "DEB1C", "DEB19": "DEB1D"}, inplace=True) + except Exception as e: + logger.warning(f"Couldnt get profiles from FFE, will use pickeld fallback! \n {e}") + hh_load_timeseries = pd.read_pickle(Path(".", "df_load_profiles.pkl").resolve()) + + def change_year(dt, year): + return dt.replace(year=year) + + year = 2023 if scenario == "status2023" else year # TODO status2023 + hh_load_timeseries.index = hh_load_timeseries.index.map(lambda dt: change_year(dt, year)) + + if scenario == "status2023": + hh_load_timeseries = hh_load_timeseries.shift(24 * 2) + + hh_load_timeseries.iloc[:24 * 7] = hh_load_timeseries.iloc[24 * 7:24 * 7 * 2].values + + write_demandregio_hh_profiles_to_db(hh_load_timeseries) + def insert_cts_ind(scenario, year, engine, target_values): """Calculates electrical demands of CTS and industry using demandregio's @@ -550,7 +663,7 @@ def insert_cts_ind(scenario, year, engine, target_values): # scale values according to target_values if sector in target_values[scenario].keys(): ec_cts_ind *= ( - target_values[scenario][sector] * 1e3 / ec_cts_ind.sum().sum() + target_values[scenario][sector] / ec_cts_ind.sum().sum() ) # include new largescale consumers according to NEP 2021 @@ -595,8 +708,6 @@ def insert_household_demand(): scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] - scenarios.append("eGon2021") - for t in targets: db.execute_sql( f"DELETE FROM {targets[t]['schema']}.{targets[t]['table']};" @@ -632,7 +743,6 @@ def insert_cts_ind_demands(): scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] - scenarios.append("eGon2021") for scn in scenarios: year = scenario_parameters.global_settings(scn)["population_year"] @@ -644,14 +754,23 @@ def insert_cts_ind_demands(): target_values = { # according to NEP 2021 # new consumers will be added seperatly - "eGon2035": {"CTS": 135300, "industry": 225400}, + "eGon2035": { + "CTS": 135300 * 1e3, + "industry": 225400 * 1e3 + }, # CTS: reduce overall demand from demandregio (without traffic) # by share of heat according to JRC IDEES, data from 2011 # industry: no specific heat demand, use data from demandregio - "eGon100RE": {"CTS": (1 - (5.96 + 6.13) / 154.64) * 125183.403}, + "eGon100RE": { + "CTS": ((1 - (5.96 + 6.13) / 154.64) * 125183.403) * 1e3 + }, # no adjustments for status quo "eGon2021": {}, "status2019": {}, + "status2023": { + "CTS": 121160 * 1e3, + "industry": 200380 * 1e3 + }, } insert_cts_ind(scn, year, engine, target_values) @@ -802,8 +921,26 @@ def timeseries_per_wz(): """ - years = get_sector_parameters("global").weather_year.unique() + scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] + + for scn in scenarios: + year = int(scenario_parameters.global_settings(scn)["weather_year"]) - for year in years: for sector in ["CTS", "industry"]: insert_timeseries_per_wz(sector, int(year)) + + +def get_cached_tables(): + """Get cached demandregio tables and db-dump from former runs""" + data_config = egon.data.config.datasets() + for s in ["cache", "dbdump"]: + url = data_config["demandregio_workaround"]["source"][s]["url"] + target_path = data_config["demandregio_workaround"]["targets"][s]["path"] + filename = os.path.basename(url) + file_path = Path(".", target_path, filename).resolve() + os.makedirs(file_path.parent, exist_ok=True) + logger.info(f"Downloading: {filename} from {url}.") + download_and_check(url, file_path, max_iteration=5) + with zipfile.ZipFile(file_path, "r") as zip_ref: + zip_ref.extractall(file_path.parent) + diff --git a/src/egon/data/datasets/demandregio/install_disaggregator.py b/src/egon/data/datasets/demandregio/install_disaggregator.py index 552242628..fa80831f9 100644 --- a/src/egon/data/datasets/demandregio/install_disaggregator.py +++ b/src/egon/data/datasets/demandregio/install_disaggregator.py @@ -1,16 +1,18 @@ """This module downloads and installs demandregio's disaggregator from GitHub """ +from pathlib import Path +from subprocess import check_output +import importlib.util import os import shutil -from pathlib import Path +from egon.data import logger, subprocess import egon.data.config -from egon.data import subprocess def clone_and_install(): - """ Clone and install repository of demandregio's disaggregator + """Clone and install repository of demandregio's disaggregator Returns ------- @@ -26,27 +28,48 @@ def clone_and_install(): ] ) - # Delete repository if it already exists - if repo_path.exists() and repo_path.is_dir(): - shutil.rmtree(repo_path) - - # Create subfolder - os.mkdir(repo_path) - - # Clone from GitHub repository - subprocess.run( - [ - "git", - "clone", - "--single-branch", - "--branch", - source["branch"], - source["git-repository"], - ], - cwd=repo_path, - ) + try: + status = check_output( + ["git", "status"], cwd=(repo_path / "disaggregator").absolute() + ) + if status.startswith(b"Auf Branch features/pandas-update"): + logger.info("Demandregio cloned and right branch checked out.") + else: + raise ImportError( + "Demandregio cloned but wrong branch checked " + "out. Please checkout features/pandas-update" + ) + spec = importlib.util.find_spec("disaggregator") + if spec is not None: + logger.info("Demandregio is not installed. Installing now.") + # Install disaggregator from path + subprocess.run( + [ + "pip", + "install", + "-e", + (repo_path / "disaggregator").absolute(), + ] + ) - # Install disaggregator from path - subprocess.run( - ["pip", "install", "-e", (repo_path / "disaggregator").absolute()] - ) + except subprocess.CalledProcessError and FileNotFoundError: + # Create subfolder + os.makedirs(repo_path, exist_ok=True) + + # Clone from GitHub repository + subprocess.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + source["branch"], + source["git-repository"], + ], + cwd=repo_path, + ) + + # Install disaggregator from path + subprocess.run( + ["pip", "install", "-e", (repo_path / "disaggregator").absolute()] + ) diff --git a/src/egon/data/datasets/district_heating_areas/__init__.py b/src/egon/data/datasets/district_heating_areas/__init__.py index ce906ba2c..02ff01192 100644 --- a/src/egon/data/datasets/district_heating_areas/__init__.py +++ b/src/egon/data/datasets/district_heating_areas/__init__.py @@ -521,9 +521,10 @@ def district_heating_areas(scenario_name, plotting=False): minimum_connection_rate = 0.3 - # Adjust minimum connection rate for status2019, + # Adjust minimum connection rate for status2019, and other statusquo scn # otherwise the existing district heating grids would have too much demand - if scenario_name == "status2019": + # if scenario_name == "status2019": + if "status" in scenario_name: minimum_connection_rate = 0.6 # heat_demand is scenario specific diff --git a/src/egon/data/datasets/district_heating_areas/plot.py b/src/egon/data/datasets/district_heating_areas/plot.py index c0c81f3e5..9cd3b2f58 100644 --- a/src/egon/data/datasets/district_heating_areas/plot.py +++ b/src/egon/data/datasets/district_heating_areas/plot.py @@ -53,7 +53,7 @@ def plot_heat_density_sorted(heat_denisty_per_scenario, scenario_name=None): fig, ax = plt.subplots(1, 1) colors = pd.DataFrame( - columns=["share", "curve"], index=["status2019", "eGon2035", "eGon100RE"] + columns=["share", "curve"], index=["status2019", "status2023", "eGon2035", "eGon100RE"] ) colors["share"]["eGon2035"] = "darkblue" @@ -62,6 +62,9 @@ def plot_heat_density_sorted(heat_denisty_per_scenario, scenario_name=None): colors["curve"]["eGon100RE"] = "orange" colors["share"]["status2019"] = "darkgreen" colors["curve"]["status2019"] = "green" + colors["share"]["status2023"] = "darkgrey" + colors["curve"]["status2023"] = "grey" + # TODO status2023 set plotting=False? for scenario in heat_denisty_per_scenario.keys(): diff --git a/src/egon/data/datasets/electrical_neighbours.py b/src/egon/data/datasets/electrical_neighbours.py index 0daed54fd..90b4a678c 100644 --- a/src/egon/data/datasets/electrical_neighbours.py +++ b/src/egon/data/datasets/electrical_neighbours.py @@ -1,25 +1,28 @@ """The central module containing all code dealing with electrical neighbours """ +from os import path +from pathlib import Path +import datetime +import logging +import os.path import zipfile +from shapely.geometry import LineString +from sqlalchemy.orm import sessionmaker import entsoe -import requests -import logging - import geopandas as gpd import pandas as pd -from shapely.geometry import LineString -from sqlalchemy.orm import sessionmaker +import requests -import egon.data.datasets.etrago_setup as etrago -import egon.data.datasets.scenario_parameters.parameters as scenario_parameters -from egon.data import config, db -from egon.data.datasets import Dataset -from egon.data.datasets.fix_ehv_subnetworks import select_bus_id +from egon.data import config, db, logger +from egon.data.db import session_scope +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.fill_etrago_gen import add_marginal_costs +from egon.data.datasets.fix_ehv_subnetworks import select_bus_id from egon.data.datasets.scenario_parameters import get_sector_parameters -from os import path +import egon.data.datasets.etrago_setup as etrago +import egon.data.datasets.scenario_parameters.parameters as scenario_parameters def get_cross_border_buses(scenario, sources): @@ -248,7 +251,11 @@ def buses(scenario, sources, targets): central_buses.scn_name = scenario # Insert all central buses for eGon2035 - if scenario in ["eGon2035", "status2019"]: + if scenario in [ + "eGon2035", + "status2019", + "status2023", + ]: # TODO: status2023 this is hardcoded shit central_buses.to_postgis( targets["buses"]["table"], schema=targets["buses"]["schema"], @@ -588,7 +595,7 @@ def foreign_dc_lines(scenario, sources, targets, central_buses): AND country != 'DE') """ ) - capital_cost = get_sector_parameters("electricity", "eGon2035")[ + capital_cost = get_sector_parameters("electricity", scenario)[ "capital_cost" ] @@ -879,7 +886,7 @@ def calc_capacities(): ] -def insert_generators(capacities): +def insert_generators_tyndp(capacities): """Insert generators for foreign countries based on TYNDP-data Parameters @@ -1037,7 +1044,7 @@ def insert_generators(capacities): session.commit() -def insert_storage(capacities): +def insert_storage_tyndp(capacities): """Insert storage units for foreign countries based on TYNDP-data Parameters @@ -1108,9 +1115,9 @@ def insert_storage(capacities): for x in parameters: store.loc[store["carrier"] == "battery", x] = parameters_battery[x] - store.loc[ - store["carrier"] == "pumped_hydro", x - ] = parameters_pumped_hydro[x] + store.loc[store["carrier"] == "pumped_hydro", x] = ( + parameters_pumped_hydro[x] + ) # insert data session = sessionmaker(bind=db.engine())() @@ -1168,9 +1175,9 @@ def tyndp_generation(): capacities = calc_capacities() - insert_generators(capacities) + insert_generators_tyndp(capacities) - insert_storage(capacities) + insert_storage_tyndp(capacities) def tyndp_demand(): @@ -1231,9 +1238,9 @@ def tyndp_demand(): ] # Assign etrago bus_id to TYNDP nodes buses = pd.DataFrame({"nodes": nodes}) - buses.loc[ - buses[buses.nodes.isin(map_buses.keys())].index, "nodes" - ] = buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses) + buses.loc[buses[buses.nodes.isin(map_buses.keys())].index, "nodes"] = ( + buses[buses.nodes.isin(map_buses.keys())].nodes.map(map_buses) + ) buses.loc[:, "bus"] = ( get_foreign_bus_id(scenario="eGon2035") .loc[buses.loc[:, "nodes"]] @@ -1300,12 +1307,26 @@ def tyndp_demand(): session.commit() +def get_entsoe_token(): + """Check for token in home dir. If not exists, check in working dir""" + token_path = path.join(path.expanduser("~"), ".entsoe-token") + if not os.path.isfile(token_path): + logger.info( + f"Token file not found at {token_path}. Will check in working directory." + ) + token_path = Path(".entsoe-token") + if os.path.isfile(token_path): + logger.info(f"Token found at {token_path}") + entsoe_token = open(token_path, "r").read(36) + if entsoe_token is None: + raise FileNotFoundError("No entsoe-token found.") + return entsoe_token + + def entsoe_historic_generation_capacities( year_start="20190101", year_end="20200101" ): - entsoe_token = open( - path.join(path.expanduser("~"), ".entsoe-token"), "r" - ).read(36) + entsoe_token = get_entsoe_token() client = entsoe.EntsoePandasClient(api_key=entsoe_token) start = pd.Timestamp(year_start, tz="Europe/Brussels") @@ -1327,7 +1348,12 @@ def entsoe_historic_generation_capacities( "SE", "GB", ] - + # No GB data after Brexit + if int(year_start[:4]) > 2021: + logger.warning( + "No GB data after Brexit. GB is dropped from entsoe query!" + ) + countries = [c for c in countries if c != "GB"] # todo: define wanted countries not_retrieved = [] @@ -1338,29 +1364,53 @@ def entsoe_historic_generation_capacities( else: kwargs = dict(start=start, end=end) try: - dfs.append( - client.query_installed_generation_capacity(country, **kwargs) + country_data = client.query_installed_generation_capacity( + country, **kwargs ) - + dfs.append(country_data) except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): + logger.warning( + f"Data for country: {country} could not be retrieved." + ) not_retrieved.append(country) pass - if not_retrieved: - logging.warning( - f"Data for country (-ies) {', '.join(not_retrieved)} could not be retrieved." - ) - df = pd.concat(dfs) - df["country"] = countries - df.set_index("country", inplace=True) - df.fillna(0, inplace=True) - return df + if dfs: + df = pd.concat(dfs) + df["country"] = [c for c in countries if c not in not_retrieved] + df.set_index("country", inplace=True) + if int(year_start[:4]) == 2023: + # https://www.bmreports.com/bmrs/?q=foregeneration/capacityaggregated + # could probably somehow be automised + # https://www.elexonportal.co.uk/category/view/178 + # in MW + installed_capacity_gb = pd.Series( + { + "Biomass": 4438, + "Fossil Gas": 37047, + "Fossil Hard coal": 1491, + "Hydro Pumped Storage": 5603, + "Hydro Run-of-river and poundage": 2063, + "Nuclear": 4950, + "Other": 3313, + "Other renewable": 1462, + "Solar": 14518, + "Wind Offshore": 13038, + "Wind Onshore": 13907, + }, + name="GB", + ) + df = pd.concat([df.T, installed_capacity_gb], axis=1).T + logger.info("Manually added generation capacities for GB 2023.") + not_retrieved = [c for c in not_retrieved if c != "GB"] + df.fillna(0, inplace=True) + else: + df = pd.DataFrame() + return df, not_retrieved def entsoe_historic_demand(year_start="20190101", year_end="20200101"): - entsoe_token = open( - path.join(path.expanduser("~"), ".entsoe-token"), "r" - ).read(36) + entsoe_token = get_entsoe_token() client = entsoe.EntsoePandasClient(api_key=entsoe_token) start = pd.Timestamp(year_start, tz="Europe/Brussels") @@ -1407,17 +1457,18 @@ def entsoe_historic_demand(year_start="20190101", year_end="20200101"): dfs.append(country_data) except (entsoe.exceptions.NoMatchingDataError, requests.HTTPError): not_retrieved.append(country) + logger.warning( + f"Data for country: {country} could not be retrieved." + ) pass - if not_retrieved: - logging.warning( - f"Data for country (-ies) {', '.join(not_retrieved)} could not be retrieved." - ) - df = pd.concat(dfs, axis=1) - df.columns = countries - df.index = pd.date_range(year_start, periods=8760, freq="H") - - return df + if dfs: + df = pd.concat(dfs, axis=1) + df.columns = [c for c in countries if c not in not_retrieved] + df.index = pd.date_range(year_start, periods=8760, freq="H") + else: + df = pd.DataFrame() + return df, not_retrieved def map_carriers_entsoe(): @@ -1451,7 +1502,7 @@ def map_carriers_entsoe(): } -def entsoe_to_bus_etrago(): +def entsoe_to_bus_etrago(scn_name): map_entsoe = pd.Series( { "LU": "LU00", @@ -1470,11 +1521,46 @@ def entsoe_to_bus_etrago(): } ) - for_bus = get_foreign_bus_id(scenario="status2019") + for_bus = get_foreign_bus_id(scenario=scn_name) return map_entsoe.map(for_bus) +def save_entsoe_data(df: pd.DataFrame, file_path: Path): + os.makedirs(file_path.parent, exist_ok=True) + if not df.empty: + df.to_csv(file_path, index_label="Index") + logger.info( + f"Saved entsoe data for {file_path.stem} " + f"to {file_path.parent} for countries: {df.index}" + ) + + +def fill_by_backup_data_from_former_runs(df_sq, file_path, not_retrieved): + """ + Fills missing data from former runs + Parameters + ---------- + df_sq: pd.DataFrame + file_path: str, Path + not_retrieved: list + + Returns + ------- + df_sq, not_retrieved + + """ + sq_backup = pd.read_csv(file_path, index_col="Index") + # check for missing columns in backup (former runs) + c_backup = [c for c in sq_backup.columns if c in not_retrieved] + # remove columns, if found in backup + not_retrieved = [c for c in not_retrieved if c not in c_backup] + if c_backup: + df_sq = pd.concat([df_sq, sq_backup.loc[:, c_backup]], axis=1) + logger.info(f"Appended data from former runs for {c_backup}") + return df_sq, not_retrieved + + def insert_generators_sq(scn_name="status2019"): """ Insert generators for foreign countries based on ENTSO-E data @@ -1492,16 +1578,41 @@ def insert_generators_sq(scn_name="status2019"): None. """ - try: - gen_sq = entsoe_historic_generation_capacities() - except: - logging.warning( - """Generation data from entsoe could not be retrieved. - Backup data is used instead""" + if "status" in scn_name: + year = int(scn_name.split("status")[-1]) + year_start_end = { + "year_start": f"{year}0101", + "year_end": f"{year+1}0101", + } + else: + raise ValueError("No valid scenario name!") + + df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( + **year_start_end + ) + + if not_retrieved: + logger.warning("Generation data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_gen_sq, file_path, not_retrieved + ) + save_entsoe_data(df_gen_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" ) - gen_sq = pd.read_csv( + df_gen_sq_backup = pd.read_csv( "data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" ) + df_gen_sq = pd.concat( + [df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 + ) targets = config.datasets()["electrical_neighbours"]["targets"] # Delete existing data @@ -1531,20 +1642,20 @@ def insert_generators_sq(scn_name="status2019"): AND scn_name = '{scn_name}' """ ) - entsoe_to_bus = entsoe_to_bus_etrago() + entsoe_to_bus = entsoe_to_bus_etrago(scn_name) carrier_entsoe = map_carriers_entsoe() - gen_sq = gen_sq.groupby(axis=1, by=carrier_entsoe).sum() + df_gen_sq = df_gen_sq.groupby(axis=1, by=carrier_entsoe).sum() # Filter generators modeled as storage and geothermal - gen_sq = gen_sq.loc[ - :, ~gen_sq.columns.isin(["Hydro Pumped Storage", "geo_thermal"]) + df_gen_sq = df_gen_sq.loc[ + :, ~df_gen_sq.columns.isin(["Hydro Pumped Storage", "geo_thermal"]) ] list_gen_sq = pd.DataFrame( dtype=int, columns=["carrier", "country", "capacity"] ) - for carrier in gen_sq.columns: - gen_carry = gen_sq[carrier] + for carrier in df_gen_sq.columns: + gen_carry = df_gen_sq[carrier] for country, cap in gen_carry.items(): gen = pd.DataFrame( {"carrier": carrier, "country": country, "capacity": cap}, @@ -1596,8 +1707,8 @@ def insert_generators_sq(scn_name="status2019"): AND scn_name = '{scn_name}') AND scn_name = '{scn_name}' """ - gen_sq = pd.read_sql_query(sql, db.engine()) - gen_sq = gen_sq[gen_sq.carrier.isin(renew_carriers_sq)] + df_gen_sq = pd.read_sql_query(sql, db.engine()) + df_gen_sq = df_gen_sq[df_gen_sq.carrier.isin(renew_carriers_sq)] sql = f""" SELECT * FROM {targets['generators']['schema']}.{targets['generators']['table']} @@ -1614,7 +1725,7 @@ def insert_generators_sq(scn_name="status2019"): # egon_sq_to_100 map the timeseries used in the scenario eGon100RE # to the same bus and carrier for the status quo scenario egon_sq_to_100 = {} - for i, gen in gen_sq.iterrows(): + for i, gen in df_gen_sq.iterrows(): gen_id_100 = gen_100[ (gen_100["bus"] == gen["bus"]) & (gen_100["carrier"] == gen["carrier"]) @@ -1623,20 +1734,126 @@ def insert_generators_sq(scn_name="status2019"): egon_sq_to_100[gen["generator_id"]] = gen_id_100 # insert generators_timeseries data - session = sessionmaker(bind=db.engine())() + with session_scope() as session: + for gen_id in df_gen_sq.generator_id: + serie = series_egon100[ + series_egon100.generator_id == egon_sq_to_100[gen_id] + ]["p_max_pu"].values[0] + entry = etrago.EgonPfHvGeneratorTimeseries( + scn_name=scn_name, generator_id=gen_id, temp_id=1, p_max_pu=serie + ) - for gen_id in gen_sq.generator_id: - serie = series_egon100[ - series_egon100.generator_id == egon_sq_to_100[gen_id] - ]["p_max_pu"].values[0] - entry = etrago.EgonPfHvGeneratorTimeseries( - scn_name=scn_name, generator_id=gen_id, temp_id=1, p_max_pu=serie + session.add(entry) + session.commit() + + return + + +def insert_storage_units_sq(scn_name="status2019"): + """ + Insert storage_units for foreign countries based on ENTSO-E data + + Parameters + ---------- + scn_name : str + Scenario to which the foreign storage units will be assigned. + The default is "status2019". + + Returns + ------- + None. + + """ + if "status" in scn_name: + year = int(scn_name.split("status")[-1]) + year_start_end = { + "year_start": f"{year}0101", + "year_end": f"{year+1}0101", + } + else: + raise ValueError("No valid scenario name!") + + df_gen_sq, not_retrieved = entsoe_historic_generation_capacities( + **year_start_end + ) + + if not_retrieved: + logger.warning("Generation data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"gen_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_gen_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_gen_sq, file_path, not_retrieved + ) + save_entsoe_data(df_gen_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" + ) + df_gen_sq_backup = pd.read_csv( + "data_bundle_egon_data/entsoe/gen_entsoe.csv", index_col="Index" + ) + df_gen_sq = pd.concat( + [df_gen_sq, df_gen_sq_backup.loc[not_retrieved]], axis=1 ) - session.add(entry) - session.commit() + sto_sq = df_gen_sq.loc[:, df_gen_sq.columns == "Hydro Pumped Storage"] + sto_sq.rename(columns={"Hydro Pumped Storage": "p_nom"}, inplace=True) - return + targets = config.datasets()["electrical_neighbours"]["targets"] + + # Delete existing data + db.execute_sql( + f""" + DELETE FROM {targets['storage']['schema']}.{targets['storage']['table']} + WHERE bus IN ( + SELECT bus_id FROM + {targets['buses']['schema']}.{targets['buses']['table']} + WHERE country != 'DE' + AND scn_name = '{scn_name}') + AND scn_name = '{scn_name}' + """ + ) + + # Add missing information suitable for eTraGo selected from scenario_parameter table + parameters_pumped_hydro = get_sector_parameters(sector="electricity", scenario=scn_name)["efficiency"]["pumped_hydro"] + + # Set bus_id + entsoe_to_bus = entsoe_to_bus_etrago(scn_name=scn_name) + sto_sq["bus"] = sto_sq.index.map(entsoe_to_bus) + + # Insert carrier specific parameters + sto_sq["carrier"] = "pumped_hydro" + sto_sq["scn_name"] = scn_name + sto_sq["dispatch"] = parameters_pumped_hydro["dispatch"] + sto_sq["store"] = parameters_pumped_hydro["store"] + sto_sq["standing_loss"] = parameters_pumped_hydro["standing_loss"] + sto_sq["max_hours"] = parameters_pumped_hydro["max_hours"] + + # Delete entrances without any installed capacity + sto_sq = sto_sq[sto_sq["p_nom"] > 0] + + # insert data pumped_hydro storage + + with session_scope() as session: + for i, row in sto_sq.iterrows(): + entry = etrago.EgonPfHvStorage( + scn_name=scn_name, + storage_id=int(db.next_etrago_id("storage")), + bus=row.bus, + max_hours=row.max_hours, + efficiency_store=row.store, + efficiency_dispatch=row.dispatch, + standing_loss=row.standing_loss, + carrier=row.carrier, + p_nom=row.p_nom, + ) + + session.add(entry) + session.commit() def insert_loads_sq(scn_name="status2019"): @@ -1650,16 +1867,39 @@ def insert_loads_sq(scn_name="status2019"): """ sources = config.datasets()["electrical_neighbours"]["sources"] targets = config.datasets()["electrical_neighbours"]["targets"] - try: - load_sq = entsoe_historic_demand() - except: - logging.warning( - """Demand data from entsoe could not be retrieved. - Backup data is used instead""" + + if scn_name == "status2019": + year_start_end = {"year_start": "20190101", "year_end": "20200101"} + elif scn_name == "status2023": + year_start_end = {"year_start": "20230101", "year_end": "20240101"} + else: + raise ValueError("No valid scenario name!") + + df_load_sq, not_retrieved = entsoe_historic_demand(**year_start_end) + + if not_retrieved: + logger.warning("Demand data from entsoe could not be retrieved.") + # check for generation backup from former runs + file_path = Path( + "./", "entsoe_data", f"load_entsoe_{scn_name}.csv" + ).resolve() + if os.path.isfile(file_path): + df_load_sq, not_retrieved = fill_by_backup_data_from_former_runs( + df_load_sq, file_path, not_retrieved + ) + save_entsoe_data(df_load_sq, file_path=file_path) + + if not_retrieved: + logger.warning( + f"Backup data of 2019 is used instead for {not_retrieved}" ) - load_sq = pd.read_csv( + df_load_sq_backup = pd.read_csv( "data_bundle_egon_data/entsoe/load_entsoe.csv", index_col="Index" ) + df_load_sq_backup.index = df_load_sq.index + df_load_sq = pd.concat( + [df_load_sq, df_load_sq_backup.loc[:, not_retrieved]], axis=1 + ) # Delete existing data db.execute_sql( @@ -1695,45 +1935,57 @@ def insert_loads_sq(scn_name="status2019"): """ ) - # Connect to database - engine = db.engine() - session = sessionmaker(bind=engine)() - # get the corresponding bus per foreign country - entsoe_to_bus = entsoe_to_bus_etrago() + entsoe_to_bus = entsoe_to_bus_etrago(scn_name) # Calculate and insert demand timeseries per etrago bus_id - for country in load_sq.columns: - load_id = db.next_etrago_id("load") - - entry = etrago.EgonPfHvLoad( - scn_name=scn_name, - load_id=int(load_id), - carrier="AC", - bus=int(entsoe_to_bus[country]), - ) + with session_scope() as session: + for country in df_load_sq.columns: + load_id = db.next_etrago_id("load") + + entry = etrago.EgonPfHvLoad( + scn_name=scn_name, + load_id=int(load_id), + carrier="AC", + bus=int(entsoe_to_bus[country]), + ) - entry_ts = etrago.EgonPfHvLoadTimeseries( - scn_name=scn_name, - load_id=int(load_id), - temp_id=1, - p_set=list(load_sq[country]), - ) + entry_ts = etrago.EgonPfHvLoadTimeseries( + scn_name=scn_name, + load_id=int(load_id), + temp_id=1, + p_set=list(df_load_sq[country]), + ) - session.add(entry) - session.add(entry_ts) - session.commit() + session.add(entry) + session.add(entry_ts) + session.commit() tasks = (grid,) insert_per_scenario = set() -if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: - insert_per_scenario.update([tyndp_generation, tyndp_demand]) +for scn_name in config.settings()["egon-data"]["--scenarios"]: + + if scn_name == "eGon2035": + insert_per_scenario.update([tyndp_generation, tyndp_demand]) -if "status2019" in config.settings()["egon-data"]["--scenarios"]: - insert_per_scenario.update([insert_generators_sq, insert_loads_sq]) + if "status" in scn_name: + postfix = f"_{scn_name.split('status')[-1]}" + insert_per_scenario.update( + [ + wrapped_partial( + insert_generators_sq, scn_name=scn_name, postfix=postfix + ), + wrapped_partial( + insert_loads_sq, scn_name=scn_name, postfix=postfix + ), + wrapped_partial( + insert_storage_units_sq, scn_name=scn_name, postfix=postfix + ), + ] + ) tasks = tasks + (insert_per_scenario,) @@ -1742,7 +1994,7 @@ class ElectricalNeighbours(Dataset): def __init__(self, dependencies): super().__init__( name="ElectricalNeighbours", - version="0.0.10", + version="0.0.11", dependencies=dependencies, tasks=tasks, ) diff --git a/src/egon/data/datasets/electricity_demand/__init__.py b/src/egon/data/datasets/electricity_demand/__init__.py index e0653663f..8ebdce98a 100644 --- a/src/egon/data/datasets/electricity_demand/__init__.py +++ b/src/egon/data/datasets/electricity_demand/__init__.py @@ -28,7 +28,7 @@ class HouseholdElectricityDemand(Dataset): def __init__(self, dependencies): super().__init__( name="HouseholdElectricityDemand", - version="0.0.4", + version="0.0.5", dependencies=dependencies, tasks=(create_tables, get_annual_household_el_demand_cells), ) @@ -93,6 +93,7 @@ def get_annual_household_el_demand_cells(): HouseholdElectricityProfilesOfBuildings, HouseholdElectricityProfilesInCensusCells.nuts3, HouseholdElectricityProfilesInCensusCells.factor_2019, + HouseholdElectricityProfilesInCensusCells.factor_2023, HouseholdElectricityProfilesInCensusCells.factor_2035, HouseholdElectricityProfilesInCensusCells.factor_2050, ) @@ -148,6 +149,12 @@ def ve(s): df_profiles.loc[:, df["profile_id"]].sum(axis=0) * df["factor_2019"].values ) + + if "status2023" in scenarios: + df_annual_demand_iter["status2023"] = ( + df_profiles.loc[:, df["profile_id"]].sum(axis=0) + * df["factor_2023"].values + ) df_annual_demand_iter["zensus_population_id"] = df["cell_id"].values df_annual_demand = pd.concat([df_annual_demand, df_annual_demand_iter]) @@ -227,11 +234,10 @@ def distribute_cts_demands(): peta["nuts3"] = map_nuts3.nuts3 # Calculate share of nuts3 heat demand per zensus cell - peta["share"] = ( - peta.heat_demand.groupby(peta.nuts3) - .apply(lambda grp: grp / grp.sum()) - .values - ) + for nuts3, df in peta.groupby("nuts3"): + peta.loc[df.index, "share"] = ( + df["heat_demand"] / df["heat_demand"].sum() + ) # Select forecasted electrical demands from demandregio table demand_nuts3 = db.select_dataframe( diff --git a/src/egon/data/datasets/electricity_demand/temporal.py b/src/egon/data/datasets/electricity_demand/temporal.py index f0943d58a..6e59e9607 100644 --- a/src/egon/data/datasets/electricity_demand/temporal.py +++ b/src/egon/data/datasets/electricity_demand/temporal.py @@ -10,6 +10,8 @@ from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base +import egon.data.datasets.scenario_parameters.parameters as scenario_parameters + Base = declarative_base() @@ -35,13 +37,15 @@ def create_table(): EgonEtragoElectricityCts.__table__.create(bind=engine, checkfirst=True) -def calc_load_curve(share_wz, annual_demand=1): +def calc_load_curve(share_wz, scn, annual_demand=1): """Create aggregated demand curve for service sector Parameters ---------- share_wz : pandas.Series or pandas.DataFrame Share of annual demand per cts branch + scn : str + Scenario name annual_demand : float or pandas.Series, optional Annual demand in MWh. The default is 1. @@ -51,7 +55,7 @@ def calc_load_curve(share_wz, annual_demand=1): Annual load curve of combindes cts branches """ - year = 2011 + year = int(scenario_parameters.global_settings(scn)["weather_year"]) sources = egon.data.config.datasets()["electrical_load_curves_cts"][ "sources" @@ -198,7 +202,7 @@ def calc_load_curves_cts(scenario): annual_demand_subst = demands_zensus.groupby("bus_id").demand.sum() # Return electrical load curves per hvmv substation - return calc_load_curve(share_subst, annual_demand_subst) + return calc_load_curve(share_subst, scenario, annual_demand_subst) def insert_cts_load(): diff --git a/src/egon/data/datasets/electricity_demand_etrago.py b/src/egon/data/datasets/electricity_demand_etrago.py index a52f521ad..5c21efde5 100644 --- a/src/egon/data/datasets/electricity_demand_etrago.py +++ b/src/egon/data/datasets/electricity_demand_etrago.py @@ -31,16 +31,12 @@ def demands_per_bus(scenario): # Select data on CTS electricity demands per bus cts_curves = db.select_dataframe( - f"""SELECT bus_id, p_set FROM + f"""SELECT bus_id AS bus, p_set FROM {sources['cts_curves']['schema']}. {sources['cts_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus_id", ) - # Rename index - cts_curves.index.rename("bus", inplace=True) - # Select data on industrial demands assigned to osm landuse areas ind_curves_osm = db.select_dataframe( @@ -48,7 +44,6 @@ def demands_per_bus(scenario): {sources['osm_curves']['schema']}. {sources['osm_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus", ) # Select data on industrial demands assigned to industrial sites @@ -58,17 +53,15 @@ def demands_per_bus(scenario): {sources['sites_curves']['schema']}. {sources['sites_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus", ) # Select data on household electricity demands per bus hh_curves = db.select_dataframe( - f"""SELECT bus_id, p_set FROM + f"""SELECT bus_id AS bus, p_set FROM {sources['household_curves']['schema']}. {sources['household_curves']['table']} WHERE scn_name = '{scenario}'""", - index_col="bus_id", ) # Create one df by appending all imported dataframes @@ -76,7 +69,7 @@ def demands_per_bus(scenario): demand_curves = pd.concat( [cts_curves, ind_curves_osm, ind_curves_sites, hh_curves], ignore_index=True, - ) + ).set_index("bus") # Split array to single columns in the dataframe demand_curves_split = demand_curves @@ -271,7 +264,7 @@ class ElectricalLoadEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="Electrical_load_etrago", - version="0.0.7", + version="0.0.8", dependencies=dependencies, tasks=(export_to_db,), ) diff --git a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py index 21af89b8b..e9db069fb 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/cts_buildings.py @@ -168,7 +168,7 @@ from geoalchemy2 import Geometry from geoalchemy2.shape import to_shape from psycopg2.extensions import AsIs, register_adapter -from sqlalchemy import REAL, Column, Integer, String, func +from sqlalchemy import REAL, Column, Integer, String, func, cast from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd import numpy as np @@ -260,7 +260,7 @@ def __init__(self, dependencies): version="0.0.4", dependencies=dependencies, tasks=( - cts_buildings, + cts_buildings, # TODO: status2023, currently fixed for only 2023 {cts_electricity, cts_heat}, get_cts_electricity_peak_load, map_all_used_buildings, @@ -397,17 +397,17 @@ def create_synthetic_buildings(df, points=None, crs="EPSG:3035"): # TODO remove after #772 implementation of egon_building_id df.rename(columns={"id": "egon_building_id"}, inplace=True) - # get max number of building ids from synthetic residential table + # get max number of building ids from synthetic table with db.session_scope() as session: - max_synth_residential_id = session.execute( - func.max(OsmBuildingsSynthetic.id) + max_synth_building_id = session.execute( + func.max(cast(OsmBuildingsSynthetic.id, Integer)) ).scalar() - max_synth_residential_id = int(max_synth_residential_id) + max_synth_building_id = int(max_synth_building_id) # create sequential ids df["egon_building_id"] = range( - max_synth_residential_id + 1, - max_synth_residential_id + df.shape[0] + 1, + max_synth_building_id + 1, + max_synth_building_id + df.shape[0] + 1, ) df["area"] = df["geom_building"].area @@ -442,6 +442,7 @@ def buildings_with_amenities(): Contains synthetic amenities in lost cells. Might be empty """ + from saio.boundaries import egon_map_zensus_buildings_filtered_all from saio.openstreetmap import osm_amenities_in_buildings_filtered @@ -461,7 +462,7 @@ def buildings_with_amenities(): ) .filter( EgonDemandRegioZensusElectricity.sector == "service", - EgonDemandRegioZensusElectricity.scenario == "status2019", + EgonDemandRegioZensusElectricity.scenario == "status2023", # TODO: status2023 ) ) df_amenities_in_buildings = pd.read_sql( @@ -1210,7 +1211,7 @@ def adapt_numpy_int64(numpy_int64): log.info("Start logging!") # Buildings with amenities - df_buildings_with_amenities, df_lost_cells = buildings_with_amenities() + df_buildings_with_amenities, df_lost_cells = buildings_with_amenities() # TODO: status2023 this is fixed to 2023 log.info("Buildings with amenities selected!") # Median number of amenities per cell @@ -1229,7 +1230,7 @@ def adapt_numpy_int64(numpy_int64): # Amenities not assigned to buildings df_amenities_without_buildings = amenities_without_buildings() - log.info("Amenities without buildlings selected!") + log.info("Amenities without buildings selected!") # Append lost cells due to duplicated ids, to cover all demand cells if not df_lost_cells.empty: @@ -1256,7 +1257,7 @@ def adapt_numpy_int64(numpy_int64): df_amenities_without_buildings, points="geom_amenity" ) log.info("Synthetic buildings created!") - + # df_synthetic_buildings_with_amenities["scn_name"] = scn_name # TODO: status 2023, add eventually # TODO remove renaming after #722 write_table_to_postgis( df_synthetic_buildings_with_amenities.rename( diff --git a/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py b/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py index e9ea3e770..19029936b 100755 --- a/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/hh_buildings.py @@ -155,6 +155,7 @@ class OsmBuildingsSynthetic(Base): id = Column(String, primary_key=True) cell_id = Column(String, index=True) + # scn_name = Column(String, index=True) # TODO: status2023 currently fixed to 2023 geom_building = Column(Geometry("Polygon", 3035), index=True) geom_point = Column(Geometry("POINT", 3035)) n_amenities_inside = Column(Integer) @@ -342,7 +343,7 @@ def generate_synthetic_buildings(missing_buildings, edge_length): destatis_zensus_population_per_ha_inside_germany ).filter( destatis_zensus_population_per_ha_inside_germany.c.id.in_( - missing_buildings.index + missing_buildings.index.unique() ) ) @@ -642,6 +643,7 @@ def get_building_peak_loads(): HouseholdElectricityProfilesOfBuildings, HouseholdElectricityProfilesInCensusCells.nuts3, HouseholdElectricityProfilesInCensusCells.factor_2019, + HouseholdElectricityProfilesInCensusCells.factor_2023, HouseholdElectricityProfilesInCensusCells.factor_2035, HouseholdElectricityProfilesInCensusCells.factor_2050, ) @@ -656,6 +658,9 @@ def get_building_peak_loads(): cells_query.statement, cells_query.session.bind, index_col="id" ) + # fill columns with None with np.nan to allow multiplication with emtpy columns + df_buildings_and_profiles = df_buildings_and_profiles.fillna(np.nan) + # Read demand profiles from egon-data-bundle df_profiles = get_iee_hh_demand_profiles_raw() @@ -692,11 +697,13 @@ def ve(s): df_building_peak_load_nuts3 = pd.DataFrame( [ df_building_peak_load_nuts3 * df["factor_2019"].unique(), + df_building_peak_load_nuts3 * df["factor_2023"].unique(), df_building_peak_load_nuts3 * df["factor_2035"].unique(), df_building_peak_load_nuts3 * df["factor_2050"].unique(), ], index=[ "status2019", + "status2023", "eGon2035", "eGon100RE", ], diff --git a/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py b/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py index 67aa44014..30508a46b 100644 --- a/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py +++ b/src/egon/data/datasets/electricity_demand_timeseries/hh_profiles.py @@ -126,7 +126,7 @@ import os import random -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.dialects.postgresql import CHAR, INTEGER, REAL from sqlalchemy.ext.declarative import declarative_base @@ -135,6 +135,7 @@ from egon.data import db from egon.data.datasets import Dataset +from egon.data.datasets.scenario_parameters import get_scenario_year from egon.data.datasets.zensus_mv_grid_districts import MapZensusGridDistricts import egon.data.config @@ -165,6 +166,7 @@ class HouseholdElectricityProfilesInCensusCells(Base): nuts3 = Column(String) nuts1 = Column(String) factor_2019 = Column(Float) + factor_2023 = Column(Float) factor_2035 = Column(Float) factor_2050 = Column(Float) @@ -211,6 +213,19 @@ def __init__(self, dependencies): tasks = tasks + (mv_hh_electricity_load_2035,) + if ( + "status2023" + in egon.data.config.settings()["egon-data"]["--scenarios"] + ): + mv_hh_electricity_load_2035 = PythonOperator( + task_id="MV-hh-electricity-load-2023", + python_callable=mv_grid_district_HH_electricity_load, + op_args=["status2023", 2023], + op_kwargs={"drop_table": True}, + ) + + tasks = tasks + (mv_hh_electricity_load_2035,) + if ( "eGon2035" in egon.data.config.settings()["egon-data"]["--scenarios"] @@ -237,7 +252,7 @@ def __init__(self, dependencies): super().__init__( name="Household Demands", - version="0.0.11", + version="0.0.12", dependencies=dependencies, tasks=tasks, ) @@ -1365,38 +1380,12 @@ def adjust_to_demand_regio_nuts3_annual( # demand regio in MWh # profiles in Wh - if ( - "status2019" - in egon.data.config.settings()["egon-data"]["--scenarios"] - ): - df_hh_profiles_in_census_cells.loc[ - nuts3_cell_ids, "factor_2019" - ] = ( - df_demand_regio.loc[(2019, nuts3_id), "demand_mwha"] - * 1e3 - / (nuts3_profiles_sum_annual / 1e3) - ) - - if ( - "eGon2035" - in egon.data.config.settings()["egon-data"]["--scenarios"] - ): - df_hh_profiles_in_census_cells.loc[ - nuts3_cell_ids, "factor_2035" - ] = ( - df_demand_regio.loc[(2035, nuts3_id), "demand_mwha"] - * 1e3 - / (nuts3_profiles_sum_annual / 1e3) - ) - - if ( - "eGon100RE" - in egon.data.config.settings()["egon-data"]["--scenarios"] - ): + for scn in egon.data.config.settings()["egon-data"]["--scenarios"]: + year = get_scenario_year(scn) df_hh_profiles_in_census_cells.loc[ - nuts3_cell_ids, "factor_2050" + nuts3_cell_ids, f"factor_{year}" ] = ( - df_demand_regio.loc[(2050, nuts3_id), "demand_mwha"] + df_demand_regio.loc[(year, nuts3_id), "demand_mwha"] * 1e3 / (nuts3_profiles_sum_annual / 1e3) ) @@ -1759,16 +1748,39 @@ def get_hh_profiles_from_db(profile_ids): return df_profile_loads +def get_demand_regio_hh_profiles_from_db(year): + """ + Retrieve demand regio household electricity demand profiles in nuts3 level + + Parameters + ---------- + year: int + To which year belong the required demand profile + + Returns + ------- + pd.DataFrame + Selection of household demand profiles + """ + + query = """Select * from demand.demandregio_household_load_profiles + Where year = year""" + + df_profile_loads = pd.read_sql( + query, db.engine(), index_col="id" + ) + + return df_profile_loads def mv_grid_district_HH_electricity_load( - scenario_name, scenario_year, drop_table=False + scenario_name, scenario_year, drop_table ): """ Aggregated household demand time series at HV/MV substation level Calculate the aggregated demand time series based on the demand profiles of each zensus cell inside each MV grid district. Profiles are read from - local hdf5-file. + local hdf5-file or demand timeseries per nuts3 in db. Parameters ---------- @@ -1807,33 +1819,82 @@ def tuple_format(x): cells_query.statement, cells_query.session.bind, index_col="cell_id" ) - # convert profile ids to tuple (type, id) format - cells["cell_profile_ids"] = cells["cell_profile_ids"].apply( - lambda x: list(map(tuple_format, x)) - ) + method = egon.data.config.settings()["egon-data"][ + "--household-demand-source" + ] - # Read demand profiles from egon-data-bundle - df_iee_profiles = get_iee_hh_demand_profiles_raw() + if method == "demand-regio": + #Import demand regio timeseries demand per nuts3 area + dr_series = pd.read_sql_query(""" + SELECT year, nuts3, load_in_mwh FROM demand.demandregio_household_load_profiles + """, + con = engine + ) + dr_series = dr_series[dr_series["year"] == scenario_year] + dr_series.drop(columns=["year"], inplace=True) + dr_series.set_index("nuts3", inplace=True) + dr_series = dr_series.squeeze() + + #Population data per cell_id is used to scale the demand per nuts3 + population = pd.read_sql_query(""" + SELECT grid_id, population FROM society.destatis_zensus_population_per_ha + """, + con = engine + ) + population.set_index("grid_id", inplace=True) + population = population.squeeze() + population.loc[population==-1] = 0 - # Process profiles for further use - df_iee_profiles = set_multiindex_to_profiles(df_iee_profiles) + cells["population"] = cells["grid_id"].map(population) + + factor_column = f"""factor_{scenario_year}""" - # Create aggregated load profile for each MV grid district - mvgd_profiles_dict = {} - for grid_district, data in cells.groupby("bus_id"): - mvgd_profile = get_load_timeseries( - df_iee_profiles=df_iee_profiles, - df_hh_profiles_in_census_cells=data, - cell_ids=data.index, - year=scenario_year, - peak_load_only=False, + mvgd_profiles = pd.DataFrame( + columns=["p_set", "q_set"], index=cells.bus_id.unique() ) - mvgd_profiles_dict[grid_district] = [mvgd_profile.round(3).to_list()] - mvgd_profiles = pd.DataFrame.from_dict(mvgd_profiles_dict, orient="index") + mvgd_profiles.index.name = "bus_id" - # Reshape data: put MV grid ids in columns to a single index column - mvgd_profiles = mvgd_profiles.reset_index() - mvgd_profiles.columns = ["bus_id", "p_set"] + for nuts3, df in cells.groupby("nuts3"): + cells.loc[df.index, factor_column] = df["population"] / df["population"].sum() + + for bus, df_bus in cells.groupby("bus_id"): + load_nuts = [0] * 8760 + for nuts3, df_nuts in df_bus.groupby("nuts3"): + factor_nuts = df_nuts[factor_column].sum() + total_load = [x * factor_nuts for x in dr_series[nuts3]] + load_nuts = [sum(x) for x in zip(load_nuts, total_load)] + mvgd_profiles.at[bus, "p_set"] = load_nuts + + mvgd_profiles.reset_index(inplace=True) + + elif method == "IEE": + # convert profile ids to tuple (type, id) format + cells["cell_profile_ids"] = cells["cell_profile_ids"].apply( + lambda x: list(map(tuple_format, x)) + ) + + # Read demand profiles from egon-data-bundle + df_iee_profiles = get_iee_hh_demand_profiles_raw() + + # Process profiles for further use + df_iee_profiles = set_multiindex_to_profiles(df_iee_profiles) + + # Create aggregated load profile for each MV grid district + mvgd_profiles_dict = {} + for grid_district, data in cells.groupby("bus_id"): + mvgd_profile = get_load_timeseries( + df_iee_profiles=df_iee_profiles, + df_hh_profiles_in_census_cells=data, + cell_ids=data.index, + year=scenario_year, + peak_load_only=False, + ) + mvgd_profiles_dict[grid_district] = [mvgd_profile.round(3).to_list()] + mvgd_profiles = pd.DataFrame.from_dict(mvgd_profiles_dict, orient="index") + + # Reshape data: put MV grid ids in columns to a single index column + mvgd_profiles = mvgd_profiles.reset_index() + mvgd_profiles.columns = ["bus_id", "p_set"] # Add remaining columns mvgd_profiles["scn_name"] = scenario_name @@ -1856,7 +1917,6 @@ def tuple_format(x): index=False, ) - def get_scaled_profiles_from_db( attribute, list_of_identifiers, year, aggregate=True, peak_load_only=False ): diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py b/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py index 7e264cd0b..132aa104e 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py @@ -77,7 +77,7 @@ import os import tarfile -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from psycopg2.extensions import AsIs, register_adapter import numpy as np import pandas as pd @@ -110,6 +110,7 @@ delete_model_data_from_db, generate_model_data_bunch, generate_model_data_status2019_remaining, + generate_model_data_status2023_remaining, generate_model_data_eGon100RE_remaining, generate_model_data_eGon2035_remaining, read_simbev_metadata_file, @@ -448,6 +449,8 @@ def generate_model_data_tasks(scenario_name): if scenario_name == "status2019": tasks.add(generate_model_data_status2019_remaining) + if scenario_name == "status2023": + tasks.add(generate_model_data_status2023_remaining) elif scenario_name == "eGon2035": tasks.add(generate_model_data_eGon2035_remaining) elif scenario_name == "eGon100RE": @@ -482,7 +485,7 @@ def generate_model_data_tasks(scenario_name): super().__init__( name="MotorizedIndividualTravel", - version="0.0.8", + version="0.0.10", dependencies=dependencies, tasks=tasks, ) diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py b/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py index f4a6c93fc..3a2bea57b 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/ev_allocation.py @@ -631,7 +631,7 @@ def get_random_evs(row): np.testing.assert_allclose( int(ev_actual), ev_target, - rtol=0.05, + rtol=0.001, err_msg=f"Dataset on EV numbers allocated to MVGDs " f"seems to be flawed. " f"Scenario: [{scn}], " diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py index af0f1ee2c..a81022d04 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/model_timeseries.py @@ -735,8 +735,9 @@ def write_load( # * regular (flex): use driving load # * lowflex: use dumb charging load # * status2019: also dumb charging + # * status2023: also dumb charging - if scenario_name=='status2019': + if scenario_name in ["status2019", "status2023"]: write_load( scenario_name=scenario_name, connection_bus_id=etrago_bus.bus_id, @@ -1077,6 +1078,15 @@ def generate_model_data_status2019_remaining(): bunch=range(MVGD_MIN_COUNT, len(load_grid_district_ids())), ) +def generate_model_data_status2023_remaining(): + """Generates timeseries for status2023 scenario for grid districts which + has not been processed in the parallel tasks before. + """ + generate_model_data_bunch( + scenario_name="status2023", + bunch=range(MVGD_MIN_COUNT, len(load_grid_district_ids())), + ) + def generate_model_data_eGon2035_remaining(): """Generates timeseries for eGon2035 scenario for grid districts which has not been processed in the parallel tasks before. diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py b/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py index a992cdcf9..649aeaeda 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/tests.py @@ -30,6 +30,6 @@ def validate_electric_vehicles_numbers(dataset_name, ev_data, ev_target): assert_allclose( ev_data[[_ for _ in CONFIG_EV.keys()]].sum().sum(), ev_target, - rtol=0.05, + rtol=0.001, err_msg=f"Dataset on EV numbers [{dataset_name}] seems to be flawed.", ) diff --git a/src/egon/data/datasets/era5.py b/src/egon/data/datasets/era5.py index 6b5624e21..0f6f11a52 100644 --- a/src/egon/data/datasets/era5.py +++ b/src/egon/data/datasets/era5.py @@ -23,7 +23,7 @@ class WeatherData(Dataset): def __init__(self, dependencies): super().__init__( name="Era5", - version="0.0.2", + version="0.0.3", dependencies=dependencies, tasks=({create_tables, download_era5}, insert_weather_cells), ) @@ -69,7 +69,8 @@ def import_cutout(boundary="Europe"): Weather data stored in cutout """ - weather_year = get_sector_parameters("global", "status2019")["weather_year"] + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # This is fixed to one scenario as its currently not possible to have multiple weather-years if boundary == "Europe": xs = slice(-12.0, 35.1) diff --git a/src/egon/data/datasets/fill_etrago_gen.py b/src/egon/data/datasets/fill_etrago_gen.py index 920454f5b..f461c518f 100644 --- a/src/egon/data/datasets/fill_etrago_gen.py +++ b/src/egon/data/datasets/fill_etrago_gen.py @@ -107,7 +107,7 @@ def add_marginal_costs(power_plants): marginal_costs.at[carrier, "marginal_cost"] = 0 if warning: print( - f"""There are not marginal_cost values for: \n{warning} + f"""There are no marginal_cost values for: \n{warning} in the scenario {scenario}. Missing values set to 0""" ) pp = pd.concat( @@ -235,10 +235,10 @@ def numpy_nan(data): def power_timeser(weather_data): - if len(set(weather_data)) <= 1: - return weather_data.iloc[0] - else: + if weather_data.isna().any(): return -1 + else: + return weather_data.iloc[0] def adjust_renew_feedin_table(renew_feedin, cfg): @@ -287,30 +287,56 @@ def delete_previuos_gen(cfg, con, etrago_gen_orig, power_plants): def set_timeseries(power_plants, renew_feedin): + """ + Create a function to calculate the feed-in timeseries for power plants. + + Parameters + ---------- + power_plants : DataFrame + A DataFrame containing information about power plants, including their bus IDs, + carriers, weather cell IDs, and electrical capacities. + renew_feedin : DataFrame + A DataFrame containing feed-in values for different carriers and weather cell IDs. + + Returns + ------- + function + A function that takes a power plant object and returns its feed-in value based on + either its direct weather cell ID or the aggregated feed-in of all power plants + connected to the same bus and having the same carrier. + """ + def timeseries(pp): + """Calculate the feed-in for a given power plant based on weather cell ID or aggregation.""" if pp.weather_cell_id != -1: - feedin_time = renew_feedin[ + # Directly fetch feed-in value for power plants with an associated weather cell ID + return renew_feedin.loc[ (renew_feedin["w_id"] == pp.weather_cell_id) - & (renew_feedin["carrier"] == pp.carrier) - ].feedin.iloc[0] - return feedin_time + & (renew_feedin["carrier"] == pp.carrier), + "feedin", + ].iat[0] else: - df = power_plants[ + # Aggregate feed-in for power plants without a direct weather cell association + df = power_plants.loc[ (power_plants["bus_id"] == pp.bus_id) & (power_plants["carrier"] == pp.carrier) - ] - total_int_cap = df.el_capacity.sum() - df["feedin"] = 0 + ].dropna(subset=["weather_cell_id"]) + + total_int_cap = df["el_capacity"].sum() + + # Fetch and calculate proportional feed-in for each power plant df["feedin"] = df.apply( - lambda x: renew_feedin[ + lambda x: renew_feedin.loc[ (renew_feedin["w_id"] == x.weather_cell_id) - & (renew_feedin["carrier"] == x.carrier) - ].feedin.iloc[0], + & (renew_feedin["carrier"] == x.carrier), + "feedin", + ].iat[0], axis=1, ) - df["feedin"] = df.apply( - lambda x: x.el_capacity / total_int_cap * x.feedin, axis=1 - ) - return df.feedin.sum() + + # Calculate and return aggregated feed-in based on electrical capacity + return df.apply( + lambda x: x["el_capacity"] / total_int_cap * x["feedin"], axis=1 + ).sum() return timeseries diff --git a/src/egon/data/datasets/fix_ehv_subnetworks.py b/src/egon/data/datasets/fix_ehv_subnetworks.py index 16872ce3f..7226d7fa6 100644 --- a/src/egon/data/datasets/fix_ehv_subnetworks.py +++ b/src/egon/data/datasets/fix_ehv_subnetworks.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd -from egon.data import config, db +from egon.data import config, db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.datasets.etrago_setup import link_geom_from_buses @@ -35,7 +35,9 @@ def select_bus_id(x, y, v_nom, scn_name, carrier, find_closest=False): ) if bus_id.empty: + logger.info("No bus found") if find_closest: + logger.info(f"Finding closest to x = {x}, y = {y}") bus_id = db.select_dataframe( f""" SELECT bus_id, st_distance(geom, 'SRID=4326;POINT({x} {y})'::geometry) @@ -47,10 +49,13 @@ def select_bus_id(x, y, v_nom, scn_name, carrier, find_closest=False): Limit 1 """ ) + logger.info(f"Bus ID = {bus_id.bus_id[0]} selected") return bus_id.bus_id[0] else: + logger.info("Find closest == False.") return None else: + logger.info(f"Exact match with bus ID = {bus_id.bus_id[0]} found.") return bus_id.bus_id[0] diff --git a/src/egon/data/datasets/gas_areas.py b/src/egon/data/datasets/gas_areas.py index e8d907ad7..ee9f3d557 100755 --- a/src/egon/data/datasets/gas_areas.py +++ b/src/egon/data/datasets/gas_areas.py @@ -6,8 +6,8 @@ from sqlalchemy import BigInteger, Column, Text from sqlalchemy.ext.declarative import declarative_base -from egon.data import db -from egon.data.datasets import Dataset +from egon.data import db, config +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.generate_voronoi import get_voronoi_geodataframe @@ -69,7 +69,7 @@ def __init__(self, dependencies): ) -class GasAreasstatus2019(Dataset): +class GasAreasStatusQuo(Dataset): """Create the gas voronoi table and the gas voronoi areas for status2019 *Dependencies* @@ -83,16 +83,24 @@ class GasAreasstatus2019(Dataset): """ #: - name: str = "GasAreasstatus2019" + name: str = "GasAreasStatusQuo" #: - version: str = "0.0.1" + version: str = "0.0.2" def __init__(self, dependencies): + tasks = (create_gas_voronoi_table,) + + for scn_name in config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += (wrapped_partial( + voronoi_status, scn_name=scn_name, postfix=f"_{scn_name[-4:]}" + ),) + super().__init__( name=self.name, version=self.version, dependencies=dependencies, - tasks=(create_gas_voronoi_table, voronoi_status2019), + tasks=tasks, ) @@ -142,12 +150,12 @@ def voronoi_egon100RE(): create_voronoi("eGon100RE", carrier) -def voronoi_status2019(): +def voronoi_status(scn_name): """ - Create voronoi polygons for all gas carriers in status2019 scenario + Create voronoi polygons for all gas carriers in status_x scenario """ for carrier in ["CH4"]: - create_voronoi("status2019", carrier) + create_voronoi(scn_name, carrier) def create_voronoi(scn_name, carrier): diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index 4c8b83d43..7ddbbe85f 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -31,7 +31,7 @@ from egon.data import config, db from egon.data.config import settings -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.electrical_neighbours import central_buses_egon100 from egon.data.datasets.etrago_helpers import copy_and_modify_buses from egon.data.datasets.scenario_parameters import get_sector_parameters @@ -936,7 +936,7 @@ def insert_gas_data_eGon100RE(): ) -def insert_gas_data_status2019(): +def insert_gas_data_status(scn_name): """ Function to deal with the gas network for the status2019 scenario. For this scenario just one CH4 bus is consider in the center of Germany. @@ -951,7 +951,6 @@ def insert_gas_data_status2019(): None. """ - scn_name = "status2019" # delete old entries db.execute_sql( @@ -1019,9 +1018,16 @@ class GasNodesAndPipes(Dataset): #: name: str = "GasNodesAndPipes" #: - version: str = "0.0.10" + version: str = "0.0.11" - tasks = (insert_gas_data_status2019,) + tasks = () + + for scn_name in config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += (wrapped_partial( + insert_gas_data_status, scn_name=scn_name, postfix=f"_{scn_name[-4:]}" + ),) + # tasks = (insert_gas_data_status,) if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: tasks = tasks + (insert_gas_data,) diff --git a/src/egon/data/datasets/gas_neighbours/__init__.py b/src/egon/data/datasets/gas_neighbours/__init__.py index 7732553b1..50195a517 100755 --- a/src/egon/data/datasets/gas_neighbours/__init__.py +++ b/src/egon/data/datasets/gas_neighbours/__init__.py @@ -17,13 +17,18 @@ class GasNeighbours(Dataset): def __init__(self, dependencies): super().__init__( name="GasNeighbours", - version="0.0.4", + version="0.0.5", dependencies=dependencies, tasks=( - tyndp_gas_generation, - tyndp_gas_demand, - grid, - insert_ocgt_abroad, - insert_gas_neigbours_eGon100RE, + # tyndp_gas_generation, + # tyndp_gas_demand, + # grid, + # insert_ocgt_abroad, + # insert_gas_neigbours_eGon100RE, + notasks, ), ) + + +def notasks(): + return None diff --git a/src/egon/data/datasets/heat_demand/__init__.py b/src/egon/data/datasets/heat_demand/__init__.py index a346bf82e..31954d6a9 100644 --- a/src/egon/data/datasets/heat_demand/__init__.py +++ b/src/egon/data/datasets/heat_demand/__init__.py @@ -47,7 +47,7 @@ def __init__(self, dependencies): super().__init__( name="heat-demands", # version=self.target_files + "_0.0", - version="0.0.3", # maybe rethink the naming + version="0.0.4", # maybe rethink the naming dependencies=dependencies, tasks=(scenario_data_import), ) @@ -357,6 +357,17 @@ def future_heat_demand_germany(scenario_name): ser_hd_reduction = ( heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 ) + elif scenario_name == "status2023": + heat_parameters = get_sector_parameters("heat", scenario=scenario_name) # currently data for 2019 is used + # see scenario_paramters/__init__ for this. + + # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 + res_hd_reduction = ( + heat_parameters["DE_demand_residential_TJ"] / 3600 / 443.788483 # TODO status2023 can values stay same? + ) + ser_hd_reduction = ( + heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 # TODO status2023 can values stay same? + ) else: heat_parameters = get_sector_parameters("heat", scenario=scenario_name) diff --git a/src/egon/data/datasets/heat_demand_timeseries/__init__.py b/src/egon/data/datasets/heat_demand_timeseries/__init__.py index ed9f1811e..3345a011b 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/__init__.py +++ b/src/egon/data/datasets/heat_demand_timeseries/__init__.py @@ -299,117 +299,113 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): aggregation_level="district" ) - # TODO: use session_scope! - from sqlalchemy.orm import sessionmaker - - session = sessionmaker(bind=db.engine())() - print(datetime.now() - start_time) start_time = datetime.now() for area in district_heating_grids.area_id.unique(): - selected_profiles = db.select_dataframe( - f""" - SELECT a.zensus_population_id, building_id, c.climate_zone, - selected_idp, ordinality as day, b.area_id - FROM demand.egon_heat_timeseries_selected_profiles a - INNER JOIN boundaries.egon_map_zensus_climate_zones c - ON a.zensus_population_id = c.zensus_population_id - INNER JOIN ( - SELECT * FROM demand.egon_map_zensus_district_heating_areas - WHERE scenario = '{scenario}' - AND area_id = '{area}' - ) b ON a.zensus_population_id = b.zensus_population_id , - - UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp - - """ - ) - if not selected_profiles.empty: - df = pd.merge( - selected_profiles, - daily_demand_shares, - on=["day", "climate_zone"], - ) - - slice_df = pd.merge( - df[df.area_id == area], - idp_df, - left_on="selected_idp", - right_on="index", - ) + with db.session_scope() as session: + + sql = f""" + SELECT a.zensus_population_id, building_id, c.climate_zone, + selected_idp, ordinality as day, b.area_id + FROM demand.egon_heat_timeseries_selected_profiles a + INNER JOIN boundaries.egon_map_zensus_climate_zones c + ON a.zensus_population_id = c.zensus_population_id + INNER JOIN ( + SELECT * FROM demand.egon_map_zensus_district_heating_areas + WHERE scenario = '{scenario}' + AND area_id = '{area}' + ) b ON a.zensus_population_id = b.zensus_population_id, + UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp + """ + + selected_profiles = db.select_dataframe(sql) + + if not selected_profiles.empty: + df = pd.merge( + selected_profiles, + daily_demand_shares, + on=["day", "climate_zone"], + ) - for hour in range(24): - slice_df[hour] = ( - slice_df.idp.str[hour] - .mul(slice_df.daily_demand_share) - .mul( - annual_demand.loc[ - slice_df.zensus_population_id.values, - "per_building", - ].values - ) + slice_df = pd.merge( + df[df.area_id == area], + idp_df, + left_on="selected_idp", + right_on="index", ) - diff = ( - slice_df[range(24)].sum().sum() - - annual_demand[ - annual_demand.area_id == area - ].demand_total.sum() - ) / ( - annual_demand[annual_demand.area_id == area].demand_total.sum() - ) + for hour in range(24): + slice_df[hour] = ( + slice_df.idp.str[hour] + .mul(slice_df.daily_demand_share) + .mul( + annual_demand.loc[ + slice_df.zensus_population_id.values, + "per_building", + ].values + ) + ) - assert ( - abs(diff) < 0.03 - ), f"""Deviation of residential heat demand time - series for district heating grid {str(area)} is {diff}""" - - hh = np.concatenate( - slice_df.drop( - [ - "zensus_population_id", - "building_id", - "climate_zone", - "selected_idp", - "area_id", - "daily_demand_share", - "idp", - ], - axis="columns", + diff = ( + slice_df[range(24)].sum().sum() + - annual_demand[ + annual_demand.area_id == area + ].demand_total.sum() + ) / ( + annual_demand[annual_demand.area_id == area].demand_total.sum() ) - .groupby("day") - .sum()[range(24)] - .values - ).ravel() - - cts = CTS_demand_dist[ - (CTS_demand_dist.scenario == scenario) - & (CTS_demand_dist.index == area) - ].drop("scenario", axis="columns") - if (not selected_profiles.empty) and not cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(hh + cts.values[0]).tolist(), - ) - elif (not selected_profiles.empty) and cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(hh).tolist(), - ) - elif not cts.empty: - entry = EgonTimeseriesDistrictHeating( - area_id=int(area), - scenario=scenario, - dist_aggregated_mw=(cts.values[0]).tolist(), - ) + assert ( + abs(diff) < 0.04 + ), f"""Deviation of residential heat demand time + series for district heating grid {str(area)} is {diff}""" + + hh = np.concatenate( + slice_df.drop( + [ + "zensus_population_id", + "building_id", + "climate_zone", + "selected_idp", + "area_id", + "daily_demand_share", + "idp", + ], + axis="columns", + ) + .groupby("day") + .sum()[range(24)] + .values + ).ravel() + + cts = CTS_demand_dist[ + (CTS_demand_dist.scenario == scenario) + & (CTS_demand_dist.index == area) + ].drop("scenario", axis="columns") + + if (not selected_profiles.empty) and not cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(hh + cts.values[0]).tolist(), + ) + elif (not selected_profiles.empty) and cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(hh).tolist(), + ) + elif not cts.empty: + entry = EgonTimeseriesDistrictHeating( + area_id=int(area), + scenario=scenario, + dist_aggregated_mw=(cts.values[0]).tolist(), + ) - session.add(entry) - session.commit() + session.add(entry) + session.commit() print( f"Time to create time series for district heating scenario {scenario}" @@ -748,7 +744,7 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): assert ( abs(diff) < 0.03 - ), f"""Deviation of residential heat demand time + ), f"""Deviation of residential heat demand time series for mv grid {str(grid)} is {diff}""" if not (slice_df[hour].empty or cts.empty): diff --git a/src/egon/data/datasets/heat_demand_timeseries/daily.py b/src/egon/data/datasets/heat_demand_timeseries/daily.py index 95da233e0..d98211cd0 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/daily.py +++ b/src/egon/data/datasets/heat_demand_timeseries/daily.py @@ -9,6 +9,7 @@ from egon.data import db import egon.data.datasets.era5 as era +from egon.data.datasets.scenario_parameters import get_sector_parameters from math import ceil @@ -155,7 +156,7 @@ def map_climate_zones_to_zensus(): def daily_demand_shares_per_climate_zone(): - """Calculates shares of heat demand per day for each cliamte zone + """Calculates shares of heat demand per day for each climate zone Returns ------- @@ -171,14 +172,17 @@ def daily_demand_shares_per_climate_zone(): bind=engine, checkfirst=True ) + # Get temperature profiles of all TRY Climate Zones 2011 + temp_profile = temperature_profile_extract() + # Calulate daily demand shares - h = h_value() + h = h_value(temp_profile) # Normalize data to sum()=1 daily_demand_shares = h.resample("d").sum() / h.sum() # Extract temperature class for each day and climate zone - temperature_classes = temp_interval().resample("D").max() + temperature_classes = temp_interval(temp_profile).resample("D").max() # Initilize dataframe df = pd.DataFrame( @@ -317,18 +321,22 @@ def temperature_profile_extract(): return temperature_profile -def temp_interval(): +def temp_interval(temp_profile): """ Description: Create Dataframe with temperature data for TRY Climate Zones + + temp_profile: pandas.DataFrame + temperature profiles of all TRY Climate Zones 2011 Returns ------- temperature_interval : pandas.DataFrame Hourly temperature intrerval of all 15 TRY Climate station#s temperature profile """ - index = pd.date_range(datetime(2019, 1, 1, 0), periods=8760, freq="H") + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # TODO" status2023 this is currenlty fixed to one scenario possible as only one weather year is possible + index = pd.date_range(datetime(weather_year, 1, 1, 0), periods=8760, freq="H") temperature_interval = pd.DataFrame() - temp_profile = temperature_profile_extract() for x in range(len(temp_profile.columns)): name_station = temp_profile.columns[x] @@ -342,10 +350,12 @@ def temp_interval(): return temperature_interval -def h_value(): +def h_value(temp_profile): """ Description: Assignment of daily demand scaling factor to each day of all TRY Climate Zones + temp_profile: pandas.DataFrame + temperature profiles of all TRY Climate Zones 2011 Returns ------- h : pandas.DataFrame @@ -353,7 +363,10 @@ def h_value(): Extracted from demandlib. """ - index = pd.date_range(datetime(2019, 1, 1, 0), periods=8760, freq="H") + + weather_year = get_sector_parameters("global", "status2023")["weather_year"] + # TODO status2023: this is fixed to 2023 as only one weather year is currently possible + index = pd.date_range(datetime(weather_year, 1, 1, 0), periods=8760, freq="H") a = 3.0469695 @@ -363,7 +376,6 @@ def h_value(): d = 0.1163157 - temp_profile = temperature_profile_extract() temperature_profile_res = ( temp_profile.resample("D") .mean() diff --git a/src/egon/data/datasets/heat_demand_timeseries/service_sector.py b/src/egon/data/datasets/heat_demand_timeseries/service_sector.py index f16b37e6f..9cc04b2e9 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/service_sector.py +++ b/src/egon/data/datasets/heat_demand_timeseries/service_sector.py @@ -74,7 +74,7 @@ def cts_demand_per_aggregation_level(aggregation_level, scenario): df_CTS_gas_2011 = df_CTS_gas_2011.asfreq("H") else: df_CTS_gas_2011 = temporal.disagg_temporal_gas_CTS( - use_nuts3code=True, year=2019 + use_nuts3code=True, year=2017 ) df_CTS_gas_2011.to_csv("CTS_heat_demand_profile_nuts3.csv") diff --git a/src/egon/data/datasets/heat_etrago/__init__.py b/src/egon/data/datasets/heat_etrago/__init__.py index 9b062b5eb..69c21e1b8 100644 --- a/src/egon/data/datasets/heat_etrago/__init__.py +++ b/src/egon/data/datasets/heat_etrago/__init__.py @@ -69,14 +69,14 @@ def insert_buses(carrier, scenario): SELECT ST_Centroid(geom) AS geom FROM {sources['mv_grids']['schema']}. {sources['mv_grids']['table']} - WHERE bus_id IN - (SELECT DISTINCT bus_id + WHERE bus_id IN + (SELECT DISTINCT bus_id FROM boundaries.egon_map_zensus_grid_districts a - JOIN demand.egon_peta_heat b + JOIN demand.egon_peta_heat b ON a.zensus_population_id = b.zensus_population_id WHERE b.scenario = '{scenario}' AND b.zensus_population_id NOT IN ( - SELECT zensus_population_id FROM + SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario}' ) @@ -263,7 +263,7 @@ def insert_store(scenario, carrier): def store(): for scenario in config.settings()["egon-data"]["--scenarios"]: - if scenario != "status2019": + if "status" not in scenario: insert_store(scenario, "central_heat") insert_store(scenario, "rural_heat") @@ -290,8 +290,8 @@ def insert_central_direct_heat(scenario): {targets['heat_generators']['table']} WHERE carrier IN ('solar_thermal_collector', 'geo_thermal') AND scn_name = '{scenario}' - AND bus IN - (SELECT bus_id + AND bus IN + (SELECT bus_id FROM {targets['heat_buses']['schema']}. {targets['heat_buses']['table']} WHERE scn_name = '{scenario}' @@ -368,13 +368,15 @@ def insert_central_direct_heat(scenario): # Map solar thermal collectors to weather cells join = gpd.sjoin(weather_cells, solar_thermal)[["index_right"]] + weather_year = get_sector_parameters("global", scenario)["weather_year"] + feedin = db.select_dataframe( f""" SELECT w_id, feedin FROM {sources['feedin_timeseries']['schema']}. {sources['feedin_timeseries']['table']} WHERE carrier = 'solar_thermal' - AND weather_year = 2019 + AND weather_year = {weather_year} """, index_col="w_id", ) @@ -511,14 +513,14 @@ def insert_rural_gas_boilers(scenario): {targets['heat_links']['table']} WHERE carrier = 'rural_gas_boiler' AND scn_name = '{scenario}' - AND bus0 IN - (SELECT bus_id + AND bus0 IN + (SELECT bus_id FROM {targets['heat_buses']['schema']}. {targets['heat_buses']['table']} WHERE scn_name = '{scenario}' AND country = 'DE') - AND bus1 IN - (SELECT bus_id + AND bus1 IN + (SELECT bus_id FROM {targets['heat_buses']['schema']}. {targets['heat_buses']['table']} WHERE scn_name = '{scenario}' @@ -607,8 +609,8 @@ def supply(): """ for scenario in config.settings()["egon-data"]["--scenarios"]: - # There is no direct heat in status2019 scenario - if scenario != "status2019": + # There is no direct heat in status quo scenario + if "status" not in scenario: insert_central_direct_heat(scenario) insert_central_power_to_heat(scenario) insert_individual_power_to_heat(scenario) @@ -619,7 +621,7 @@ class HeatEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="HeatEtrago", - version="0.0.11", + version="0.0.12", dependencies=dependencies, tasks=(buses, supply, store), - ) \ No newline at end of file + ) diff --git a/src/egon/data/datasets/heat_etrago/hts_etrago.py b/src/egon/data/datasets/heat_etrago/hts_etrago.py index d19357cc0..aaee6187e 100644 --- a/src/egon/data/datasets/heat_etrago/hts_etrago.py +++ b/src/egon/data/datasets/heat_etrago/hts_etrago.py @@ -12,7 +12,7 @@ def hts_to_etrago(scenario): targets = config.datasets()["etrago_heat"]["targets"] carriers = ["central_heat", "rural_heat", "rural_gas_boiler"] - if scenario == "status2019": + if "status" in scenario: carriers = ["central_heat", "rural_heat"] for carrier in carriers: diff --git a/src/egon/data/datasets/heat_supply/__init__.py b/src/egon/data/datasets/heat_supply/__init__.py index b2c97a36c..3781cfddd 100644 --- a/src/egon/data/datasets/heat_supply/__init__.py +++ b/src/egon/data/datasets/heat_supply/__init__.py @@ -94,8 +94,9 @@ def district_heating(): if_exists="append", ) - # Do not check data for status2019 as is it not listed in the table - if scenario != "status2019": + + # Do not check data for status quo as is it not listed in the table + if "status" not in scenario: # Compare target value with sum of distributed heat supply df_check = db.select_dataframe( f""" @@ -128,8 +129,9 @@ def district_heating(): if_exists="append", ) - # Insert resistive heaters which are not available in status2019 - if scenario != "status2019": + + # Insert resistive heaters which are not available in status quo + if "status" not in scenario: backup_rh = backup_resistive_heaters(scenario) if not backup_rh.empty: @@ -182,7 +184,7 @@ class HeatSupply(Dataset): def __init__(self, dependencies): super().__init__( name="HeatSupply", - version="0.0.9", + version="0.0.10", dependencies=dependencies, tasks=( create_tables, diff --git a/src/egon/data/datasets/heat_supply/district_heating.py b/src/egon/data/datasets/heat_supply/district_heating.py index 3b62c87f6..15d0f3dfe 100644 --- a/src/egon/data/datasets/heat_supply/district_heating.py +++ b/src/egon/data/datasets/heat_supply/district_heating.py @@ -307,7 +307,7 @@ def cascade_heat_supply(scenario, plotting=True): district_heating_areas = select_district_heating_areas(scenario) # Select technolgies per district heating size - if scenario != "status2019": + if "status" not in scenario: map_dh_technologies = { "small": [ "CHP", diff --git a/src/egon/data/datasets/heat_supply/individual_heating.py b/src/egon/data/datasets/heat_supply/individual_heating.py index f0d3d4092..ea1148f57 100644 --- a/src/egon/data/datasets/heat_supply/individual_heating.py +++ b/src/egon/data/datasets/heat_supply/individual_heating.py @@ -184,7 +184,7 @@ import os import random -from airflow.operators.python_operator import PythonOperator +from airflow.operators.python import PythonOperator from psycopg2.extensions import AsIs, register_adapter from sqlalchemy import ARRAY, REAL, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -194,7 +194,7 @@ import saio from egon.data import config, db, logger -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.district_heating_areas import ( MapZensusDistrictHeatingAreas, ) @@ -264,6 +264,7 @@ def dyn_parallel_tasks_pypsa_eur_sec(): ) tasks = set() + for i in range(parallel_tasks): tasks.add( PythonOperator( @@ -295,9 +296,9 @@ def dyn_parallel_tasks_pypsa_eur_sec(): ) -class HeatPumps2019(Dataset): +class HeatPumpsStatusQuo(Dataset): def __init__(self, dependencies): - def dyn_parallel_tasks_2019(): + def dyn_parallel_tasks_status_quo(scenario): """Dynamically generate tasks The goal is to speed up tasks by parallelising bulks of mvgds. @@ -310,40 +311,67 @@ def dyn_parallel_tasks_2019(): set of airflow.PythonOperators The tasks. Each element is of :func:`egon.data.datasets.heat_supply.individual_heating. - determine_hp_cap_peak_load_mvgd_ts_2019` + determine_hp_cap_peak_load_mvgd_ts_status_quo` """ parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( "parallel_tasks", 1 ) + tasks = set() + for i in range(parallel_tasks): tasks.add( PythonOperator( task_id=( "individual_heating." - f"determine-hp-capacity-2019-" + f"determine-hp-capacity-{scenario}-" f"mvgd-bulk{i}" ), python_callable=split_mvgds_into_bulks, op_kwargs={ "n": i, "max_n": parallel_tasks, - "func": determine_hp_cap_peak_load_mvgd_ts_2019, + "scenario": scenario, + "func": determine_hp_cap_peak_load_mvgd_ts_status_quo, }, ) ) return tasks + tasks = () + + for scenario in config.settings()["egon-data"]["--scenarios"]: + if "status" in scenario: + postfix = f"_{scenario[-4:]}" + + tasks += ( + wrapped_partial( + delete_heat_peak_loads_status_quo, + scenario=scenario, + postfix=postfix, + ), + wrapped_partial( + delete_hp_capacity_status_quo, + scenario=scenario, + postfix=postfix, + ), + wrapped_partial( + delete_mvgd_ts_status_quo, + scenario=scenario, + postfix=postfix, + ), + ) + + tasks += ( + {*dyn_parallel_tasks_status_quo(scenario)}, + ) + + super().__init__( - name="HeatPumps2019", - version="0.0.2", + name="HeatPumpsStatusQuo", + version="0.0.4", dependencies=dependencies, - tasks=( - delete_heat_peak_loads_2019, - delete_hp_capacity_2019, - delete_mvgd_ts_2019, - {*dyn_parallel_tasks_2019()}, - ), + tasks=tasks, ) @@ -367,7 +395,9 @@ def dyn_parallel_tasks_2035(): parallel_tasks = config.datasets()["demand_timeseries_mvgd"].get( "parallel_tasks", 1 ) + tasks = set() + for i in range(parallel_tasks): tasks.add( PythonOperator( @@ -612,12 +642,14 @@ def cascade_heat_supply_indiv(scenario, distribution_level, plotting=True): columns=["estimated_flh", "priority"], data={"estimated_flh": [4000, 8000], "priority": [2, 1]}, ) - elif scenario == "status2019": + elif "status" in scenario: technologies = pd.DataFrame( index=["heat_pump"], columns=["estimated_flh", "priority"], data={"estimated_flh": [4000], "priority": [2]}, ) + else: + raise ValueError(f"{scenario=} is not valid.") # In the beginning, the remaining demand equals demand heat_per_mv["remaining_demand"] = heat_per_mv["demand"] @@ -1882,9 +1914,9 @@ def determine_hp_cap_peak_load_mvgd_ts_2035(mvgd_ids): ) -def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): +def determine_hp_cap_peak_load_mvgd_ts_status_quo(mvgd_ids, scenario): """ - Main function to determine HP capacity per building in status2019 scenario. + Main function to determine HP capacity per building in status quo scenario. Further, creates heat demand time series for all buildings with heat pumps in MV grid, as well as for all buildings with gas boilers, used in eTraGo. @@ -1901,7 +1933,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): # ===================================================== df_peak_loads_db = pd.DataFrame() - df_hp_cap_per_building_2019_db = pd.DataFrame() + df_hp_cap_per_building_status_quo_db = pd.DataFrame() df_heat_mvgd_ts_db = pd.DataFrame() for mvgd in mvgd_ids: @@ -1910,20 +1942,20 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): # ############# aggregate residential and CTS demand profiles ##### df_heat_ts = aggregate_residential_and_cts_profiles( - mvgd, scenario="status2019" + mvgd, scenario=scenario ) # ##################### determine peak loads ################### logger.info(f"MVGD={mvgd} | Determine peak loads.") - peak_load_2019 = df_heat_ts.max().rename("status2019") + peak_load_status_quo = df_heat_ts.max().rename(scenario) # ######## determine HP capacity per building ######### logger.info(f"MVGD={mvgd} | Determine HP capacities.") buildings_decentral_heating = ( get_buildings_with_decentral_heat_demand_in_mv_grid( - mvgd, scenario="status2019" + mvgd, scenario=scenario ) ) @@ -1931,33 +1963,30 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): # TODO maybe remove after succesfull DE run # Might be fixed in #990 buildings_decentral_heating = catch_missing_buidings( - buildings_decentral_heating, peak_load_2019 + buildings_decentral_heating, peak_load_status_quo ) - hp_cap_per_building_2019 = determine_hp_cap_buildings_pvbased_per_mvgd( - "status2019", + hp_cap_per_building_status_quo = determine_hp_cap_buildings_pvbased_per_mvgd( + scenario, mvgd, - peak_load_2019, + peak_load_status_quo, buildings_decentral_heating, ) - buildings_gas_2019 = pd.Index(buildings_decentral_heating).drop( - hp_cap_per_building_2019.index - ) # ################ aggregated heat profiles ################### logger.info(f"MVGD={mvgd} | Aggregate heat profiles.") - df_mvgd_ts_2019_hp = df_heat_ts.loc[ + df_mvgd_ts_status_quo_hp = df_heat_ts.loc[ :, - hp_cap_per_building_2019.index, + hp_cap_per_building_status_quo.index, ].sum(axis=1) df_heat_mvgd_ts = pd.DataFrame( data={ "carrier": "heat_pump", "bus_id": mvgd, - "scenario": "status2019", - "dist_aggregated_mw": [df_mvgd_ts_2019_hp.to_list()], + "scenario": scenario, + "dist_aggregated_mw": [df_mvgd_ts_status_quo_hp.to_list()], } ) @@ -1965,7 +1994,7 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): logger.info(f"MVGD={mvgd} | Collect results.") df_peak_loads_db = pd.concat( - [df_peak_loads_db, peak_load_2019.reset_index()], + [df_peak_loads_db, peak_load_status_quo.reset_index()], axis=0, ignore_index=True, ) @@ -1974,10 +2003,10 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): [df_heat_mvgd_ts_db, df_heat_mvgd_ts], axis=0, ignore_index=True ) - df_hp_cap_per_building_2019_db = pd.concat( + df_hp_cap_per_building_status_quo_db = pd.concat( [ - df_hp_cap_per_building_2019_db, - hp_cap_per_building_2019.reset_index(), + df_hp_cap_per_building_status_quo_db, + hp_cap_per_building_status_quo.reset_index(), ], axis=0, ) @@ -1987,11 +2016,11 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): export_to_db(df_peak_loads_db, df_heat_mvgd_ts_db, drop=False) - df_hp_cap_per_building_2019_db["scenario"] = "status2019" + df_hp_cap_per_building_status_quo_db["scenario"] = scenario # TODO debug duplicated building_ids - duplicates = df_hp_cap_per_building_2019_db.loc[ - df_hp_cap_per_building_2019_db.duplicated("building_id", keep=False) + duplicates = df_hp_cap_per_building_status_quo_db.loc[ + df_hp_cap_per_building_status_quo_db.duplicated("building_id", keep=False) ] if not duplicates.empty: @@ -2000,14 +2029,14 @@ def determine_hp_cap_peak_load_mvgd_ts_2019(mvgd_ids): f"{duplicates.loc[:,['building_id', 'hp_capacity']]}" ) - df_hp_cap_per_building_2019_db.drop_duplicates("building_id", inplace=True) + df_hp_cap_per_building_status_quo_db.drop_duplicates("building_id", inplace=True) - df_hp_cap_per_building_2019_db.building_id = ( - df_hp_cap_per_building_2019_db.building_id.astype(int) + df_hp_cap_per_building_status_quo_db.building_id = ( + df_hp_cap_per_building_status_quo_db.building_id.astype(int) ) write_table_to_postgres( - df_hp_cap_per_building_2019_db, + df_hp_cap_per_building_status_quo_db, EgonHpCapacityBuildings, drop=False, ) @@ -2118,7 +2147,7 @@ def determine_hp_cap_peak_load_mvgd_ts_pypsa_eur_sec(mvgd_ids): export_min_cap_to_csv(df_hp_min_cap_mv_grid_pypsa_eur_sec) -def split_mvgds_into_bulks(n, max_n, func): +def split_mvgds_into_bulks(n, max_n, func, scenario=None): """ Generic function to split task into multiple parallel tasks, dividing the number of MVGDs into even bulks. @@ -2156,7 +2185,11 @@ def split_mvgds_into_bulks(n, max_n, func): mvgd_ids = mvgd_ids[n] logger.info(f"Bulk takes care of MVGD: {min(mvgd_ids)} : {max(mvgd_ids)}") - func(mvgd_ids) + + if scenario is not None: + func(mvgd_ids, scenario=scenario) + else: + func(mvgd_ids) def delete_hp_capacity(scenario): @@ -2199,10 +2232,10 @@ def delete_hp_capacity_100RE(): delete_hp_capacity(scenario="eGon100RE") -def delete_hp_capacity_2019(): - """Remove all hp capacities for the selected status2019""" +def delete_hp_capacity_status_quo(scenario): + """Remove all hp capacities for the selected status quo""" EgonHpCapacityBuildings.__table__.create(bind=engine, checkfirst=True) - delete_hp_capacity(scenario="status2019") + delete_hp_capacity(scenario=scenario) def delete_hp_capacity_2035(): @@ -2211,12 +2244,12 @@ def delete_hp_capacity_2035(): delete_hp_capacity(scenario="eGon2035") -def delete_mvgd_ts_2019(): - """Remove all mvgd ts for the selected status2019""" +def delete_mvgd_ts_status_quo(scenario): + """Remove all mvgd ts for the selected status quo""" EgonEtragoTimeseriesIndividualHeating.__table__.create( bind=engine, checkfirst=True ) - delete_mvgd_ts(scenario="status2019") + delete_mvgd_ts(scenario=scenario) def delete_mvgd_ts_2035(): @@ -2235,13 +2268,13 @@ def delete_mvgd_ts_100RE(): delete_mvgd_ts(scenario="eGon100RE") -def delete_heat_peak_loads_2019(): - """Remove all heat peak loads for status2019.""" +def delete_heat_peak_loads_status_quo(scenario): + """Remove all heat peak loads for status quo.""" BuildingHeatPeakLoads.__table__.create(bind=engine, checkfirst=True) with db.session_scope() as session: # Buses session.query(BuildingHeatPeakLoads).filter( - BuildingHeatPeakLoads.scenario == "status2019" + BuildingHeatPeakLoads.scenario == scenario ).delete(synchronize_session=False) diff --git a/src/egon/data/datasets/industry/temporal.py b/src/egon/data/datasets/industry/temporal.py index b1fb05900..523d2a1d3 100644 --- a/src/egon/data/datasets/industry/temporal.py +++ b/src/egon/data/datasets/industry/temporal.py @@ -212,7 +212,7 @@ def calc_load_curves_ind_osm(scenario): annual_demand_osm = demands_osm_area.groupby("osm_id").demand.sum() # Return electrical load curves per osm industrial landuse area - load_curves = calc_load_curve(share_wz_transpose, annual_demand_osm) + load_curves = calc_load_curve(share_wz_transpose, scenario, annual_demand_osm) curves_da = identify_bus(load_curves, demand_area) @@ -373,7 +373,7 @@ def calc_load_curves_ind_sites(scenario): .demand ) - load_curves = calc_load_curve(share_transpose, demands_ind_sites["demand"]) + load_curves = calc_load_curve(share_transpose, scenario, demands_ind_sites["demand"]) curves_da = identify_bus(load_curves, demand_area) diff --git a/src/egon/data/datasets/mastr.py b/src/egon/data/datasets/mastr.py index 3c1d6b5d8..27f244705 100644 --- a/src/egon/data/datasets/mastr.py +++ b/src/egon/data/datasets/mastr.py @@ -28,7 +28,7 @@ import egon.data.config WORKING_DIR_MASTR_OLD = Path(".", "bnetza_mastr", "dump_2021-05-03") -WORKING_DIR_MASTR_NEW = Path(".", "bnetza_mastr", "dump_2022-11-17") +WORKING_DIR_MASTR_NEW = Path(".", "bnetza_mastr", "dump_2024-01-08") def download_mastr_data(): @@ -69,7 +69,7 @@ def download(dataset_name, download_dir): mastr_data_setup = partial( Dataset, name="MastrData", - version="0.0.2", + version="0.0.3", dependencies=[], tasks=(download_mastr_data,), ) diff --git a/src/egon/data/datasets/osm/__init__.py b/src/egon/data/datasets/osm/__init__.py index b0cfaa11b..b86ef189a 100644 --- a/src/egon/data/datasets/osm/__init__.py +++ b/src/egon/data/datasets/osm/__init__.py @@ -20,7 +20,7 @@ import importlib_resources as resources -from egon.data import db +from egon.data import db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.metadata import ( @@ -77,8 +77,10 @@ def to_postgres(cache_size=4096): if settings()["egon-data"]["--dataset-boundary"] == "Everything": input_filename = osm_config["target"]["file"] + logger.info("Using Everything DE dataset.") else: input_filename = osm_config["target"]["file_testmode"] + logger.info("Using testmode SH dataset.") input_file = Path(".") / "openstreetmap" / input_filename style_file = ( @@ -299,7 +301,7 @@ class OpenStreetMap(Dataset): def __init__(self, dependencies): super().__init__( name="OpenStreetMap", - version="0.0.4", + version="0.0.5", dependencies=dependencies, tasks=(download, to_postgres, modify_tables, add_metadata), ) diff --git a/src/egon/data/datasets/osmtgmod/__init__.py b/src/egon/data/datasets/osmtgmod/__init__.py index 21a84b73a..4cbe64b13 100644 --- a/src/egon/data/datasets/osmtgmod/__init__.py +++ b/src/egon/data/datasets/osmtgmod/__init__.py @@ -6,11 +6,12 @@ import logging import os import shutil +import subprocess import sys import psycopg2 -from egon.data import db +from egon.data import db, logger from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.datasets.osmtgmod.substation import extract @@ -19,7 +20,6 @@ import egon.data.subprocess as subproc - def run(): sys.setrecursionlimit(5000) # execute osmTGmod @@ -52,18 +52,47 @@ def import_osm_data(): # Delete repository if it already exists if osmtgmod_repos.exists() and osmtgmod_repos.is_dir(): - shutil.rmtree(osmtgmod_repos) + try: + status = subprocess.check_output( + ["git", "status"], cwd=(osmtgmod_repos).absolute() + ) + if status.startswith( + b"Auf Branch features/egon" + ) or status.startswith(b"On branch features/egon"): + logger.info("OsmTGmod cloned and right branch checked out.") - subproc.run( - [ - "git", - "clone", - "--single-branch", - "--branch", - "features/egon", - "https://github.com/openego/osmTGmod.git", - ] - ) + else: + subproc.run( + [ + "git", + "checkout", + "features/egon", + ] + ) + except subprocess.CalledProcessError: + shutil.rmtree(osmtgmod_repos) + subproc.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "features/egon", + "https://github.com/openego/osmTGmod.git", + ] + ) + else: + + subproc.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "features/egon", + "https://github.com/openego/osmTGmod.git", + ] + ) data_config = egon.data.config.datasets() osm_config = data_config["openstreetmap"]["original_data"] @@ -412,9 +441,11 @@ def osmtgmod( # beware: comments in C-like style (such as /* comment */) arn't parsed! sqlfile_without_comments = "".join( [ - line.lstrip().split("--")[0] + "\n" - if not line.lstrip().split("--")[0] == "" - else "" + ( + line.lstrip().split("--")[0] + "\n" + if not line.lstrip().split("--")[0] == "" + else "" + ) for line in sqlfile.split("\n") ] ) @@ -521,8 +552,14 @@ def to_pypsa(): """ ) - for scenario_name in ["'eGon2035'", "'eGon100RE'", "'status2019'"]: + # for scenario_name in ["'eGon2035'", "'eGon100RE'", "'status2019'"]: + scenario_list = egon.data.config.settings()["egon-data"]["--scenarios"] + scenario_list = [ + f"'{scn}'" if not scn[1] == "'" else scn for scn in scenario_list + ] + for scenario_name in scenario_list: + # TODO maybe not needed anymore? capital_cost = get_sector_parameters( "electricity", scenario_name.replace("'", "") )["capital_cost"] @@ -628,19 +665,19 @@ def to_pypsa(): WHERE a.line_id = result.line_id AND scn_name = {scenario_name}; - -- set capital costs for eHV-lines + -- set capital costs for eHV-lines UPDATE grid.egon_etrago_line SET capital_cost = {capital_cost['ac_ehv_overhead_line']} * length WHERE v_nom > 110 AND scn_name = {scenario_name}; - -- set capital costs for HV-lines + -- set capital costs for HV-lines UPDATE grid.egon_etrago_line SET capital_cost = {capital_cost['ac_hv_overhead_line']} * length WHERE v_nom = 110 AND scn_name = {scenario_name}; - - -- set capital costs for transformers + + -- set capital costs for transformers UPDATE grid.egon_etrago_transformer a SET capital_cost = {capital_cost['transformer_380_220']} WHERE (a.bus0 IN ( @@ -688,20 +725,20 @@ def to_pypsa(): SELECT bus_id FROM grid.egon_etrago_bus WHERE v_nom = 220)) AND scn_name = {scenario_name}; - - -- set lifetime for eHV-lines + + -- set lifetime for eHV-lines UPDATE grid.egon_etrago_line - SET lifetime = {lifetime['ac_ehv_overhead_line']} + SET lifetime = {lifetime['ac_ehv_overhead_line']} WHERE v_nom > 110 AND scn_name = {scenario_name}; - -- set capital costs for HV-lines + -- set capital costs for HV-lines UPDATE grid.egon_etrago_line SET lifetime = {lifetime['ac_hv_overhead_line']} WHERE v_nom = 110 AND scn_name = {scenario_name}; - - -- set capital costs for transformers + + -- set capital costs for transformers UPDATE grid.egon_etrago_transformer a SET lifetime = {lifetime['transformer_380_220']} WHERE (a.bus0 IN ( @@ -749,7 +786,7 @@ def to_pypsa(): SELECT bus_id FROM grid.egon_etrago_bus WHERE v_nom = 220)) AND scn_name = {scenario_name}; - + -- delete buses without connection to AC grid and generation or -- load assigned @@ -772,11 +809,30 @@ def to_pypsa(): ) +def fix_transformer_snom(): + db.execute_sql( + """ + UPDATE grid.egon_etrago_transformer AS t + SET s_nom = CAST( + LEAST( + (SELECT SUM(COALESCE(l.s_nom,0)) + FROM grid.egon_etrago_line AS l + WHERE (l.bus0 = t.bus0 OR l.bus1 = t.bus0) + AND l.scn_name = t.scn_name), + (SELECT SUM(COALESCE(l.s_nom,0)) + FROM grid.egon_etrago_line AS l + WHERE (l.bus0 = t.bus1 OR l.bus1 = t.bus1) + AND l.scn_name = t.scn_name) + ) AS smallint + ); + """) + + class Osmtgmod(Dataset): def __init__(self, dependencies): super().__init__( name="Osmtgmod", - version="0.0.5", + version="0.0.7", dependencies=dependencies, tasks=( import_osm_data, @@ -785,5 +841,6 @@ def __init__(self, dependencies): extract, to_pypsa, }, + fix_transformer_snom, ), ) diff --git a/src/egon/data/datasets/power_plants/__init__.py b/src/egon/data/datasets/power_plants/__init__.py index 5bd19b268..830cf3eb0 100755 --- a/src/egon/data/datasets/power_plants/__init__.py +++ b/src/egon/data/datasets/power_plants/__init__.py @@ -1,7 +1,9 @@ """The central module containing all code dealing with power plant data. """ -from geoalchemy2 import Geometry + from pathlib import Path + +from geoalchemy2 import Geometry from shapely.geometry import Point from sqlalchemy import BigInteger, Column, Float, Integer, Sequence, String from sqlalchemy.dialects.postgresql import JSONB @@ -9,10 +11,9 @@ from sqlalchemy.orm import sessionmaker import geopandas as gpd import numpy as np -import logging import pandas as pd -from egon.data import db +from egon.data import db, logger from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.mastr import ( WORKING_DIR_MASTR_NEW, @@ -904,81 +905,9 @@ def allocate_other_power_plants(): session.commit() -def power_plants_status_quo(scn_name="status2019"): - con = db.engine() - session = sessionmaker(bind=db.engine())() - cfg = egon.data.config.datasets()["power_plants"] - db.execute_sql( - f""" - DELETE FROM {cfg['target']['schema']}.{cfg['target']['table']} - WHERE carrier IN ('wind_onshore', 'solar', 'biomass', - 'run_of_river', 'reservoir', 'solar_rooftop', - 'wind_offshore', 'nuclear', 'coal', 'lignite', 'oil', - 'gas') - AND scenario = '{scn_name}' - """ - ) - - # import municipalities to assign missing geom and bus_id - geom_municipalities = gpd.GeoDataFrame.from_postgis( - """ - SELECT gen, ST_UNION(geometry) as geom - FROM boundaries.vg250_gem - GROUP BY gen - """, - con, - geom_col="geom", - ).set_index("gen") - geom_municipalities["geom"] = geom_municipalities["geom"].centroid - - mv_grid_districts = gpd.GeoDataFrame.from_postgis( - f""" - SELECT * FROM {cfg['sources']['egon_mv_grid_district']} - """, - con, - ) - mv_grid_districts.geom = mv_grid_districts.geom.to_crs(4326) - - def fill_missing_bus_and_geom(gens, carrier): - # drop generators without data to get geometry. - drop_id = gens[ - (gens.geom.is_empty) - & ~(gens.location.isin(geom_municipalities.index)) - ].index - new_geom = gens["capacity"][ - (gens.geom.is_empty) - & (gens.location.isin(geom_municipalities.index)) - ] - logging.info( - f"""{len(drop_id)} {carrier} generator(s) ({gens.loc[drop_id, 'capacity'] - .sum()}MW) were drop""" - ) - - logging.info( - f"""{len(new_geom)} {carrier} generator(s) ({new_geom - .sum()}MW) received a geom based on location - """ - ) - gens.drop(index=drop_id, inplace=True) - - # assign missing geometries based on location and buses based on geom - - gens["geom"] = gens.apply( - lambda x: geom_municipalities.at[x["location"], "geom"] - if x["geom"].is_empty - else x["geom"], - axis=1, - ) - gens["bus_id"] = gens.sjoin( - mv_grid_districts[["bus_id", "geom"]], how="left" - ).bus_id_right.values - - gens = gens.dropna(subset=["bus_id"]) - # convert geom to WKB - gens["geom"] = gens["geom"].to_wkt() - - return gens +def get_conventional_power_plants_non_chp(scn_name): + cfg = egon.data.config.datasets()["power_plants"] # Write conventional power plants in supply.egon_power_plants common_columns = [ "EinheitMastrNummer", @@ -988,6 +917,8 @@ def fill_missing_bus_and_geom(gens, carrier): "Breitengrad", "Gemeinde", "Inbetriebnahmedatum", + "EinheitBetriebsstatus", + "DatumEndgueltigeStilllegung", ] # import nuclear power plants nuclear = pd.read_csv( @@ -1014,29 +945,60 @@ def fill_missing_bus_and_geom(gens, carrier): ) ] + # drop plants that are decommissioned + conv["DatumEndgueltigeStilllegung"] = pd.to_datetime( + conv["DatumEndgueltigeStilllegung"] + ) + + # keep plants that were decommissioned after the max date + conv.loc[ + ( + conv.DatumEndgueltigeStilllegung + > egon.data.config.datasets()["mastr_new"][f"{scn_name}_date_max"] + ), + "EinheitBetriebsstatus", + ] = "InBetrieb" + + conv = conv.loc[conv.EinheitBetriebsstatus == "InBetrieb"] + + conv = conv.drop( + columns=["EinheitBetriebsstatus", "DatumEndgueltigeStilllegung"] + ) + # convert from KW to MW conv["Nettonennleistung"] = conv["Nettonennleistung"] / 1000 + # drop generators installed after 2019 conv["Inbetriebnahmedatum"] = pd.to_datetime(conv["Inbetriebnahmedatum"]) conv = conv[ conv["Inbetriebnahmedatum"] - < egon.data.config.datasets()["mastr_new"]["status2019_date_max"] + < egon.data.config.datasets()["mastr_new"][f"{scn_name}_date_max"] ] + conv_cap_chp = ( + conv.groupby("Energietraeger")["Nettonennleistung"].sum() / 1e3 + ) # drop chp generators conv["ThermischeNutzleistung"] = conv["ThermischeNutzleistung"].fillna(0) conv = conv[conv.ThermischeNutzleistung == 0] + conv_cap_no_chp = ( + conv.groupby("Energietraeger")["Nettonennleistung"].sum() / 1e3 + ) + + logger.info("Dropped CHP generators in GW") + logger.info(conv_cap_chp - conv_cap_no_chp) # rename carriers - conv.loc[conv.Energietraeger == "Braunkohle", "Energietraeger"] = "lignite" - conv.loc[conv.Energietraeger == "Steinkohle", "Energietraeger"] = "coal" - conv.loc[conv.Energietraeger == "Erdgas", "Energietraeger"] = "gas" - conv.loc[ - conv.Energietraeger == "Mineralölprodukte", "Energietraeger" - ] = "oil" - conv.loc[ - conv.Energietraeger == "Kernenergie", "Energietraeger" - ] = "nuclear" + # rename carriers + conv["Energietraeger"] = conv["Energietraeger"].replace( + to_replace={ + "Braunkohle": "lignite", + "Steinkohle": "coal", + "Erdgas": "gas", + "Mineralölprodukte": "oil", + "Kernenergie": "nuclear", + } + ) # rename columns conv.rename( @@ -1049,49 +1011,135 @@ def fill_missing_bus_and_geom(gens, carrier): inplace=True, ) conv["bus_id"] = np.nan - conv["geom"] = gpd.points_from_xy( conv.Laengengrad, conv.Breitengrad, crs=4326 ) - conv.loc[ - (conv.Laengengrad.isna() | conv.Breitengrad.isna()), "geom" - ] = Point() + conv.loc[(conv.Laengengrad.isna() | conv.Breitengrad.isna()), "geom"] = ( + Point() + ) conv = gpd.GeoDataFrame(conv, geometry="geom") - conv = fill_missing_bus_and_geom(conv, carrier="conventional") + # assign voltage level by capacity conv["voltage_level"] = np.nan - conv["voltage_level"] = assign_voltage_level_by_capacity( conv.rename(columns={"capacity": "Nettonennleistung"}) ) + # Add further information + conv["sources"] = [{"el_capacity": "MaStR"}] * conv.shape[0] + conv["source_id"] = conv["gens_id"].apply(lambda x: {"MastrNummer": x}) + conv["scenario"] = scn_name - for i, row in conv.iterrows(): - entry = EgonPowerPlants( - sources={"el_capacity": "MaStR"}, - source_id={"MastrNummer": row.gens_id}, - carrier=row.carrier, - el_capacity=row.capacity, - scenario=scn_name, - bus_id=row.bus_id, - voltage_level=row.voltage_level, - geom=row.geom, + return conv + + +def power_plants_status_quo(scn_name="status2019"): + def fill_missing_bus_and_geom(gens, carrier): + # drop generators without data to get geometry. + drop_id = gens[ + (gens.geom.is_empty) + & ~(gens.location.isin(geom_municipalities.index)) + ].index + new_geom = gens["capacity"][ + (gens.geom.is_empty) + & (gens.location.isin(geom_municipalities.index)) + ] + logger.info( + f"""{len(drop_id)} {carrier} generator(s) ({int(gens.loc[drop_id, 'capacity'] + .sum())}MW) were drop""" ) - session.add(entry) - session.commit() - logging.info( + logger.info( + f"""{len(new_geom)} {carrier} generator(s) ({int(new_geom + .sum())}MW) received a geom based on location + """ + ) + gens.drop(index=drop_id, inplace=True) + + # assign missing geometries based on location and buses based on geom + + gens["geom"] = gens.apply( + lambda x: ( + geom_municipalities.at[x["location"], "geom"] + if x["geom"].is_empty + else x["geom"] + ), + axis=1, + ) + gens["bus_id"] = gens.sjoin( + mv_grid_districts[["bus_id", "geom"]], how="left" + ).bus_id_right.values + + gens = gens.dropna(subset=["bus_id"]) + # convert geom to WKB + gens["geom"] = gens["geom"].to_wkt() + + return gens + + def convert_master_info(df): + # Add further information + df["sources"] = [{"el_capacity": "MaStR"}] * df.shape[0] + df["source_id"] = df["gens_id"].apply(lambda x: {"MastrNummer": x}) + return df + + def log_insert_capacity(df, tech): + logger.info( + f""" + {len(df)} {tech} generators with a total installed capacity of + {int(df["el_capacity"].sum())} MW were inserted into the db + """ + ) + + con = db.engine() + cfg = egon.data.config.datasets()["power_plants"] + + db.execute_sql( f""" - {len(conv)} conventional generators with a total installed capacity of - {conv.capacity.sum()}MW were inserted into the db - """ + DELETE FROM {cfg['target']['schema']}.{cfg['target']['table']} + WHERE carrier IN ('wind_onshore', 'solar', 'biomass', + 'run_of_river', 'reservoir', 'solar_rooftop', + 'wind_offshore', 'nuclear', 'coal', 'lignite', 'oil', + 'gas') + AND scenario = '{scn_name}' + """ ) - # Write hydro power plants in supply.egon_power_plants - map_hydro = { - "Laufwasseranlage": "run_of_river", - "Speicherwasseranlage": "reservoir", - } + # import municipalities to assign missing geom and bus_id + geom_municipalities = gpd.GeoDataFrame.from_postgis( + """ + SELECT gen, ST_UNION(geometry) as geom + FROM boundaries.vg250_gem + GROUP BY gen + """, + con, + geom_col="geom", + ).set_index("gen") + geom_municipalities["geom"] = geom_municipalities["geom"].centroid + + mv_grid_districts = gpd.GeoDataFrame.from_postgis( + f""" + SELECT * FROM {cfg['sources']['egon_mv_grid_district']} + """, + con, + ) + mv_grid_districts.geom = mv_grid_districts.geom.to_crs(4326) + + # Conventional non CHP + # ################### + conv = get_conventional_power_plants_non_chp(scn_name) + conv = fill_missing_bus_and_geom(conv, carrier="conventional") + conv= conv.rename(columns={"capacity": "el_capacity"}) + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + conv.to_dict(orient="records"), + ) + + log_insert_capacity(conv, tech="conventional non chp") + + # Hydro Power Plants + # ################### hydro = gpd.GeoDataFrame.from_postgis( f"""SELECT *, city AS location FROM {cfg['sources']['hydro']} WHERE plant_type IN ('Laufwasseranlage', 'Speicherwasseranlage')""", @@ -1101,28 +1149,28 @@ def fill_missing_bus_and_geom(gens, carrier): hydro = fill_missing_bus_and_geom(hydro, carrier="hydro") - for i, row in hydro.iterrows(): - entry = EgonPowerPlants( - sources={"el_capacity": "MaStR"}, - source_id={"MastrNummer": row.gens_id}, - carrier=map_hydro[row.plant_type], - el_capacity=row.capacity, - voltage_level=row.voltage_level, - bus_id=row.bus_id, - scenario=scn_name, - geom=row.geom, + hydro = convert_master_info(hydro) + hydro["carrier"] = hydro["plant_type"].replace( + to_replace={ + "Laufwasseranlage": "run_of_river", + "Speicherwasseranlage": "reservoir", + } + ) + hydro["scenario"] = scn_name + hydro = hydro.rename(columns={"capacity": "el_capacity"}) + hydro = hydro.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + hydro.to_dict(orient="records"), ) - session.add(entry) - session.commit() - logging.info( - f""" - {len(hydro)} hydro generators with a total installed capacity of - {hydro.capacity.sum()}MW were inserted into the db - """ - ) + log_insert_capacity(hydro, tech="hydro") - # Write biomass power plants in supply.egon_power_plants + # Biomass + # ################### biomass = gpd.GeoDataFrame.from_postgis( f"""SELECT *, city AS location FROM {cfg['sources']['biomass']}""", con, @@ -1135,28 +1183,23 @@ def fill_missing_bus_and_geom(gens, carrier): biomass = fill_missing_bus_and_geom(biomass, carrier="biomass") - for i, row in biomass.iterrows(): - entry = EgonPowerPlants( - sources={"el_capacity": "MaStR"}, - source_id={"MastrNummer": row.gens_id}, - carrier="biomass", - el_capacity=row.capacity, - scenario=scn_name, - bus_id=row.bus_id, - voltage_level=row.voltage_level, - geom=row.geom, + biomass = convert_master_info(biomass) + biomass["scenario"] = scn_name + biomass["carrier"] = "biomass" + biomass = biomass.rename(columns={"capacity": "el_capacity"}) + biomass = biomass.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + biomass.to_dict(orient="records"), ) - session.add(entry) - session.commit() - logging.info( - f""" - {len(biomass)} biomass generators with a total installed capacity of - {biomass.capacity.sum()}MW were inserted into the db - """ - ) + log_insert_capacity(biomass, tech="biomass") - # Write solar power plants in supply.egon_power_plants + # Solar + # ################### solar = gpd.GeoDataFrame.from_postgis( f"""SELECT *, city AS location FROM {cfg['sources']['pv']} WHERE site_type IN ('Freifläche', @@ -1168,33 +1211,25 @@ def fill_missing_bus_and_geom(gens, carrier): "Freifläche": "solar", "Bauliche Anlagen (Hausdach, Gebäude und Fassade)": "solar_rooftop", } - solar["site_type"] = solar["site_type"].map(map_solar) + solar["carrier"] = solar["site_type"].replace(to_replace=map_solar) solar = fill_missing_bus_and_geom(solar, carrier="solar") - - solar = pd.DataFrame(solar, index=solar.index) - for i, row in solar.iterrows(): - entry = EgonPowerPlants( - sources={"el_capacity": "MaStR"}, - source_id={"MastrNummer": row.gens_id}, - carrier=row.site_type, - el_capacity=row.capacity, - scenario=scn_name, - bus_id=row.bus_id, - voltage_level=row.voltage_level, - geom=row.geom, + solar = convert_master_info(solar) + solar["scenario"] = scn_name + solar = solar.rename(columns={"capacity": "el_capacity"}) + solar = solar.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + solar.to_dict(orient="records"), ) - session.add(entry) - session.commit() - logging.info( - f""" - {len(solar)} solar generators with a total installed capacity of - {solar.capacity.sum()}MW were inserted into the db - """ - ) + log_insert_capacity(solar, tech="solar") - # Write wind_onshore power plants in supply.egon_power_plants + # Wind + # ################### wind_onshore = gpd.GeoDataFrame.from_postgis( f"""SELECT *, city AS location FROM {cfg['sources']['wind']}""", con, @@ -1204,29 +1239,21 @@ def fill_missing_bus_and_geom(gens, carrier): wind_onshore = fill_missing_bus_and_geom( wind_onshore, carrier="wind_onshore" ) - - for i, row in wind_onshore.iterrows(): - entry = EgonPowerPlants( - sources={"el_capacity": "MaStR"}, - source_id={"MastrNummer": row.gens_id}, - carrier="wind_onshore", - el_capacity=row.capacity, - scenario=scn_name, - bus_id=row.bus_id, - voltage_level=row.voltage_level, - geom=row.geom, + wind_onshore = convert_master_info(wind_onshore) + wind_onshore["scenario"] = scn_name + wind_onshore = wind_onshore.rename(columns={"capacity": "el_capacity"}) + wind_onshore["carrier"] = "wind_onshore" + wind_onshore = wind_onshore.drop(columns="id") + + # Write into DB + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonPowerPlants, + wind_onshore.to_dict(orient="records"), ) - session.add(entry) - session.commit() - logging.info( - f""" - {len(wind_onshore)} wind_onshore generators with a total installed capacity of - {wind_onshore.capacity.sum()}MW were inserted into the db - """ - ) + log_insert_capacity(wind_onshore, tech="wind_onshore") - return tasks = ( @@ -1234,8 +1261,15 @@ def fill_missing_bus_and_geom(gens, carrier): import_mastr, ) -if "status2019" in egon.data.config.settings()["egon-data"]["--scenarios"]: - tasks = tasks + (power_plants_status_quo,) +for scn_name in egon.data.config.settings()["egon-data"]["--scenarios"]: + if "status" in scn_name: + tasks += ( + wrapped_partial( + power_plants_status_quo, + scn_name=scn_name, + postfix=f"_{scn_name[-4:]}", + ), + ) if ( "eGon2035" in egon.data.config.settings()["egon-data"]["--scenarios"] @@ -1269,7 +1303,7 @@ class PowerPlants(Dataset): def __init__(self, dependencies): super().__init__( name="PowerPlants", - version="0.0.23", + version="0.0.26", dependencies=dependencies, tasks=tasks, ) diff --git a/src/egon/data/datasets/power_plants/mastr.py b/src/egon/data/datasets/power_plants/mastr.py index 9eda90703..1208311a4 100644 --- a/src/egon/data/datasets/power_plants/mastr.py +++ b/src/egon/data/datasets/power_plants/mastr.py @@ -317,7 +317,7 @@ def voltage_levels(p: float) -> int: # (eGon2021 scenario) len_old = len(units) ts = pd.Timestamp( - egon.data.config.datasets()["mastr_new"]["status2019_date_max"] + egon.data.config.datasets()["mastr_new"]["status2023_date_max"] ) units = units.loc[pd.to_datetime(units.Inbetriebnahmedatum) <= ts] print( diff --git a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py index 53a7416b8..aadd3f654 100644 --- a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py +++ b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py @@ -38,6 +38,7 @@ * Plant metadata (e.g. plant orientation) is also added random and weighted from MaStR data as basis. """ + from __future__ import annotations from collections import Counter @@ -66,6 +67,7 @@ from egon.data.datasets.mastr import WORKING_DIR_MASTR_NEW from egon.data.datasets.power_plants.mastr import EgonPowerPlantsPv from egon.data.datasets.scenario_capacities import EgonScenarioCapacities +from egon.data.datasets.scenario_parameters import get_scenario_year from egon.data.datasets.zensus_vg250 import Vg250Gem engine = db.engine() @@ -161,6 +163,7 @@ SCENARIOS = config.settings()["egon-data"]["--scenarios"] SCENARIO_TIMESTAMP = { "status2019": pd.Timestamp("2020-01-01", tz="UTC"), + "status2023": pd.Timestamp("2024-01-01", tz="UTC"), "eGon2035": pd.Timestamp("2035-01-01", tz="UTC"), "eGon100RE": pd.Timestamp("2050-01-01", tz="UTC"), } @@ -978,10 +981,14 @@ def drop_buildings_outside_muns( def egon_building_peak_loads(): - sql = """ + + # use active scenario wich is closest to today + scenario = sorted(SCENARIOS, key=get_scenario_year)[0] + + sql = f""" SELECT building_id FROM demand.egon_building_electricity_peak_loads - WHERE scenario = 'eGon2035' + WHERE scenario = '{scenario}' """ return ( @@ -1248,9 +1255,9 @@ def allocate_pv( assert len(assigned_buildings) == len(assigned_buildings.gens_id.unique()) - q_mastr_gdf.loc[ - assigned_buildings.gens_id, "building_id" - ] = assigned_buildings.index + q_mastr_gdf.loc[assigned_buildings.gens_id, "building_id"] = ( + assigned_buildings.index + ) assigned_gens = q_mastr_gdf.loc[~q_mastr_gdf.building_id.isna()] @@ -2768,6 +2775,7 @@ def add_bus_ids_sq( grid_districts_gdf = grid_districts(EPSG) mask = buildings_gdf.scenario == "status_quo" + buildings_gdf.loc[mask, "bus_id"] = ( buildings_gdf.loc[mask] .sjoin(grid_districts_gdf, how="left") @@ -2782,7 +2790,9 @@ def pv_rooftop_to_buildings(): mastr_gdf = load_mastr_data() - ts = pd.Timestamp(config.datasets()["mastr_new"]["status2019_date_max"]) + status_quo = "status2023" + + ts = pd.Timestamp(config.datasets()["mastr_new"][f"{status_quo}_date_max"]) mastr_gdf = mastr_gdf.loc[ pd.to_datetime(mastr_gdf.Inbetriebnahmedatum) <= ts @@ -2795,9 +2805,10 @@ def pv_rooftop_to_buildings(): ) all_buildings_gdf = ( - desagg_mastr_gdf.assign(scenario="status_quo") + desagg_mastr_gdf.assign(scenario=status_quo) .reset_index() .rename(columns={"geometry": "geom", "EinheitMastrNummer": "gens_id"}) + .set_geometry("geom") ) scenario_buildings_gdf = all_buildings_gdf.copy() @@ -2805,25 +2816,34 @@ def pv_rooftop_to_buildings(): cap_per_bus_id_df = pd.DataFrame() for scenario in SCENARIOS: - if scenario == "status2019": - desagg_mastr_gdf = desagg_mastr_gdf.loc[ - pd.to_datetime(desagg_mastr_gdf.Inbetriebnahmedatum) <= ts - ] + if scenario == status_quo: + continue + elif "status" in scenario: + ts = pd.Timestamp( + config.datasets()["mastr_new"][f"{scenario}_date_max"] + ) + scenario_buildings_gdf = scenario_buildings_gdf.loc[ pd.to_datetime(scenario_buildings_gdf.Inbetriebnahmedatum) <= ts ] - logger.debug(f"Desaggregating scenario {scenario}.") - ( - scenario_buildings_gdf, - cap_per_bus_id_scenario_df, - ) = allocate_scenarios( # noqa: F841 - desagg_mastr_gdf, - desagg_buildings_gdf, - scenario_buildings_gdf, - scenario, - ) + else: + logger.debug(f"Desaggregating scenario {scenario}.") + + ( + scenario_buildings_gdf, + cap_per_bus_id_scenario_df, + ) = allocate_scenarios( # noqa: F841 + desagg_mastr_gdf, + desagg_buildings_gdf, + scenario_buildings_gdf, + scenario, + ) + + cap_per_bus_id_df = pd.concat( + [cap_per_bus_id_df, cap_per_bus_id_scenario_df] + ) all_buildings_gdf = gpd.GeoDataFrame( pd.concat( @@ -2833,10 +2853,6 @@ def pv_rooftop_to_buildings(): geometry="geom", ) - cap_per_bus_id_df = pd.concat( - [cap_per_bus_id_df, cap_per_bus_id_scenario_df] - ) - # add weather cell all_buildings_gdf = add_weather_cell_id(all_buildings_gdf) diff --git a/src/egon/data/datasets/power_plants/wind_offshore.py b/src/egon/data/datasets/power_plants/wind_offshore.py index 68cb09d59..85aa2083e 100644 --- a/src/egon/data/datasets/power_plants/wind_offshore.py +++ b/src/egon/data/datasets/power_plants/wind_offshore.py @@ -39,7 +39,9 @@ def map_id_bus(scenario): "inhausen": "29420322", "Cloppenburg": "50643382", } - elif scenario == "status2019": + elif "status" in scenario: + year = int(scenario[-4:]) + id_bus = { "UW Inhausen": "29420322", "UW Bentwisch": "32063539", @@ -52,6 +54,14 @@ def map_id_bus(scenario): "UW Diele": "177829920", "UW Lubmin": "460134233", } + + if year >= 2023: + # No update needed as no new stations used for offshore wind + # between 2019 and 2023 + pass + + # TODO: If necessary add new stations when generating status quo > 2023 + else: id_bus = {} @@ -59,7 +69,7 @@ def map_id_bus(scenario): def assign_ONEP_areas(): - assign_onep = { + return { "Büttel": "NOR-4-1", "Heide/West": "NOR-10-2", "Suchraum Gemeinden Ibbenbüren/Mettingen/Westerkappeln": "NOR-9-2", @@ -85,11 +95,10 @@ def assign_ONEP_areas(): "inhausen": "NOR-0-2", "Cloppenburg": "NOR-4-1", } - return assign_onep def map_ONEP_areas(): - onep = { + return { "NOR-0-1": Point(6.5, 53.6), "NOR-0-2": Point(8.07, 53.76), "NOR-1": Point(6.21, 54.06), @@ -121,7 +130,6 @@ def map_ONEP_areas(): "OST-3-2": Point(13.16, 54.98), "OST-7-1": Point(12.25, 54.5), } - return onep def insert(): @@ -187,7 +195,9 @@ def insert(): ) offshore.dropna(subset=["Netzverknuepfungspunkt"], inplace=True) - elif scenario == "status2019": + elif "status" in scenario: + year = int(scenario[-4:]) + offshore_path = ( Path(".") / "data_bundle_egon_data" @@ -214,7 +224,10 @@ def insert(): }, inplace=True, ) - offshore = offshore[offshore["Inbetriebnahme"] <= 2019] + offshore = offshore[offshore["Inbetriebnahme"] <= year] + + else: + raise ValueError(f"{scenario=} is not valid.") id_bus = map_id_bus(scenario) @@ -263,7 +276,7 @@ def insert(): offshore.drop(["Name ONEP/NEP"], axis=1, inplace=True) - if scenario == "status2019": + if "status" in scenario: offshore.drop(["Inbetriebnahme"], axis=1, inplace=True) # Scale capacities for eGon100RE diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index b46ac6e49..46d02e04d 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -13,7 +13,7 @@ from egon.data import db from egon.data.config import settings -from egon.data.datasets import Dataset +from egon.data.datasets import Dataset, wrapped_partial import egon.data.config # will be later imported from another file @@ -97,8 +97,8 @@ def nuts_mapping(): return nuts_mapping -def insert_capacities_status2019(): - """Insert capacity of rural heat pumps for status2019 +def insert_capacities_status_quo(scenario: str) -> None: + """Insert capacity of rural heat pumps for status quo Returns ------- @@ -114,19 +114,32 @@ def insert_capacities_status2019(): DELETE FROM {targets['scenario_capacities']['schema']}. {targets['scenario_capacities']['table']} - WHERE scenario_name = 'status2019' + WHERE scenario_name = '{scenario}' """ ) - # Rural heat capacity for 2019 according to NEP 2035, version 2021 - rural_heat_capacity = 1e6 * 5e-3 + rural_heat_capacity = { + # Rural heat capacity for 2019 according to NEP 2035, version 2021 + "status2019": 1e6 * 5e-3, + # Rural heat capacity for 2023 according to NEP 2037, version 2023 + # 1.2 Mio. for 2020 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ + # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # and 3 kW per heat pump + # https://www.netzentwicklungsplan.de/sites/default/files/2022-11/ + # NEP_2035_V2021_2_Entwurf_Teil1.pdf#page=33 + # plus 0.15 Mio. 2021 and 0.24 Mio. in 2022 + # https://www.enercity.de/magazin/unsere-welt/waermepumpen-boom + # plus 0.2 Mio. in H1 2023 -> Assumption 2023: 2 * 0.2 Mio = 0.4 Mio. + "status2023": (1.2 + 0.15 + 0.24 + 0.4) * 1e6 * 3e-3, + }[scenario] if settings()["egon-data"]["--dataset-boundary"] != "Everything": rural_heat_capacity *= population_share() db.execute_sql( f""" - INSERT INTO + INSERT INTO {targets['scenario_capacities']['schema']}. {targets['scenario_capacities']['table']} (component, carrier, capacity, nuts, scenario_name) @@ -135,17 +148,28 @@ def insert_capacities_status2019(): 'residential_rural_heat_pump', {rural_heat_capacity}, 'DE', - 'status2019' + '{scenario}' ) """ ) # Include small storages for scenario2019 - small_storages = 600 # MW for Germany + small_storages = { + # MW for Germany + "status2019": 600, + # 1.3 GW in 2020/2021 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ + # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # Installed quantity 2020: 272,000 + # Installed quantity 2023: 1,197,000 + # https://www.photovoltaik.eu/solarspeicher/ + # bsw-speicherkapazitaet-von-heimspeichern-2023-verdoppelt + "status2023": 1300 * 1197 / 272, + }[scenario] db.execute_sql( f""" - INSERT INTO + INSERT INTO {targets['scenario_capacities']['schema']}. {targets['scenario_capacities']['table']} (component, carrier, capacity, nuts, scenario_name) @@ -154,7 +178,7 @@ def insert_capacities_status2019(): 'battery', {small_storages}, 'DE', - 'status2019' + '{scenario}' ) """ ) @@ -442,6 +466,7 @@ def map_carrier(): "Kernenergie": "nuclear", "Pumpspeicher": "pumped_hydro", "Mineralöl-\nProdukte": "oil", + "Biomasse": "biomass", } ) @@ -814,25 +839,32 @@ def eGon100_capacities(): tasks = (create_table,) -if "status2019" in egon.data.config.settings()["egon-data"]["--scenarios"]: - tasks = tasks + (insert_capacities_status2019, insert_data_nep) +scenarios = egon.data.config.settings()["egon-data"]["--scenarios"] -if ( - "eGon2035" in egon.data.config.settings()["egon-data"]["--scenarios"] -) and not ( - "status2019" in egon.data.config.settings()["egon-data"]["--scenarios"] -): - tasks = tasks + (insert_data_nep,) +status_quo = False -if "eGon100RE" in egon.data.config.settings()["egon-data"]["--scenarios"]: - tasks = tasks + (eGon100_capacities,) +for scenario in scenarios: + if "status" in scenario: + tasks += ( + wrapped_partial( + insert_capacities_status_quo, scenario=scenario, + postfix=f"_{scenario[-2:]}" + ), + ) + status_quo = True + +if status_quo or ("eGon2035" in scenarios): + tasks += (insert_data_nep,) + +if "eGon100RE" in scenarios: + tasks += (eGon100_capacities,) class ScenarioCapacities(Dataset): def __init__(self, dependencies): super().__init__( name="ScenarioCapacities", - version="0.0.14", + version="0.0.17", dependencies=dependencies, tasks=tasks, ) diff --git a/src/egon/data/datasets/scenario_parameters/__init__.py b/src/egon/data/datasets/scenario_parameters/__init__.py index 3133fc287..0c4b2b24e 100755 --- a/src/egon/data/datasets/scenario_parameters/__init__.py +++ b/src/egon/data/datasets/scenario_parameters/__init__.py @@ -1,5 +1,6 @@ """The central module containing all code dealing with scenario table. """ + from pathlib import Path from urllib.request import urlretrieve import shutil @@ -45,6 +46,19 @@ def create_table(): EgonScenario.__table__.create(bind=engine, checkfirst=True) +def get_scenario_year(scenario_name): + """Derives scenarios year from scenario name. Scenario + eGon100RE is an exception as year is not in the name.""" + try: + year = int(scenario_name[-4:]) + except ValueError as e: + if e.args[0] == "invalid literal for int() with base 10: '00RE'": + year = 2050 # eGon100RE + else: + raise ValueError("The names of the scenarios do not end with the year!") + return year + + def insert_scenarios(): """Insert scenarios and their parameters to scenario table @@ -123,9 +137,9 @@ def insert_scenarios(): eGon2021.mobility_parameters = parameters.mobility(eGon2021.name) session.add(eGon2021) - - session.commit() - + + session.commit() + # Scenario status2019 status2019 = EgonScenario(name="status2019") @@ -146,6 +160,27 @@ def insert_scenarios(): session.commit() + # Scenario status2023 + status2023 = EgonScenario(name="status2023") + + status2023.description = """ + Status quo ante scenario for 2023. + """ + # TODO status2023 all settings from 2019 are used + status2023.global_parameters = parameters.global_settings(status2023.name) + + status2023.electricity_parameters = parameters.electricity(status2019.name) + + status2023.gas_parameters = parameters.gas(status2019.name) + + status2023.heat_parameters = parameters.heat(status2019.name) + + status2023.mobility_parameters = parameters.mobility(status2023.name) + + session.add(status2023) + + session.commit() + def get_sector_parameters(sector, scenario=None): """Returns parameters for each sector as dictionary. @@ -184,36 +219,38 @@ def get_sector_parameters(sector, scenario=None): else: print(f"Scenario name {scenario} is not valid.") else: - values = pd.concat([ - pd.DataFrame( - db.select_dataframe( - f""" + values = pd.concat( + [ + pd.DataFrame( + db.select_dataframe( + f""" SELECT {sector}_parameters as val FROM scenario.egon_scenario_parameters WHERE name='eGon2035'""" - ).val[0], - index=["eGon2035"]), - pd.DataFrame( - db.select_dataframe( - f""" + ).val[0], + index=["eGon2035"], + ), + pd.DataFrame( + db.select_dataframe( + f""" SELECT {sector}_parameters as val FROM scenario.egon_scenario_parameters WHERE name='eGon100RE'""" - ).val[0], - index=["eGon100RE"], - ), - - pd.DataFrame( - db.select_dataframe( - f""" + ).val[0], + index=["eGon100RE"], + ), + pd.DataFrame( + db.select_dataframe( + f""" SELECT {sector}_parameters as val FROM scenario.egon_scenario_parameters WHERE name='eGon2021'""" - ).val[0], - index=["eGon2021"], - ) - ], ignore_index=True) - + ).val[0], + index=["eGon2021"], + ), + ], + ignore_index=True, + ) return values @@ -244,7 +281,7 @@ class ScenarioParameters(Dataset): def __init__(self, dependencies): super().__init__( name="ScenarioParameters", - version="0.0.15", + version="0.0.17", dependencies=dependencies, tasks=( create_table, diff --git a/src/egon/data/datasets/scenario_parameters/parameters.py b/src/egon/data/datasets/scenario_parameters/parameters.py index 91e7b6bc0..3316dc8f1 100755 --- a/src/egon/data/datasets/scenario_parameters/parameters.py +++ b/src/egon/data/datasets/scenario_parameters/parameters.py @@ -123,6 +123,36 @@ def global_settings(scenario): "weather_year": 2011, "population_year": 2021, } + + elif scenario == "status2023": + parameters = { + "weather_year": 2023, + "population_year": 2019, # TODO: check if possible for 2023 + "fuel_costs": { + # TYNDP 2020, data for 2023 (https://2020.entsos-tyndp-scenarios.eu/fuel-commodities-and-carbon-prices/) + "oil": 16.4 * 3.6, # [EUR/MWh] + "gas": 6.1 * 3.6, # [EUR/MWh] + "coal": 3.4 * 3.6, # [EUR/MWh] + "lignite": 1.1 * 3.6, # [EUR/MWh] + "nuclear": 0.47 * 3.6, # [EUR/MWh] + "biomass": read_costs(read_csv(2020), "biomass", "fuel"), + }, + "co2_costs": 83.66, # [EUR/t_CO2], source: + # https://www.iwr.de/news/co2-emissionshandel-deutschland-erzielt-2023-rekordeinnahmen-von-ueber-18-mrd-euro-news38528 + "co2_emissions": { + # Netzentwicklungsplan Strom 2037, Genehmigtr Scenariorahmen, p. 66, table 21 + # https://www.netzentwicklungsplan.de/sites/default/files/2023-01/Szenariorahmen_2037_Genehmigung.pdf + "waste": 0.165, # [t_CO2/MW_th] + "lignite": 0.393, # [t_CO2/MW_th] + "gas": 0.201, # [t_CO2/MW_th] + "nuclear": 0.0, # [t_CO2/MW_th] + "oil": 0.288, # [t_CO2/MW_th] + "coal": 0.337, # [t_CO2/MW_th] + "other_non_renewable": 0.268, # [t_CO2/MW_th] + }, + "interest_rate": 0.05, # [p.u.] + } + elif scenario == "status2019": parameters = { "weather_year": 2019, @@ -135,7 +165,7 @@ def global_settings(scenario): "nuclear": 0.47*3.6, # [EUR/MWh] "biomass": read_costs(read_csv(2020), "biomass", "fuel"), }, - "co2_costs": 24.7, # [EUR/t_CO2], source: + "co2_costs": 24.7, # [EUR/t_CO2], source: #https://de.statista.com/statistik/daten/studie/1304069/umfrage/preisentwicklung-von-co2-emissionsrechten-in-eu/ "co2_emissions": { # Netzentwicklungsplan Strom 2035, Version 2021, 1. Entwurf, p. 40, table 8 "waste": 0.165, # [t_CO2/MW_th] @@ -499,8 +529,7 @@ def electricity(scenario): elif scenario == "eGon2021": parameters = {} - elif scenario == "status2019": - + elif (scenario == "status2019") or (scenario == "status2023"): costs = read_csv(2020) parameters = {"grid_topology": "Status Quo"} @@ -1006,6 +1035,22 @@ def mobility(scenario): } } + elif scenario == "status2023": + parameters = { + "motorized_individual_travel": { + "status2023": { + "ev_count": 2577664, + "bev_mini_share": 0.1535, + "bev_medium_share": 0.3412, + "bev_luxury_share": 0.1017, + "phev_mini_share": 0.1038, + "phev_medium_share": 0.2310, + "phev_luxury_share": 0.0688, + "model_parameters": {}, + } + } + } + else: print(f"Scenario name {scenario} is not valid.") parameters = dict() @@ -1155,6 +1200,44 @@ def heat(scenario): ), } + # elif scenario == "status2023": + # parameters = { + # # source: AG Energiebilanzen 2022 https://ag-energiebilanzen.de/wp-content/uploads/2023/01/AGEB_22p2_rev-1.pdf + # "DE_demand_residential_TJ": 1754.2 * 1e3 + # + 407.5 * 1e3, # [TJ], Endenergieverbrauch Haushalte 2.1 Raumwärme + Warmwasser + # "DE_demand_service_TJ": 668.4 * 1e3 + # + 44.3 * 1e3 , # [TJ], Endenergieverbrauch GHD 3.1 Raumwärme + Warmwasser + # "DE_district_heating_share": (189760 + 38248) + # / ( + # 1658400 + 383300 + 567300 + 71500 + # ), # [TJ], source: AG Energiebilanzen 2019 (https://ag-energiebilanzen.de/wp-content/uploads/2021/11/bilanz19d.xlsx) + # } # TODO status2023 needs update + # + # costs = read_csv(2020) + # + # # Insert marginal_costs in EUR/MWh + # # marginal cost can include fuel, C02 and operation and maintenance costs + # parameters["marginal_cost"] = { + # "central_heat_pump": read_costs( + # costs, "central air-sourced heat pump", "VOM" + # ), + # "central_gas_chp": read_costs(costs, "central gas CHP", "VOM"), + # "central_gas_boiler": read_costs( + # costs, "central gas boiler", "VOM" + # ), + # "central_resistive_heater": read_costs( + # costs, "central resistive heater", "VOM" + # ), + # "rural_heat_pump": 0, # Danish Energy Agency, Technology Data for Individual Heating Plants + # } + # + # # Insert efficiency in p.u. + # parameters["efficiency"] = { + # "central_gas_boiler": read_costs( + # costs, "central gas boiler", "efficiency" + # ), + # } + else: print(f"Scenario name {scenario} is not valid.") diff --git a/src/egon/data/datasets/storages/__init__.py b/src/egon/data/datasets/storages/__init__.py index 63e8e81dc..9d86aa0f1 100755 --- a/src/egon/data/datasets/storages/__init__.py +++ b/src/egon/data/datasets/storages/__init__.py @@ -1,5 +1,6 @@ """The central module containing all code dealing with power plant data. """ + from pathlib import Path from geoalchemy2 import Geometry @@ -12,8 +13,12 @@ from egon.data import config, db from egon.data.datasets import Dataset -from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD -from egon.data.datasets.power_plants import assign_voltage_level +from egon.data.datasets.mastr import ( + WORKING_DIR_MASTR_NEW, + WORKING_DIR_MASTR_OLD, +) +from egon.data.datasets.mv_grid_districts import Vg250GemClean +from egon.data.datasets.power_plants import assign_bus_id, assign_voltage_level from egon.data.datasets.storages.home_batteries import ( allocate_home_batteries_to_buildings, ) @@ -24,6 +29,7 @@ select_mastr_pumped_hydro, select_nep_pumped_hydro, ) +from egon.data.db import session_scope Base = declarative_base() @@ -46,7 +52,7 @@ class Storages(Dataset): def __init__(self, dependencies): super().__init__( name="Storages", - version="0.0.6", + version="0.0.8", dependencies=dependencies, tasks=( create_tables, @@ -251,24 +257,238 @@ def allocate_pumped_hydro(scn, export=True): if export: # Insert into target table session = sessionmaker(bind=db.engine())() - for i, row in power_plants.iterrows(): - entry = EgonStorages( - sources={"el_capacity": row.source}, - source_id={"MastrNummer": row.MaStRNummer}, - carrier=row.carrier, - el_capacity=row.el_capacity, - voltage_level=row.voltage_level, - bus_id=row.bus_id, - scenario=row.scenario, - geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})", - ) - session.add(entry) - session.commit() + with session_scope() as session: + for i, row in power_plants.iterrows(): + entry = EgonStorages( + sources={"el_capacity": row.source}, + source_id={"MastrNummer": row.MaStRNummer}, + carrier=row.carrier, + el_capacity=row.el_capacity, + voltage_level=row.voltage_level, + bus_id=row.bus_id, + scenario=row.scenario, + geom=f"SRID=4326;POINT({row.geometry.x} {row.geometry.y})", + ) + session.add(entry) + session.commit() else: return power_plants +def allocate_pumped_hydro_sq(scn_name): + """ + Allocate pumped hydro by mastr data only. Capacities outside + germany are assigned to foreign buses. Mastr dump 2024 is used. + No filter for commissioning is applied. + Parameters + ---------- + scn_name + + Returns + ------- + + """ + sources = config.datasets()["power_plants"]["sources"] + + # Read-in data from MaStR + mastr_ph = pd.read_csv( + WORKING_DIR_MASTR_NEW / sources["mastr_storage"], + delimiter=",", + usecols=[ + "Nettonennleistung", + "EinheitMastrNummer", + "Kraftwerksnummer", + "Technologie", + "Postleitzahl", + "Laengengrad", + "Breitengrad", + "EinheitBetriebsstatus", + "LokationMastrNummer", + "Ort", + "Bundesland", + ], + dtype={"Postleitzahl": str}, + ) + + if config.settings()["egon-data"]["--dataset-boundary"] == "Schleswig-Holstein": + # Filter for Schleswig-Holstein + mastr_ph = mastr_ph.loc[mastr_ph.Bundesland == "SchleswigHolstein"] + + # Rename columns + mastr_ph = mastr_ph.rename( + columns={ + "Kraftwerksnummer": "bnetza_id", + "Technologie": "carrier", + "Postleitzahl": "plz", + "Ort": "city", + "Bundesland": "federal_state", + "Nettonennleistung": "el_capacity", + } + ) + + # Select only pumped hydro units + mastr_ph = mastr_ph.loc[mastr_ph.carrier == "Pumpspeicher"] + + # Select only pumped hydro units which are in operation + mastr_ph = mastr_ph.loc[mastr_ph.EinheitBetriebsstatus == "InBetrieb"] + + # Calculate power in MW + mastr_ph.loc[:, "el_capacity"] *= 1e-3 + + # Create geodataframe from long, lat + mastr_ph = gpd.GeoDataFrame( + mastr_ph, + geometry=gpd.points_from_xy( + mastr_ph["Laengengrad"], mastr_ph["Breitengrad"] + ), + crs="4326", + ) + + # Identify pp without geocord + mastr_ph_nogeo = mastr_ph.loc[mastr_ph["Laengengrad"].isna()] + + # Remove all PP without geocord (PP<= 30kW) + mastr_ph = mastr_ph.dropna(subset="Laengengrad") + + # Get geometry of villages/cities with same name of pp with missing geocord + with session_scope() as session: + query = session.query( + Vg250GemClean.gen, Vg250GemClean.geometry + ).filter(Vg250GemClean.gen.in_(mastr_ph_nogeo.loc[:, "city"].unique())) + df_cities = gpd.read_postgis( + query.statement, + query.session.bind, + geom_col="geometry", + crs="4326", + ) + + # Just take the first entry, inaccuracy is negligible as centroid is taken afterwards + df_cities = df_cities.drop_duplicates("gen", keep="first") + + # Use the centroid instead of polygon of region + df_cities.loc[:, "geometry"] = df_cities["geometry"].centroid + + # Add centroid geometry to pp without geometry + mastr_ph_nogeo = pd.merge( + left=df_cities, + right=mastr_ph_nogeo, + right_on="city", + left_on="gen", + how="inner", + ).drop("gen", axis=1) + + mastr_ph = pd.concat([mastr_ph, mastr_ph_nogeo], axis=0) + + # aggregate capacity per location + agg_cap = mastr_ph.groupby("geometry")["el_capacity"].sum() + + # list mastr number by location + agg_mastr = mastr_ph.groupby("geometry")["EinheitMastrNummer"].apply(list) + + # remove duplicates by location and keep only first + mastr_ph = mastr_ph.drop_duplicates(subset="geometry", keep="first").drop( + ["el_capacity", "EinheitMastrNummer"], axis=1 + ) + + # Add aggregated capacity by location + mastr_ph = pd.merge( + left=mastr_ph, right=agg_cap, left_on="geometry", right_on="geometry" + ) + + # Add list of mastr nr by location + mastr_ph = pd.merge( + left=mastr_ph, right=agg_mastr, left_on="geometry", right_on="geometry" + ) + + # Drop small pp <= 03 kW + mastr_ph = mastr_ph.loc[mastr_ph["el_capacity"] > 30] + + # Apply voltage level by capacity + mastr_ph = apply_voltage_level_thresholds(mastr_ph) + mastr_ph["voltage_level"] = mastr_ph["voltage_level"].astype(int) + + # Capacity located outside germany -> will be assigned to foreign buses + mastr_ph_foreign = mastr_ph.loc[mastr_ph["federal_state"].isna()] + + if not mastr_ph_foreign.empty: + # Get foreign buses + sql = f""" + SELECT * FROM grid.egon_etrago_bus + WHERE scn_name = '{scn_name}' + and country != 'DE' + """ + df_foreign_buses = db.select_geodataframe( + sql, geom_col="geom", epsg="4326" + ) + + # Assign closest foreign bus at voltage level to foreign pp + nearest_neighbors = [] + for vl, v_nom in {1: 380, 2: 220, 3: 110}.items(): + ph = mastr_ph_foreign.loc[mastr_ph_foreign["voltage_level"] == vl] + if ph.empty: + continue + bus = df_foreign_buses.loc[ + df_foreign_buses["v_nom"] == v_nom, + ["v_nom", "country", "bus_id", "geom"], + ] + results = gpd.sjoin_nearest( + left_df=ph, right_df=bus, how="left", distance_col="distance" + ) + nearest_neighbors.append(results) + mastr_ph_foreign = pd.concat(nearest_neighbors) + + # Keep only capacities within germany + mastr_ph = mastr_ph.dropna(subset="federal_state") + + # Assign buses within germany + mastr_ph = assign_bus_id(mastr_ph, cfg=config.datasets()["power_plants"]) + mastr_ph["bus_id"] = mastr_ph["bus_id"].astype(int) + + if not mastr_ph_foreign.empty: + # Merge foreign pp + mastr_ph = pd.concat([mastr_ph, mastr_ph_foreign]) + + # Reduce to necessary columns + mastr_ph = mastr_ph[ + [ + "el_capacity", + "voltage_level", + "bus_id", + "geometry", + "EinheitMastrNummer", + ] + ] + + # Rename and format columns + mastr_ph["carrier"] = "pumped_hydro" + mastr_ph = mastr_ph.rename( + columns={"EinheitMastrNummer": "source_id", "geometry": "geom"} + ) + mastr_ph["source_id"] = mastr_ph["source_id"].apply( + lambda x: {"MastrNummer": ", ".join(x)} + ) + mastr_ph = mastr_ph.set_geometry("geom") + mastr_ph["geom"] = mastr_ph["geom"].apply(lambda x: x.wkb_hex) + mastr_ph["scenario"] = "status2023" + mastr_ph["sources"] = [ + {"el_capacity": "MaStR aggregated by location"} + ] * mastr_ph.shape[0] + + # Delete existing units in the target table + db.execute_sql( + f""" DELETE FROM supply.egon_storages + WHERE carrier = 'pumped_hydro' + AND scenario= '{scn_name}';""" + ) + + with db.session_scope() as session: + session.bulk_insert_mappings( + EgonStorages, + mastr_ph.to_dict(orient="records"), + ) + + def allocate_pumped_hydro_eGon100RE(): """Allocates pumped_hydro plants for eGon100RE scenario based on a prox-to-now method applied on allocated pumped-hydro plants in the eGon2035 @@ -312,6 +532,7 @@ def allocate_pumped_hydro_eGon100RE(): # Get allocation of pumped_hydro plants in eGon2035 scenario as the # reference for the distribution in eGon100RE scenario allocation = allocate_pumped_hydro(scn="status2019", export=False) + # TODO status2023 leave same as status2019 scaling_factor = capacity_phes / allocation.el_capacity.sum() @@ -403,7 +624,7 @@ def home_batteries_per_scenario(scenario): battery["carrier"] = "home_battery" battery["scenario"] = scenario - if (scenario == "eGon2035") | (scenario == "status2019"): + if (scenario == "eGon2035") | ("status" in scenario): source = "NEP" else: @@ -434,11 +655,10 @@ def allocate_pv_home_batteries_to_grids(): def allocate_pumped_hydro_scn(): - if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: - allocate_pumped_hydro(scn="eGon2035") - - if "status2019" in config.settings()["egon-data"]["--scenarios"]: - allocate_pumped_hydro(scn="status2019") - - if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: - allocate_pumped_hydro_eGon100RE() + for scn in config.settings()["egon-data"]["--scenarios"]: + if scn == "eGon2035": + allocate_pumped_hydro(scn="eGon2035") + elif scn == "eGon100RE": + allocate_pumped_hydro_eGon100RE() + elif "status" in scn: + allocate_pumped_hydro_sq(scn_name=scn) diff --git a/src/egon/data/datasets/storages/pumped_hydro.py b/src/egon/data/datasets/storages/pumped_hydro.py index 68e10b888..c9f2bace3 100755 --- a/src/egon/data/datasets/storages/pumped_hydro.py +++ b/src/egon/data/datasets/storages/pumped_hydro.py @@ -11,7 +11,7 @@ from egon.data import config, db from egon.data.datasets.chp.match_nep import match_nep_chp from egon.data.datasets.chp.small_chp import assign_use_case -from egon.data.datasets.mastr import WORKING_DIR_MASTR_OLD +from egon.data.datasets.mastr import WORKING_DIR_MASTR_NEW from egon.data.datasets.power_plants import ( assign_bus_id, assign_voltage_level, @@ -49,8 +49,10 @@ def select_nep_pumped_hydro(scn): nep_ph.rename( columns={"c2035_capacity": "elec_capacity"}, inplace=True ) - elif scn == "status2019": + elif "status" in scn: # Select plants with geolocation from list of conventional power plants + year = int(scn[-4:]) + nep_ph = db.select_dataframe( f""" SELECT bnetza_id, name, carrier, postcode, capacity, city, @@ -59,7 +61,7 @@ def select_nep_pumped_hydro(scn): WHERE carrier = '{carrier}' AND capacity > 0 AND postcode != 'None' - AND commissioned < '2020'; + AND commissioned < '{year+1}'; """ ) nep_ph["elec_capacity"] = nep_ph["capacity"] @@ -91,7 +93,7 @@ def select_mastr_pumped_hydro(): # Read-in data from MaStR mastr_ph = pd.read_csv( - WORKING_DIR_MASTR_OLD / sources["mastr_storage"], + WORKING_DIR_MASTR_NEW / sources["mastr_storage"], delimiter=",", usecols=[ "Nettonennleistung", @@ -135,6 +137,22 @@ def select_mastr_pumped_hydro(): ), ) + mastr_ph = mastr_ph.set_crs(4326) + + # drop hydropower without federal state + # Obervermunterwerk II in Austria + mastr_ph = mastr_ph[~(mastr_ph["federal_state"].isnull())] + + if ( + config.settings()["egon-data"]["--dataset-boundary"] + == "Schleswig-Holstein" + ): + # Drop hydropower outside the test mode area + mastr_ph = filter_mastr_geometry(mastr_ph, federal_state="Schleswig-Holstein") + else: + # Drop hydropower outside of germany + mastr_ph = filter_mastr_geometry(mastr_ph, federal_state=None) + # Drop rows without post code and update datatype of postcode mastr_ph = mastr_ph[~mastr_ph["plz"].isnull()] mastr_ph["plz"] = mastr_ph["plz"].astype(int) @@ -142,12 +160,6 @@ def select_mastr_pumped_hydro(): # Calculate power in MW mastr_ph.loc[:, "el_capacity"] *= 1e-3 - mastr_ph = mastr_ph.set_crs(4326) - - mastr_ph = mastr_ph[~(mastr_ph["federal_state"].isnull())] - - # Drop CHP outside of Germany/ outside the test mode area - mastr_ph = filter_mastr_geometry(mastr_ph, federal_state=None) return mastr_ph @@ -355,7 +367,7 @@ def apply_voltage_level_thresholds(power_plants): # account which were defined in the eGon project. Existing entries on voltage # will be overwritten - power_plants.loc[power_plants["el_capacity"] < 0.1, "voltage_level"] = 7 + power_plants.loc[power_plants["el_capacity"] <= 0.1, "voltage_level"] = 7 power_plants.loc[power_plants["el_capacity"] > 0.1, "voltage_level"] = 6 power_plants.loc[power_plants["el_capacity"] > 0.2, "voltage_level"] = 5 power_plants.loc[power_plants["el_capacity"] > 5.5, "voltage_level"] = 4 diff --git a/src/egon/data/datasets/storages_etrago/__init__.py b/src/egon/data/datasets/storages_etrago/__init__.py index f4f8790cc..5f2793629 100644 --- a/src/egon/data/datasets/storages_etrago/__init__.py +++ b/src/egon/data/datasets/storages_etrago/__init__.py @@ -6,11 +6,9 @@ import geopandas as gpd import pandas as pd from egon.data import db, config -import egon.data.datasets.scenario_parameters.parameters as scenario_parameters from egon.data.datasets import Dataset from egon.data.datasets.scenario_parameters import ( get_sector_parameters, - EgonScenario, ) @@ -18,7 +16,7 @@ class StorageEtrago(Dataset): def __init__(self, dependencies): super().__init__( name="StorageEtrago", - version="0.0.8", + version="0.0.9", dependencies=dependencies, tasks=(insert_PHES, extendable_batteries), ) @@ -58,9 +56,9 @@ def insert_PHES(): next_bus_id = db.next_etrago_id("storage") # Add missing PHES specific information suitable for eTraGo selected from scenario_parameter table - parameters = scenario_parameters.electricity(scn)["efficiency"][ - "pumped_hydro" - ] + parameters = get_sector_parameters( + "electricity", scn + )["efficiency"]["pumped_hydro"] phes["storage_id"] = range(next_bus_id, next_bus_id + len(phes)) phes["max_hours"] = parameters["max_hours"] phes["efficiency_store"] = parameters["store"]