From 16848651e8e231e82d9bad6b56eb372a11da3ed7 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 7 Aug 2025 12:56:31 +0200 Subject: [PATCH 01/32] #1177 oemetadata v2: remove metaMetadata generator -> information is already part of the metadata specification --- src/egon/data/metadata/__init__.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index 25e267217..e65395a0e 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -37,26 +37,6 @@ def context(): } -def meta_metadata(): - """ - Meta data on metadata - - Returns - ------- - dict - OEP metadata conform metadata on metadata - """ - - return { - "metadataVersion": "OEP-1.4.1", - "metadataLicense": { - "name": "CC0-1.0", - "title": "Creative Commons Zero v1.0 Universal", - "path": ("https://creativecommons.org/publicdomain/zero/1.0/"), - }, - } - - def licenses_datenlizenz_deutschland(attribution=EGON_ATTRIBUTION): """ License information for Datenlizenz Deutschland @@ -415,7 +395,7 @@ def sources(): " developed requirements and criteria for the" " assessment of suitable sites even if their" " exploration is still at an early stage and there is" - ' little knowledge of the salinaries structures.' + " little knowledge of the salinaries structures." " Scientists at DEEP.KBB GmbH in Hanover, worked" " together with their project partners at the Federal" " Institute for Geosciences and Natural Resources and" From f9d17a3b6b1a3351e678df7fa124e0116822edc0 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 7 Aug 2025 13:18:09 +0200 Subject: [PATCH 02/32] #1177 oemetadata v2: remove version pin as all metadata will be updated to v2, old metadata is not fully supported anymore (=0.6.2", "fiona==1.9.6", "Flask-Session<0.6.0", @@ -101,7 +99,7 @@ def read(*names, **kwargs): "netcdf4", "numpy", "oedialect==0.0.8", - "oemetadata<=2.0.1", # incompatibilities with metadata 1.3 + "oemetadata", "omi", "openpyxl", "pandas>2.0.0", @@ -127,7 +125,15 @@ def read(*names, **kwargs): "fuzzywuzzy", ], extras_require={ - "dev": ["black", "flake8", "isort>=5", "pre-commit", "pytest", "tox", "jupyterlab"] + "dev": [ + "black", + "flake8", + "isort>=5", + "pre-commit", + "pytest", + "tox", + "jupyterlab", + ] # eg: # 'rst': ['docutils>=0.11'], # ':python_version=="2.6"': ['argparse'], From cc004a0ddf0cbe39a1ce82b65d76f2969879e2f3 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 7 Aug 2025 13:22:16 +0200 Subject: [PATCH 03/32] #1177 oemetadata v2: Add todo regarding open data licenses handling --- src/egon/data/metadata/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index e65395a0e..6a732aca7 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -37,6 +37,11 @@ def context(): } +# TODO @jh-RLI: All licenses should be read from SPDX data license list +# -> especially the "name" prop +# extend omi to provide list, makes sure licenses are compliant with oeplatform +# which is important for full data publication on the OEP. +# SPDX list is quite popular def licenses_datenlizenz_deutschland(attribution=EGON_ATTRIBUTION): """ License information for Datenlizenz Deutschland From f5ecba302dffb99d678966de46900401009ab134 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 7 Aug 2025 14:55:13 +0200 Subject: [PATCH 04/32] #1177 oemetadata v2: - Add settings module to manage general settings and specific input values which are important for the metadata integration - Update to latest OMI ... using local python 3.10 venv for now --- src/egon/data/metadata/__init__.py | 38 ++++++++++++++++++++---------- src/egon/data/metadata/settings.py | 6 +++++ 2 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 src/egon/data/metadata/settings.py diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index 6a732aca7..ae5a34d6d 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -1,5 +1,7 @@ from geoalchemy2 import Geometry -from omi.dialects import get_dialect +from metadata import settings +from omi.base import get_metadata_specification +from omi.validation import parse_metadata, validate_metadata from sqlalchemy import MetaData, Table from sqlalchemy.dialects.postgresql.base import ischema_names import importlib_resources @@ -8,7 +10,9 @@ from egon.data.datasets import Dataset from egon.data.db import engine -EGON_ATTRIBUTION: str = "© eGon development team" +# Easy access to oemetadata schema, template and example Dicts +# Uses the oemetadata version specified in settings module +OEMetaData = get_metadata_specification(settings.OEMETADATA_VERSION) def context(): @@ -39,10 +43,14 @@ def context(): # TODO @jh-RLI: All licenses should be read from SPDX data license list # -> especially the "name" prop +# -> see SPDX:oem licenseId:name / name:title / reference:path # extend omi to provide list, makes sure licenses are compliant with oeplatform # which is important for full data publication on the OEP. # SPDX list is quite popular -def licenses_datenlizenz_deutschland(attribution=EGON_ATTRIBUTION): +# https://github.com/OpenEnergyPlatform/omi/blob/feature-126-add-yaml-and-template-based-metadata-creation/src/omi/data/licenses.json # noqa: E501 +def licenses_datenlizenz_deutschland( + attribution=settings.EGON_ATTRIBUTION, +): """ License information for Datenlizenz Deutschland @@ -90,7 +98,7 @@ def licenses_datenlizenz_deutschland(attribution=EGON_ATTRIBUTION): } -def license_odbl(attribution=EGON_ATTRIBUTION): +def license_odbl(attribution=settings.EGON_ATTRIBUTION): """ License information for Open Data Commons Open Database License (ODbL-1.0) @@ -115,7 +123,7 @@ def license_odbl(attribution=EGON_ATTRIBUTION): } -def license_ccby(attribution=EGON_ATTRIBUTION): +def license_ccby(attribution=settings.EGON_ATTRIBUTION): """ License information for Creative Commons Attribution 4.0 International (CC-BY-4.0) @@ -140,7 +148,7 @@ def license_ccby(attribution=EGON_ATTRIBUTION): } -def license_geonutzv(attribution=EGON_ATTRIBUTION): +def license_geonutzv(attribution=settings.EGON_ATTRIBUTION): """ License information for GeoNutzV @@ -171,7 +179,7 @@ def license_geonutzv(attribution=EGON_ATTRIBUTION): } -def license_agpl(attribution=EGON_ATTRIBUTION): +def license_agpl(attribution=settings.EGON_ATTRIBUTION): """ License information for GNU Affero General Public License v3.0 @@ -201,7 +209,7 @@ def license_agpl(attribution=EGON_ATTRIBUTION): } -def license_dedl(attribution=EGON_ATTRIBUTION): +def license_dedl(attribution=settings.EGON_ATTRIBUTION): """ License information for Data licence Germany – attribution – version 2.0 @@ -254,6 +262,11 @@ def license_egon_data_odbl(): return license_odbl("© eGon development team") +# TODO @jh-RLI: Metadata schema.fields generation is a generic task and could +# be moved to OMI for others to also use it and reduce responsibilities in +# egon-data. Low prio as this here already works. +# High prio make sure this works with the oem v2 as schema is now at a +# different location. def generate_resource_fields_from_sqla_model(model): """Generate a template for the resource fields for metadata from a SQL Alchemy model. @@ -486,7 +499,7 @@ def sources(): " project eGon (https://ego-n.org/)" ), "path": "https://github.com/openego/eGon-data", - "licenses": [license_agpl(EGON_ATTRIBUTION)], + "licenses": [license_agpl(settings.EGON_ATTRIBUTION)], }, "egon-data_bundle": { "title": "Data bundle for egon-data", @@ -924,7 +937,6 @@ def contributors(authorlist): def upload_json_metadata(): """Upload json metadata into db from zenodo""" - dialect = get_dialect("oep-v1.4")() for path in importlib_resources.files(__name__).glob("*.json"): split = path.name.split(".") @@ -934,9 +946,11 @@ def upload_json_metadata(): table = split[1] with open(path, "r") as infile: - obj = dialect.parse(infile.read()) + obj = parse_metadata(infile.read()) - metadata = f"'{dialect.compile_and_render(obj)}'" + # TODO @jh-RLI: Deactivate license check for now + validate_metadata(obj, check_license=False) + metadata = f"'{obj}'" db.submit_comment(metadata, schema, table) logger.info(f"Metadata comment for {schema}.{table} stored.") diff --git a/src/egon/data/metadata/settings.py b/src/egon/data/metadata/settings.py new file mode 100644 index 000000000..8023a2f6c --- /dev/null +++ b/src/egon/data/metadata/settings.py @@ -0,0 +1,6 @@ +# The oemetadata version identifier must comply with on of: +# https://github.com/OpenEnergyPlatform/omi/blob/7e620182b0ea8eb2e437bd650b5cda3db0532307/src/omi/base.py#L17 +OEMETADATA_VERSION = "OEMetadata-2.0" + + +EGON_ATTRIBUTION: str = "© eGon development team" From 8c6c6d0b85ccfeedef4a4f74da7c31b83c56c3f6 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 7 Aug 2025 15:14:11 +0200 Subject: [PATCH 05/32] #1177: fix import location --- src/egon/data/metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index ae5a34d6d..042d61580 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -1,5 +1,4 @@ from geoalchemy2 import Geometry -from metadata import settings from omi.base import get_metadata_specification from omi.validation import parse_metadata, validate_metadata from sqlalchemy import MetaData, Table @@ -9,6 +8,7 @@ from egon.data import db, logger from egon.data.datasets import Dataset from egon.data.db import engine +from egon.data.metadata import settings # Easy access to oemetadata schema, template and example Dicts # Uses the oemetadata version specified in settings module From f147b1e55b86a6af63d252d64c1b3b7cc20184ce Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 18:17:51 +0200 Subject: [PATCH 06/32] #1177 oem v2 update: move oemetadata v141 files to results directory --- .../{ => results}/boundaries.egon_map_zensus_climate_zones.json | 0 .../{ => results}/boundaries.egon_map_zensus_mvgd_buildings.json | 0 .../{ => results}/boundaries.egon_map_zensus_weather_cell.json | 0 .../demand.egon_building_electricity_peak_loads.json | 0 .../{ => results}/demand.egon_building_heat_peak_loads.json | 0 .../demand.egon_cts_electricity_demand_building_share.json | 0 .../{ => results}/demand.egon_cts_heat_demand_building_share.json | 0 .../demand.egon_daily_heat_demand_per_climate_zone.json | 0 .../{ => results}/demand.egon_etrago_electricity_cts.json | 0 .../{ => results}/demand.egon_etrago_electricity_households.json | 0 .../data/metadata/{ => results}/demand.egon_etrago_heat_cts.json | 0 .../data/metadata/{ => results}/demand.egon_heat_idp_pool.json | 0 .../demand.egon_heat_timeseries_selected_profiles.json | 0 .../demand.egon_household_electricity_profile_in_census_cell.json | 0 .../demand.egon_household_electricity_profile_of_buildings.json | 0 .../{ => results}/demand.iee_household_load_profiles.json | 0 .../data/metadata/{ => results}/grid.egon_mv_grid_district.json | 0 .../{ => results}/openstreetmap.osm_buildings_filtered.json | 0 .../{ => results}/openstreetmap.osm_buildings_synthetic.json | 0 .../{ => results}/society.destatis_zensus_population_per_ha.json | 0 ...egon_destatis_zensus_apartment_building_population_per_ha.json | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename src/egon/data/metadata/{ => results}/boundaries.egon_map_zensus_climate_zones.json (100%) rename src/egon/data/metadata/{ => results}/boundaries.egon_map_zensus_mvgd_buildings.json (100%) rename src/egon/data/metadata/{ => results}/boundaries.egon_map_zensus_weather_cell.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_building_electricity_peak_loads.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_building_heat_peak_loads.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_cts_electricity_demand_building_share.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_cts_heat_demand_building_share.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_daily_heat_demand_per_climate_zone.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_etrago_electricity_cts.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_etrago_electricity_households.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_etrago_heat_cts.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_heat_idp_pool.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_heat_timeseries_selected_profiles.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_household_electricity_profile_in_census_cell.json (100%) rename src/egon/data/metadata/{ => results}/demand.egon_household_electricity_profile_of_buildings.json (100%) rename src/egon/data/metadata/{ => results}/demand.iee_household_load_profiles.json (100%) rename src/egon/data/metadata/{ => results}/grid.egon_mv_grid_district.json (100%) rename src/egon/data/metadata/{ => results}/openstreetmap.osm_buildings_filtered.json (100%) rename src/egon/data/metadata/{ => results}/openstreetmap.osm_buildings_synthetic.json (100%) rename src/egon/data/metadata/{ => results}/society.destatis_zensus_population_per_ha.json (100%) rename src/egon/data/metadata/{ => results}/society.egon_destatis_zensus_apartment_building_population_per_ha.json (100%) diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_climate_zones.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json similarity index 100% rename from src/egon/data/metadata/boundaries.egon_map_zensus_climate_zones.json rename to src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_mvgd_buildings.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json similarity index 100% rename from src/egon/data/metadata/boundaries.egon_map_zensus_mvgd_buildings.json rename to src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json diff --git a/src/egon/data/metadata/boundaries.egon_map_zensus_weather_cell.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json similarity index 100% rename from src/egon/data/metadata/boundaries.egon_map_zensus_weather_cell.json rename to src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json diff --git a/src/egon/data/metadata/demand.egon_building_electricity_peak_loads.json b/src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json similarity index 100% rename from src/egon/data/metadata/demand.egon_building_electricity_peak_loads.json rename to src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json diff --git a/src/egon/data/metadata/demand.egon_building_heat_peak_loads.json b/src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json similarity index 100% rename from src/egon/data/metadata/demand.egon_building_heat_peak_loads.json rename to src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json diff --git a/src/egon/data/metadata/demand.egon_cts_electricity_demand_building_share.json b/src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json similarity index 100% rename from src/egon/data/metadata/demand.egon_cts_electricity_demand_building_share.json rename to src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json diff --git a/src/egon/data/metadata/demand.egon_cts_heat_demand_building_share.json b/src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json similarity index 100% rename from src/egon/data/metadata/demand.egon_cts_heat_demand_building_share.json rename to src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json diff --git a/src/egon/data/metadata/demand.egon_daily_heat_demand_per_climate_zone.json b/src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json similarity index 100% rename from src/egon/data/metadata/demand.egon_daily_heat_demand_per_climate_zone.json rename to src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json diff --git a/src/egon/data/metadata/demand.egon_etrago_electricity_cts.json b/src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json similarity index 100% rename from src/egon/data/metadata/demand.egon_etrago_electricity_cts.json rename to src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json diff --git a/src/egon/data/metadata/demand.egon_etrago_electricity_households.json b/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json similarity index 100% rename from src/egon/data/metadata/demand.egon_etrago_electricity_households.json rename to src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json diff --git a/src/egon/data/metadata/demand.egon_etrago_heat_cts.json b/src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json similarity index 100% rename from src/egon/data/metadata/demand.egon_etrago_heat_cts.json rename to src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json diff --git a/src/egon/data/metadata/demand.egon_heat_idp_pool.json b/src/egon/data/metadata/results/demand.egon_heat_idp_pool.json similarity index 100% rename from src/egon/data/metadata/demand.egon_heat_idp_pool.json rename to src/egon/data/metadata/results/demand.egon_heat_idp_pool.json diff --git a/src/egon/data/metadata/demand.egon_heat_timeseries_selected_profiles.json b/src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json similarity index 100% rename from src/egon/data/metadata/demand.egon_heat_timeseries_selected_profiles.json rename to src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json diff --git a/src/egon/data/metadata/demand.egon_household_electricity_profile_in_census_cell.json b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json similarity index 100% rename from src/egon/data/metadata/demand.egon_household_electricity_profile_in_census_cell.json rename to src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json diff --git a/src/egon/data/metadata/demand.egon_household_electricity_profile_of_buildings.json b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json similarity index 100% rename from src/egon/data/metadata/demand.egon_household_electricity_profile_of_buildings.json rename to src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json diff --git a/src/egon/data/metadata/demand.iee_household_load_profiles.json b/src/egon/data/metadata/results/demand.iee_household_load_profiles.json similarity index 100% rename from src/egon/data/metadata/demand.iee_household_load_profiles.json rename to src/egon/data/metadata/results/demand.iee_household_load_profiles.json diff --git a/src/egon/data/metadata/grid.egon_mv_grid_district.json b/src/egon/data/metadata/results/grid.egon_mv_grid_district.json similarity index 100% rename from src/egon/data/metadata/grid.egon_mv_grid_district.json rename to src/egon/data/metadata/results/grid.egon_mv_grid_district.json diff --git a/src/egon/data/metadata/openstreetmap.osm_buildings_filtered.json b/src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json similarity index 100% rename from src/egon/data/metadata/openstreetmap.osm_buildings_filtered.json rename to src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json diff --git a/src/egon/data/metadata/openstreetmap.osm_buildings_synthetic.json b/src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json similarity index 100% rename from src/egon/data/metadata/openstreetmap.osm_buildings_synthetic.json rename to src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json diff --git a/src/egon/data/metadata/society.destatis_zensus_population_per_ha.json b/src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json similarity index 100% rename from src/egon/data/metadata/society.destatis_zensus_population_per_ha.json rename to src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json diff --git a/src/egon/data/metadata/society.egon_destatis_zensus_apartment_building_population_per_ha.json b/src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json similarity index 100% rename from src/egon/data/metadata/society.egon_destatis_zensus_apartment_building_population_per_ha.json rename to src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json From 179c241167532797af0fb2348642857760cbfdc4 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 18:21:07 +0200 Subject: [PATCH 07/32] - fix last column one, empty - Formatting - Line too long --- .../data/airflow/dags/pipeline_status_quo.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/egon/data/airflow/dags/pipeline_status_quo.py b/src/egon/data/airflow/dags/pipeline_status_quo.py index 88d9c2798..82bc72080 100755 --- a/src/egon/data/airflow/dags/pipeline_status_quo.py +++ b/src/egon/data/airflow/dags/pipeline_status_quo.py @@ -3,8 +3,8 @@ from airflow.utils.dates import days_ago import airflow -from egon.data.config import settings as egon_settings from egon.data.config import set_numexpr_threads +from egon.data.config import settings as egon_settings from egon.data.datasets import database from egon.data.datasets.ch4_prod import CH4Production from egon.data.datasets.ch4_storages import CH4Storages @@ -45,7 +45,9 @@ from egon.data.datasets.heat_etrago import HeatEtrago from egon.data.datasets.heat_etrago.hts_etrago import HtsEtragoTable from egon.data.datasets.heat_supply import HeatSupply -from egon.data.datasets.heat_supply.individual_heating import HeatPumpsStatusQuo +from egon.data.datasets.heat_supply.individual_heating import ( + HeatPumpsStatusQuo, +) from egon.data.datasets.industrial_sites import MergeIndustrialSites from egon.data.datasets.industry import IndustrialDemandCurves from egon.data.datasets.loadarea import LoadArea, OsmLanduse @@ -81,9 +83,11 @@ with airflow.DAG( f"{prefix}powerd-status-quo-processing-pipeline", description="The PoWerD Status Quo data processing DAG.", - default_args={"start_date": days_ago(1), - "email_on_failure": False, - "email":"clara.buettner@hs-flensburg.de"}, + default_args={ + "start_date": days_ago(1), + "email_on_failure": False, + "email": "clara.buettner@hs-flensburg.de", + }, template_searchpath=[ os.path.abspath( os.path.join( @@ -203,8 +207,8 @@ # TODO: What does "trans" stand for? # Calculate dynamic line rating for HV (high voltage) trans lines # dlr = Calculate_dlr( - # dependencies=[data_bundle, osmtgmod, weather_data] # , fix_subnetworks] - #) + # dependencies=[data_bundle, osmtgmod, weather_data] # , fix_subnetworks] # noqa: E501 + # ) # Map zensus grid districts zensus_mv_grid_districts = ZensusMvGridDistricts( @@ -363,7 +367,12 @@ ) # Create gas voronoi status quo create_gas_polygons_statusquo = GasAreas( - dependencies=[setup_etrago, vg250, gas_grid_insert_data, substation_voronoi] + dependencies=[ + setup_etrago, + vg250, + gas_grid_insert_data, + substation_voronoi, + ] ) # Gas abroad @@ -437,7 +446,6 @@ ] ) - # Pumped hydro and home storage units storage_units = Storages( dependencies=[ @@ -531,5 +539,3 @@ demand_curves_industry, ] ) - - From 7f8444a134cf596c1a572a2c3edf32349ee21c9b Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 21:11:49 +0200 Subject: [PATCH 08/32] =?UTF-8?q?#1177=20oem=20v2=20update:=20remove=20omi?= =?UTF-8?q?=C2=B4s=20outdated=20usage=20and=20metaMetadata=20creation=20as?= =?UTF-8?q?=20we=20can=20do=20this=20in=20a=20cleaner=20way?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/egon/data/datasets/DSM_cts_ind.py | 33 +- src/egon/data/datasets/chp/__init__.py | 89 ++-- .../district_heating_areas/__init__.py | 449 +++++++++--------- .../motorized_individual_travel/db_classes.py | 18 +- .../db_classes.py | 8 +- src/egon/data/datasets/etrago_setup.py | 33 +- src/egon/data/datasets/gas_areas.py | 63 +-- .../data/datasets/heat_demand/__init__.py | 97 ++-- .../heat_demand_timeseries/__init__.py | 131 ++--- .../data/datasets/heat_supply/__init__.py | 79 ++- src/egon/data/datasets/osm/__init__.py | 37 +- .../datasets/power_plants/mastr_db_classes.py | 28 +- .../power_plants/pv_rooftop_buildings.py | 34 +- src/egon/data/datasets/renewable_feedin.py | 136 +++--- src/egon/data/datasets/scenario_capacities.py | 277 ++++++----- .../data/datasets/storages/home_batteries.py | 19 +- src/egon/data/datasets/vg250/__init__.py | 152 +++--- src/egon/data/datasets/zensus_vg250.py | 167 ++++--- 18 files changed, 950 insertions(+), 900 deletions(-) diff --git a/src/egon/data/datasets/DSM_cts_ind.py b/src/egon/data/datasets/DSM_cts_ind.py index 1bddcfc86..4cff5b02f 100644 --- a/src/egon/data/datasets/DSM_cts_ind.py +++ b/src/egon/data/datasets/DSM_cts_ind.py @@ -11,7 +11,6 @@ import datetime import json -from omi.dialects import get_dialect from sqlalchemy import ARRAY, Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -27,8 +26,6 @@ contributors, generate_resource_fields_from_db_table, license_odbl, - meta_metadata, - meta_metadata, sources, ) @@ -368,37 +365,9 @@ def add_metadata_individual(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), - "_comment": { - "metadata": ( - "Metadata documentation and explanation (https://" - "github.com/OpenEnergyPlatform/oemetadata/blob/master/" - "metadata/v141/metadata_key_description.md)" - ), - "dates": ( - "Dates and time must follow the ISO8601 including time " - "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)" - ), - "units": "Use a space between numbers and units (100 m)", - "languages": ( - "Languages must follow the IETF (BCP47) format (en-GB, " - "en-US, de-DE)" - ), - "licenses": ( - "License name must follow the SPDX License List " - "(https://spdx.org/licenses/)" - ), - "review": ( - "Following the OEP Data Review (https://github.com/" - "OpenEnergyPlatform/data-preprocessing/wiki)" - ), - "none": "If not applicable use (none)", - }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", diff --git a/src/egon/data/datasets/chp/__init__.py b/src/egon/data/datasets/chp/__init__.py index 78f0e796a..b2b987f6f 100644 --- a/src/egon/data/datasets/chp/__init__.py +++ b/src/egon/data/datasets/chp/__init__.py @@ -1,9 +1,8 @@ """ -The central module containing all code dealing with combined heat and power -(CHP) plants. +The central module containing all code dealing with combined heat and +power (CHP) plants. """ -from pathlib import Path import datetime import json import time @@ -16,7 +15,6 @@ from sqlalchemy.orm import sessionmaker import geopandas as gpd import pandas as pd -import pypsa from egon.data import config, db from egon.data.datasets import Dataset, wrapped_partial @@ -29,8 +27,8 @@ select_target, ) from egon.data.datasets.mastr import ( - WORKING_DIR_MASTR_OLD, WORKING_DIR_MASTR_NEW, + WORKING_DIR_MASTR_OLD, ) from egon.data.datasets.power_plants import ( assign_bus_id, @@ -38,15 +36,16 @@ filter_mastr_geometry, scale_prox2now, ) +from egon.data.datasets.pypsaeur import read_network from egon.data.metadata import ( context, generate_resource_fields_from_sqla_model, license_egon_data_odbl, - meta_metadata, sources, ) -from egon.data.datasets.pypsaeur import read_network +# import pypsa + Base = declarative_base() @@ -168,10 +167,12 @@ def metadata(): "primaryKey": ["index"], "foreignKeys": [], }, - "dialect": {"delimiter": None, "decimalSeparator": "."}, + "dialect": { + "delimiter": None, + "decimalSeparator": ".", + }, } ], - "metaMetadata": meta_metadata(), } # Add metadata as a comment to the table @@ -330,7 +331,7 @@ def assign_heat_bus(): district_heating=row.district_heating, voltage_level=row.voltage_level, scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + geom=(f"SRID=4326;POINT({row.geom.x} {row.geom.y})"), ) else: entry = EgonChp( @@ -345,7 +346,7 @@ def assign_heat_bus(): district_heating=row.district_heating, voltage_level=row.voltage_level, scenario=scenario, - geom=f"SRID=4326;POINT({row.geom.x} {row.geom.y})", + geom=(f"SRID=4326;POINT({row.geom.x} {row.geom.y})"), ) session.add(entry) session.commit() @@ -366,7 +367,7 @@ def insert_biomass_chp(scenario): """ cfg = config.datasets()["chp_location"] - # import target values from NEP 2021, scneario C 2035 + # import target values from NEP 2021, scenario C 2035 target = select_target("biomass", scenario) # import data for MaStR @@ -387,7 +388,7 @@ def insert_biomass_chp(scenario): ) ] - # Scaling will be done per federal state in case of eGon2035 scenario. + # Scaling per federal state in case of eGon2035 scenario. if scenario == "eGon2035": level = "federal_state" else: @@ -424,7 +425,7 @@ def insert_biomass_chp(scenario): district_heating=row.district_heating, electrical_bus_id=row.bus_id, voltage_level=row.voltage_level, - geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", + geom=(f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})"), ) session.add(entry) session.commit() @@ -522,7 +523,8 @@ def insert_chp_statusquo(scn="status2019"): print( f""" - CHPs with a total installed electrical capacity of {dropped_capacity} kW are dropped + CHPs with a total installed electrical + capacity of {dropped_capacity} kW are dropped because of missing or wrong location data """ ) @@ -563,7 +565,7 @@ def insert_chp_statusquo(scn="status2019"): electrical_bus_id=row.bus_id, ch4_bus_id=row.gas_bus_id, voltage_level=row.voltage_level, - geom=f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})", + geom=(f"SRID=4326;POINT({row.Laengengrad} {row.Breitengrad})"), ) session.add(entry) session.commit() @@ -675,7 +677,8 @@ def extension_SH(): def insert_chp_egon100re(): - """Insert CHP plants for eGon100RE considering results from pypsa-eur-sec + """Insert CHP plants for eGon100RE considering results from + pypsa-eur-sec Returns ------- @@ -759,12 +762,16 @@ def insert_chp_egon100re(): if "status2019" in config.settings()["egon-data"]["--scenarios"]: insert_per_scenario.add( - wrapped_partial(insert_chp_statusquo, scn="status2019", postfix="_2019") + wrapped_partial( + insert_chp_statusquo, scn="status2019", postfix="_2019" + ) ) if "status2023" in config.settings()["egon-data"]["--scenarios"]: insert_per_scenario.add( - wrapped_partial(insert_chp_statusquo, scn="status2023", postfix="_2023") + wrapped_partial( + insert_chp_statusquo, scn="status2023", postfix="_2023" + ) ) if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: @@ -814,28 +821,40 @@ class Chp(Dataset): """ Extract combined heat and power plants for each scenario - This dataset creates combined heat and power (CHP) plants for each scenario and defines their use case. - The method bases on existing CHP plants from Marktstammdatenregister. For the eGon2035 scenario, - a list of CHP plans from the grid operator is used for new largescale CHP plants. CHP < 10MW are - randomly distributed. - Depending on the distance to a district heating grid, it is decided if the CHP is used to - supply a district heating grid or used by an industrial site. + This dataset creates combined heat and power (CHP) plants for each + scenario and defines their use case. The method bases on existing CHP + plants from Marktstammdatenregister. For the eGon2035 scenario, a list of + CHP plans from the grid operator is used for new largescale CHP plants. + CHP < 10MW are randomly distributed. Depending on the distance to a + district heating grid, it is decided if the CHP is used to supply a + district heating grid or used by an industrial site. *Dependencies* - * :py:class:`GasAreaseGon100RE ` - * :py:class:`GasAreaseGon2035 ` - * :py:class:`DistrictHeatingAreas ` - * :py:class:`IndustrialDemandCurves ` - * :py:class:`OsmLanduse ` - * :py:func:`download_mastr_data ` - * :py:func:`define_mv_grid_districts ` - * :py:class:`ScenarioCapacities ` + * :py:class:`GasAreaseGon100RE + ` + * :py:class:`GasAreaseGon2035 + ` + * :py:class:`DistrictHeatingAreas + ` + * :py:class:`IndustrialDemandCurves + ` + * :py:class:`OsmLanduse + ` + * :py:func:`download_mastr_data + ` + * :py:func:`define_mv_grid_districts + ` + * :py:class:`ScenarioCapacities + ` *Resulting tables* - * :py:class:`supply.egon_chp_plants ` is created and filled - * :py:class:`supply.egon_mastr_conventional_without_chp ` is created and filled + * :py:class:`supply.egon_chp_plants + ` is created and filled + * :py:class:`supply.egon_mastr_conventional_without_chp + ` is created + and filled """ diff --git a/src/egon/data/datasets/district_heating_areas/__init__.py b/src/egon/data/datasets/district_heating_areas/__init__.py index b423f4541..e0c2c0283 100644 --- a/src/egon/data/datasets/district_heating_areas/__init__.py +++ b/src/egon/data/datasets/district_heating_areas/__init__.py @@ -5,73 +5,77 @@ # license text - to be added. """ -Central module containing all code creating with district heating areas. +Central module with all code for creating district heating areas. -This module obtains the information from the census tables and the heat demand -densities, demarcates so the current and future district heating areas. In the -end it saves them in the database. +This module reads data from census tables and heat-demand densities and +then demarcates current and future district-heating areas. In the end it +saves them in the database. """ +import datetime +import json import os -from egon.data import config, db -from egon.data.datasets.scenario_parameters import ( - get_sector_parameters, - EgonScenario, -) - -import pandas as pd -import geopandas as gpd -from shapely.geometry.multipolygon import MultiPolygon -from shapely.geometry.polygon import Polygon -from matplotlib import pyplot as plt -from egon.data.datasets.district_heating_areas.plot import ( - plot_heat_density_sorted, -) # for metadata creation import time -import datetime -from egon.data.metadata import ( - context, - meta_metadata, - license_ccby, - sources, -) -import json -# import time +from geoalchemy2.types import Geometry +from matplotlib import pyplot as plt +from shapely.geometry.multipolygon import MultiPolygon +from shapely.geometry.polygon import Polygon # packages for ORM class definition -from sqlalchemy import Column, String, Integer, Sequence, Float, ForeignKey +from sqlalchemy import Column, Float, ForeignKey, Integer, Sequence, String from sqlalchemy.ext.declarative import declarative_base -from geoalchemy2.types import Geometry +import geopandas as gpd +import pandas as pd +from egon.data import config, db from egon.data.datasets import Dataset +from egon.data.datasets.district_heating_areas.plot import ( + plot_heat_density_sorted, +) +from egon.data.datasets.scenario_parameters import ( + EgonScenario, + get_sector_parameters, +) +from egon.data.metadata import context, license_ccby, sources + +# import time # class for airflow task management (and version control) class DistrictHeatingAreas(Dataset): """ - Create district heating grids for all scenarios + Create district heating grids for all scenarios. - This dataset creates district heating grids for each scenario based on a defined - district heating share, annual heat demands calcultaed within - :py:class:`HeatDemandImport ` - and information on existing heating grids from census :py:class:`ZensusMiscellaneous ` + This dataset creates district-heating grids for each scenario based on a + defined district-heating share and annual heat demands, calculated in + :py:class:`HeatDemandImport + `, and information on + existing heating grids from census + :py:class:`ZensusMiscellaneous + `. - First the tables are created using :py:func:`create_tables`. Afterwards, the - distict heating grids for each scenario are created and inserted into the database - by applying the function :py:func:`district_heating_areas` + First the tables are created using :py:func:`create_tables`. Afterwards, + the distinct heating grids for each scenario are created and inserted into + the database by applying :py:func:`district_heating_areas`. *Dependencies* - * :py:class:`HeatDemandImport ` - * :py:class:`ZensusMiscellaneous ` - * :py:class:`ScenarioParameters ` + * :py:class:`HeatDemandImport + ` + * :py:class:`ZensusMiscellaneous + ` + * :py:class:`ScenarioParameters + ` *Resulting tables* - * :py:class:`demand.egon_map_zensus_district_heating_areas ` + * :py:class:`demand.egon_map_zensus_district_heating_areas + ` + is created and filled + * :py:class:`demand.egon_district_heating_areas + ` is created and filled - * :py:class:`demand.egon_district_heating_areas ` is created and filled """ @@ -128,7 +132,7 @@ class EgonDistrictHeatingAreas(Base): def create_tables(): - """Create tables for district heating areas + """Create tables for district heating areas. Returns ------- @@ -140,34 +144,43 @@ def create_tables(): # Drop tables db.execute_sql( - """DROP TABLE IF EXISTS - demand.egon_district_heating_areas CASCADE;""" + """ + DROP TABLE IF EXISTS demand.egon_district_heating_areas CASCADE; + """ ) db.execute_sql( - """DROP TABLE IF EXISTS - demand.egon_map_zensus_district_heating_areas CASCADE;""" + """ + DROP TABLE IF EXISTS + demand.egon_map_zensus_district_heating_areas CASCADE; + """ ) db.execute_sql( - """DROP TABLE IF EXISTS - demand.district_heating_areas CASCADE;""" + """ + DROP TABLE IF EXISTS demand.district_heating_areas CASCADE; + """ ) db.execute_sql( - """DROP TABLE IF EXISTS - demand.map_zensus_district_heating_areas CASCADE;""" + """ + DROP TABLE IF EXISTS + demand.map_zensus_district_heating_areas CASCADE; + """ ) # Drop sequences db.execute_sql( - """DROP SEQUENCE IF EXISTS - demand.district_heating_areas_seq CASCADE;""" + """ + DROP SEQUENCE IF EXISTS demand.district_heating_areas_seq CASCADE; + """ ) db.execute_sql( - """DROP SEQUENCE IF EXISTS - demand.egon_map_zensus_district_heating_areas_seq CASCADE;""" + """ + DROP SEQUENCE IF EXISTS + demand.egon_map_zensus_district_heating_areas_seq CASCADE; + """ ) engine = db.engine() @@ -187,10 +200,10 @@ def load_census_data(minimum_connection_rate=0.3): The census apartment and the census building table contains information about the heating type. The information are loaded from the apartment - table, because they might be more useful when it comes to the estimation of - the connection rates. Only cells with a connection rate equal to or larger - than 30% (based on the census apartment data) are included in the returned - district_heat GeoDataFrame. + table, because they might be more useful when it comes to the estimation + of the connection rates. Only cells with a connection rate equal to or + larger than 30% (based on the census apartment data) are included in the + returned district_heat GeoDataFrame. Parameters ---------- @@ -342,8 +355,8 @@ def area_grouping( Group polygons which are close to each other. This function creates buffers around the given cell polygons (called - "raw_polygons") and unions the intersecting buffer polygons. Afterwards, it - unions the cell polygons which are within one unified buffer polygon. + "raw_polygons") and unions the intersecting buffer polygons. Afterwards, + it unions the cell polygons which are within one unified buffer polygon. If requested, the cells being in areas fulfilling the minimum heat demand criterium are selected. @@ -474,55 +487,53 @@ def area_grouping( def district_heating_areas(scenario_name, plotting=False): """ - Create scenario specific district heating areas considering on census data. + Create scenario specific district heating areas using census data. This function loads the district heating share from the scenario table and - demarcate the scenario specific district heating areas. To do so it - uses the census data on flats currently supplied with district heat, which - are supplied selected first, if the estimated connection rate >= 30%. + demarcates the scenario specific district-heating areas. It uses census + data on flats currently supplied with district heat, which are selected + first if the estimated connection rate is >= 30%. All scenarios use the Prospective Supply Districts (PSDs) made for the - eGon2035 scenario to identify the areas where additional district heating - supply is feasible. One PSD dataset is to defined which is constant over - the years to allow comparisons. Moreover, it is - assumed that the eGon2035 PSD dataset is suitable, even though the heat - demands will continue to decrease from 2035 to 2050, because district - heating systems will be to planned and built before 2050, to exist in 2050. - - It is assumed that the connection rate in cells with district heating will - be a 100%. That is because later in project the number of buildings per - cell will be used and connection rates not being 0 or 100% will create - buildings which are not fully supplied by one technology. + eGon2035 scenario to identify additional feasible district-heating areas. + One PSD dataset is defined which is constant over the years to allow + comparisons. It is assumed that the eGon2035 PSD dataset is suitable, + even though heat demands continue to decrease from 2035 to 2050, because + district heating systems must be planned and built before 2050. + + It is further assumed that the connection rate in cells with district + heating will be 100%. Later in the project, the number of buildings per + cell will be used and non-binary connection rates would create buildings + not fully supplied by one technology. The cell polygons which carry information (like heat demand etc.) are - grouped into areas which are close to each other. - Only cells with a minimum heat demand density (e.g. >100 GJ/(ha a)) are - considered when creating PSDs. Therefore, the select_high_heat_demands() - function is used. There is minimum heat demand per PSDs to achieve a - certain size. - While the grouping buffer for the creation of Prospective Supply Districts - (PSDs) is 200m as in the sEEnergies project, the buffer for grouping census - data cell with an estimated connection rate >= 30% is 500m. - The 500m buffer is also used when the resulting district heating areas are - grouped, because they are built upon the existing district heating systems. - - To reduce the final number of district heating areas having the size of - only one hectare, the minimum heat demand critrium is also applied when - grouping the cells with census data on district heat. - - To avoid huge district heating areas, as they appear in the Ruhr area, - district heating areas with an annual demand > 4,000,000 MWh are split - by nuts3 boundaries. This as set as maximum_total_demand of the - area_grouping function. + grouped into areas which are close to each other. Only cells with a + minimum heat demand density (e.g. >100 GJ/(ha a)) are considered when + creating PSDs via :func:`select_high_heat_demands`. There is a minimum + total heat demand per PSD to achieve a certain size. + + While the grouping buffer for the creation of PSDs is 200 m (as in + sEEnergies), the buffer for grouping census data (connection rate >= 30%) + is 500 m. The 500 m buffer is also used when the resulting district + heating areas are grouped, because they are built upon existing DH + systems. + + To reduce the number of tiny DH areas (single hectare), the minimum heat + demand criterion is applied when grouping the cells with census data on + district heat. + + To avoid huge DH areas (e.g. in the Ruhr area), DH areas with an annual + demand > 4,000,000 MWh are split by NUTS3 boundaries (see + ``maximum_total_demand`` in :func:`area_grouping`). Parameters ---------- scenario_name: str - name of scenario to be studies + name of scenario to be studied plotting: boolean - if True, figure showing the heat demand density curve will be created + if True, a figure of the heat demand density curve will be created Returns @@ -538,8 +549,7 @@ def district_heating_areas(scenario_name, plotting=False): Do "area_grouping(load_census_data()[0])" only once, not for all scenarios. - Check the applied buffer distances, find a justification for the - documentation + Check the applied buffer distances and justify them. """ @@ -553,9 +563,8 @@ def district_heating_areas(scenario_name, plotting=False): minimum_connection_rate = 0.3 - # Adjust minimum connection rate for status2019, and other statusquo scn - # otherwise the existing district heating grids would have too much demand - # if scenario_name == "status2019": + # Adjust min connection rate for statusquo scenarios. Otherwise the + # existing district-heating grids would have too much demand. if "status" in scenario_name: minimum_connection_rate = 0.6 @@ -563,8 +572,8 @@ def district_heating_areas(scenario_name, plotting=False): heat_demand_cells = load_heat_demands(scenario_name) # Firstly, supply the cells which already have district heating according - # to 2011 Census data and which are within likely dh areas (created - # by the area grouping function), load only the first returned result: [0] + # to 2011 Census data and which are within likely DH areas (created by the + # area grouping function). Load only the first returned result: [0] min_hd_census = 10000 / 3.6 # in MWh census_plus_heat_demand = load_census_data( @@ -573,7 +582,8 @@ def district_heating_areas(scenario_name, plotting=False): census_plus_heat_demand[ "residential_and_service_demand" ] = heat_demand_cells.loc[ - census_plus_heat_demand.index.values, "residential_and_service_demand" + census_plus_heat_demand.index.values, + "residential_and_service_demand", ] cells = area_grouping( @@ -592,12 +602,11 @@ def district_heating_areas(scenario_name, plotting=False): assert ( diff > 0 - ), """The chosen district heating share in combination with the heat - demand reduction leads to an amount of district heat which is - lower than the current one. This case is not implemented yet.""" + ), """The chosen district-heating share with the heat-demand reduction + leads to an amount of district heat which is lower than the current + one. This case is not implemented yet.""" - # Secondly, supply the cells with the highest heat demand not having - # district heating yet + # Secondly, supply the cells with the highest heat demand not having DH # ASSUMPTION HERE: 2035 HD defined the PSDs min_hd = 10000 / 3.6 PSDs = area_grouping( @@ -606,9 +615,7 @@ def district_heating_areas(scenario_name, plotting=False): minimum_total_demand=min_hd, ) - # PSDs.groupby("area_id").size().sort_values() - - # select all cells not already suppied with district heat + # select all cells not already supplied with district heat new_areas = heat_demand_cells[~heat_demand_cells.index.isin(cells.index)] # sort by heat demand density new_areas = new_areas[new_areas.index.isin(PSDs.index)].sort_values( @@ -617,26 +624,20 @@ def district_heating_areas(scenario_name, plotting=False): new_areas[ "Cumulative_Sum" ] = new_areas.residential_and_service_demand.cumsum() - # select cells to be supplied with district heating until district - # heating share is reached + # select cells to be supplied with DH until DH share is reached new_areas = new_areas[new_areas["Cumulative_Sum"] <= diff] print( - f"""Minimum heat demand density for cells with new district heat - supply in scenario {scenario_name} is + f"""Minimum heat-demand density for cells with new DH supply in + scenario {scenario_name} is {new_areas.residential_and_service_demand.tail(1).values[0]} MWh / (ha a).""" ) print( - f"""Number of cells with new district heat supply in scenario - {scenario_name} is {len(new_areas)}.""" + f"""Number of cells with new DH supply in scenario {scenario_name} + is {len(new_areas)}.""" ) - # check = gpd.GeoDataFrame( - # cells[['residential_and_service_demand', 'geom_polygon']].append( - # new_areas[['residential_and_service_demand', 'geom_polygon']]), - # geometry='geom_polygon') - # group the resulting scenario specific district heating areas scenario_dh_area = area_grouping( pd.concat( @@ -651,12 +652,9 @@ def district_heating_areas(scenario_name, plotting=False): scenario_dh_area.loc[:, "zensus_population_id"] = scenario_dh_area.loc[ :, "zensus_population_id" ].astype(int) - # scenario_dh_area.plot(column = "area_id") scenario_dh_area.groupby("area_id").size().sort_values() scenario_dh_area.residential_and_service_demand.sum() - # scenario_dh_area.sort_index() - # cells[cells.index==1416974] # store the results in the database scenario_dh_area["scenario"] = scenario_name @@ -682,25 +680,13 @@ def district_heating_areas(scenario_name, plotting=False): MultiPolygon([feature]) if type(feature) == Polygon else feature for feature in areas_dissolved["geom_polygon"] ] - # type(areas_dissolved["geom"][0]) - # print(type(areas_dissolved)) - # print(areas_dissolved.head()) if len(areas_dissolved[areas_dissolved.area == 100 * 100]) > 0: print( - f"""Number of district heating areas of single zensus cells: - {len(areas_dissolved[areas_dissolved.area == 100*100]) - }""" + f"""Number of district-heating areas of single zensus cells: + {len(areas_dissolved[areas_dissolved.area == 100*100])} + """ ) - # print(f"""District heating areas ids of single zensus cells in - # district heating areas: - # {areas_dissolved[areas_dissolved.area == 100*100].index.values - # }""") - # print(f"""Zensus_population_ids of single zensus cells - # in district heating areas: - # {scenario_dh_area[scenario_dh_area.area_id.isin( - # areas_dissolved[areas_dissolved.area == 100*100].index.values - # )].index.values}""") db.execute_sql( f"""DELETE FROM demand.egon_district_heating_areas @@ -714,8 +700,7 @@ def district_heating_areas(scenario_name, plotting=False): con=db.engine(), if_exists="append", ) - # Alternative: - # join.groupby("columnname").demand.sum() + # add the sorted heat demand density curve no_district_heating = heat_demand_cells[ ~heat_demand_cells.index.isin(scenario_dh_area.index) @@ -745,14 +730,13 @@ def district_heating_areas(scenario_name, plotting=False): def add_metadata(): """ - Writes metadata JSON string into table comment. + Write metadata JSON strings into table comments. TODO ---- - - Meta data must be check and adjusted to the egon_data standard: - - Add context - - authors and institutions + Meta data must be checked and adjusted to the egon_data standard: + - Add context + - authors and institutions """ @@ -764,13 +748,19 @@ def add_metadata(): # Metadata creation for district heating areas (polygons) meta = { "name": "district_heating_areas_metadata", - "title": "eGo^n scenario-specific future district heating areas", - "description": "Modelled future district heating areas for " - "the supply of residential and service-sector heat demands", + "title": ("eGo^n scenario-specific future district heating areas"), + "description": ( + "Modelled future district heating areas for the supply of " + "residential and service-sector heat demands" + ), "language": ["EN"], "publicationDate": datetime.date.today().isoformat(), "context": context(), - "spatial": {"location": "", "extent": "Germany", "resolution": ""}, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "", + }, "sources": [ sources()["peta"], sources()["egon-data"], @@ -812,7 +802,9 @@ def add_metadata(): }, { "name": "geom_polygon", - "description": "geo information of multipolygons", + "description": ( + "geo information of multipolygons" + ), "type": "geometry(MULTIPOLYGON, 3035)", "unit": "none", }, @@ -822,13 +814,18 @@ def add_metadata(): { "fields": ["scenario"], "reference": { - "resource": "scenario.egon_scenario_parameters", + "resource": ( + "scenario.egon_scenario_parameters" + ), "fields": ["name"], }, } ], }, - "dialect": {"delimiter": "none", "decimalSeparator": "."}, + "dialect": { + "delimiter": "none", + "decimalSeparator": ".", + }, } ], "licenses": license_district_heating_areas, @@ -848,7 +845,6 @@ def add_metadata(): "comment": "Updated metadata", }, ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" @@ -857,14 +853,22 @@ def add_metadata(): # Metadata creation for "id mapping" table meta = { "name": "map_zensus_district_heating_areas_metadata", - "title": "district heating area ids assigned to zensus_population_ids", - "description": "Ids of scenario specific future district heating areas" - " for supply of residential and service-sector heat demands" - " assigned to zensus_population_ids", + "title": ( + "district heating area ids assigned to zensus_population_ids" + ), + "description": ( + "Ids of scenario specific future district heating areas for " + "supply of residential and service-sector heat demands " + "assigned to zensus_population_ids" + ), "language": ["EN"], "publicationDate": datetime.date.today().isoformat(), "context": context(), - "spatial": {"location": "", "extent": "Germany", "resolution": ""}, + "spatial": { + "location": "", + "extent": "Germany", + "resolution": "", + }, "sources": [ sources()["peta"], sources()["egon-data"], @@ -905,20 +909,28 @@ def add_metadata(): { "fields": ["zensus_population_id"], "reference": { - "resource": "society.destatis_zensus_population_per_ha", + "resource": ( + "society.destatis_zensus_" + "population_per_ha" + ), "fields": ["id"], }, }, { "fields": ["scenario"], "reference": { - "resource": "scenario.egon_scenario_parameters", + "resource": ( + "scenario.egon_scenario_parameters" + ), "fields": ["name"], }, }, ], }, - "dialect": {"delimiter": "none", "decimalSeparator": "."}, + "dialect": { + "delimiter": "none", + "decimalSeparator": ".", + }, } ], "licenses": license_district_heating_areas, @@ -938,7 +950,6 @@ def add_metadata(): "comment": "Updated metadata", }, ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" @@ -956,10 +967,9 @@ def study_prospective_district_heating_areas(): This optional function executes the functions so that you can study the heat demand density data of different scenarios and compare them and the resulting Prospective Supply Districts (PSDs) for district heating. This - functions saves local shapefiles, because these data are not written into - database. Moreover, heat density curves are drawn. - This function is tailor-made and includes the scenarios eGon2035 and - eGon100RE. + function saves local shapefiles, because these data are not written into + the database. Moreover, heat-density curves are drawn. This function is + tailor-made and includes the scenarios eGon2035 and eGon100RE. Parameters ---------- @@ -987,14 +997,13 @@ def study_prospective_district_heating_areas(): # load the total heat demand by census cell (residential plus service) # HD_2015 = load_heat_demands('eGon2015') - # status quo heat demand data are part of the regluar database content - # to get them, line 463 ("if not '2015' in source.stem:") has to be - # deleted from - # importing/heat_demand_data/__init__.py - # and an empty row has to be added to scenario table: + # status quo heat demand data are part of the regular database content. + # To get them, remove in importing/heat_demand_data/__init__.py: + # if not '2015' in source.stem: + # and add to the scenario table: # INSERT INTO scenario.egon_scenario_parameters (name) # VALUES ('eGon2015'); - # because egon2015 is not part of the regular EgonScenario table! + # because eGon2015 is not in the regular EgonScenario table! HD_2035 = load_heat_demands("eGon2035") HD_2050 = load_heat_demands("eGon100RE") @@ -1004,20 +1013,18 @@ def study_prospective_district_heating_areas(): HD_2050_above_100GJ = select_high_heat_demands(HD_2050) # PSDs - # grouping cells applying the 201m distance buffer, including heat demand - # aggregation + # grouping cells applying the 200 m distance buffer, including + # heat-demand aggregation # after decision for one year/scenario (here 2035), in the pipeline PSDs - # are only calculeated for the one selected year/scenario; - # here you can see all years/scenarios: + # are only calculated for the selected year/scenario; below you can see + # all years/scenarios: # PSD_2015_201m = area_grouping(HD_2015_above_100GJ, distance=200, - # minimum_total_demand=(10000/3.6) - # ).dissolve('area_id', aggfunc='sum') + # minimum_total_demand=(10000/3.6)) \ + # .dissolve('area_id', aggfunc='sum') # PSD_2015_201m.to_file(results_path+"PSDs_2015based.shp") PSD_2035_201m = area_grouping( HD_2035_above_100GJ, distance=200, minimum_total_demand=(10000 / 3.6) ).dissolve("area_id", aggfunc="sum") - # HD_2035.to_file(results_path+"HD_2035.shp") - # HD_2035_above_100GJ.to_file(results_path+"HD_2035_above_100GJ.shp") PSD_2035_201m.to_file(results_path + "PSDs_2035based.shp") PSD_2050_201m = area_grouping( HD_2050_above_100GJ, distance=200, minimum_total_demand=(10000 / 3.6) @@ -1026,15 +1033,10 @@ def study_prospective_district_heating_areas(): # plotting all cells - not considering census data # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.plot.html - # https://www.earthdatascience.org/courses/scientists-guide-to-plotting-data-in-python/plot-with-matplotlib/introduction-to-matplotlib-plots/customize-plot-colors-labels-matplotlib/ + # https://www.earthdatascience.org/courses/scientists-guide-to-plotting- + # data-in-python/plot-with-matplotlib/introduction-to-matplotlib-plots/ + # customize-plot-colors-labels-matplotlib/ fig, ax = plt.subplots(1, 1) - # add the sorted heat demand densities - # HD_2015 = HD_2015.sort_values('residential_and_service_demand', - # ascending=False).reset_index() - # HD_2015["Cumulative_Sum"] = (HD_2015.residential_and_service_demand. - # cumsum()) / 1000000 - # ax.plot(HD_2015.Cumulative_Sum, - # HD_2015.residential_and_service_demand, label='eGon2015') HD_2035 = HD_2035.sort_values( "residential_and_service_demand", ascending=False @@ -1065,9 +1067,11 @@ def study_prospective_district_heating_areas(): heat_parameters = get_sector_parameters("heat", "eGon2035") district_heating_share_2035 = heat_parameters["DE_district_heating_share"] plt.axvline( - x=HD_2035.residential_and_service_demand.sum() - / 1000000 - * district_heating_share_2035, + x=( + HD_2035.residential_and_service_demand.sum() + / 1000000 + * district_heating_share_2035 + ), ls=":", lw=0.5, label="72TWh DH in 2035 in Germany => 14% DH", @@ -1076,9 +1080,11 @@ def study_prospective_district_heating_areas(): heat_parameters = get_sector_parameters("heat", "eGon100RE") district_heating_share_100RE = heat_parameters["DE_district_heating_share"] plt.axvline( - x=HD_2050.residential_and_service_demand.sum() - / 1000000 - * district_heating_share_100RE, + x=( + HD_2050.residential_and_service_demand.sum() + / 1000000 + * district_heating_share_100RE + ), ls="-.", lw=0.5, label="75TWh DH in 100RE in Germany => 19% DH", @@ -1088,7 +1094,8 @@ def study_prospective_district_heating_areas(): # axes meet in (0/0) ax.margins(x=0, y=0) # default is 0.05 # axis style - # https://matplotlib.org/stable/gallery/ticks_and_spines/centered_spines_with_arrows.html + # https://matplotlib.org/stable/gallery/ticks_and_spines/ + # centered_spines_with_arrows.html # Hide the right and top spines ax.spines["right"].set_visible(False) ax.spines["top"].set_visible(False) @@ -1107,23 +1114,21 @@ def study_prospective_district_heating_areas(): def demarcation(plotting=True): """ - Load scenario specific district heating areas with metadata into database. - - This function executes the functions that identifies the areas which will - be supplied with district heat in the two eGo^n scenarios. The creation of - heat demand density curve figures is optional. So is also the export of - scenario specific Prospective Supply Districts for district heating (PSDs) - as shapefiles including the creation of a figure showing the comparison - of sorted heat demand densities. - - The method was executed for 2015, 2035 and 2050 to find out which - scenario year defines the PSDs. The year 2035 was selected and - the function was adjusted accordingly. - If you need the 2015 scenario heat demand data, please have a look at - the heat demand script commit 270bea50332016447e869f69d51e96113073b8a0, + Load scenario specific district heating areas with metadata into DB. + + This function runs the workflow that identifies the areas supplied with + district heat in the two eGo^n scenarios. Creating the heat-demand + density-curve figures is optional. So is the export of scenario-specific + Prospective Supply Districts as shapefiles, including a comparison figure + of sorted heat-demand densities. + + The method was executed for 2015, 2035 and 2050 to find out which year + defines the PSDs. The year 2035 was selected and the function was + adjusted accordingly. If you need the 2015 scenario heat demand data, + check the heat demand script commit + 270bea50332016447e869f69d51e96113073b8a0, where the 2015 scenario was deactivated. You can study the 2015 PSDs in - the study_prospective_district_heating_areas function after - un-commenting some lines. + :func:`study_prospective_district_heating_areas` after uncommenting lines. Parameters ---------- @@ -1140,14 +1145,13 @@ def demarcation(plotting=True): TODO ---- - Create diagrams/curves, make better curves with matplotlib + Create diagrams/curves, make better curves with matplotlib. - Make PSD and DH system statistics - Check if you need the current / future number of DH - supplied flats and the total number of flats to calculate the - connection rate + Make PSD and DH system statistics. Check if you need the current / + future number of DH-supplied flats and the total number of flats to + calculate the connection rate. - Add datasets to datasets configuration + Add datasets to datasets configuration. """ @@ -1166,8 +1170,7 @@ def demarcation(plotting=True): if plotting: plot_heat_density_sorted(heat_density_per_scenario) - # if you want to study/export the Prospective Supply Districts (PSDs) - # for all scenarios + # If you want to study/export the PSDs for all scenarios: # study_prospective_district_heating_areas() add_metadata() diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py b/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py index dec64dd0f..93c8ce574 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel/db_classes.py @@ -4,7 +4,6 @@ import datetime import json -from omi.dialects import get_dialect from sqlalchemy import ( Boolean, Column, @@ -31,8 +30,6 @@ license_agpl, license_ccby, license_odbl, - meta_metadata, - meta_metadata, sources, ) @@ -382,7 +379,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github." @@ -410,9 +406,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", @@ -504,7 +498,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github." @@ -532,9 +525,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", @@ -626,7 +617,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github." @@ -654,9 +644,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", diff --git a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py index 860f1a513..f63434c61 100644 --- a/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py +++ b/src/egon/data/datasets/emobility/motorized_individual_travel_charging_infrastructure/db_classes.py @@ -6,7 +6,6 @@ import json from geoalchemy2 import Geometry -from omi.dialects import get_dialect from sqlalchemy import Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base @@ -16,8 +15,6 @@ contributors, generate_resource_fields_from_db_table, license_odbl, - meta_metadata, - meta_metadata, ) Base = declarative_base() @@ -124,7 +121,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github.com/Op" @@ -152,9 +148,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", diff --git a/src/egon/data/datasets/etrago_setup.py b/src/egon/data/datasets/etrago_setup.py index 604923c63..c06931914 100755 --- a/src/egon/data/datasets/etrago_setup.py +++ b/src/egon/data/datasets/etrago_setup.py @@ -28,8 +28,7 @@ context, contributors, license_egon_data_odbl, - meta_metadata, - sources + sources, ) Base = declarative_base() @@ -120,13 +119,9 @@ def get_meta( "primaryKey": ["scn_name", component.lower() + "_id"], "foreignKeys": [], }, - "dialect": { - "delimiter": "", - "decimalSeparator": "" - }, + "dialect": {"delimiter": "", "decimalSeparator": ""}, } ], - "metaMetadata": meta_metadata(), } # Create json dump @@ -162,7 +157,6 @@ class EgonPfHvBus(Base): contributor_list[3]["comment"] = "Added CH4 sector buses" contributor_list[4]["comment"] = "Added H2 sector buses" - __tablename__ = "egon_etrago_bus" __table_args__ = { "schema": "grid", @@ -235,7 +229,6 @@ class EgonPfHvGenerator(Base): contributor_list[3]["comment"] = "Added gas feedin generators" contributor_list[4]["comment"] = "Added pv ground mounted" - __tablename__ = "egon_etrago_generator" __table_args__ = { "schema": "grid", @@ -280,7 +273,8 @@ class EgonPfHvGenerator(Base): ramp_limit_shut_down = Column(Float(53), server_default="1.") e_nom_max = Column( Float(53), server_default="inf" - ) # [MWh(/y)] Value to be used in eTraGo to set constraint for the production over the year + ) # [MWh(/y)] Value to be used in eTraGo to set constraint for + # the production over the year class EgonPfHvGeneratorTimeseries(Base): @@ -295,7 +289,6 @@ class EgonPfHvGeneratorTimeseries(Base): "comment" ] = "Added p_max_pu timeseries for pv and wind" - __tablename__ = "egon_etrago_generator_timeseries" __table_args__ = { "schema": "grid", @@ -424,7 +417,6 @@ class EgonPfHvLink(Base): contributor_list[4]["comment"] = "Added H2 related links" contributor_list[5]["comment"] = "Added CH4 links" - __tablename__ = "egon_etrago_link" __table_args__ = { "schema": "grid", @@ -523,7 +515,6 @@ class EgonPfHvLoad(Base): contributor_list[2]["comment"] = "Added gas demands" contributor_list[3]["comment"] = "Added mobility demands" - __tablename__ = "egon_etrago_load" __table_args__ = { "schema": "grid", @@ -565,7 +556,6 @@ class EgonPfHvLoadTimeseries(Base): contributor_list[2]["comment"] = "Added e mobility load timeseries" contributor_list[3]["comment"] = "Added gas load timeseries" - __tablename__ = "egon_etrago_load_timeseries" __table_args__ = { "schema": "grid", @@ -593,7 +583,6 @@ class EgonPfHvCarrier(Base): contributor_list = contributors(["fw"]) contributor_list[0]["comment"] = "Added list of carriers" - __tablename__ = "egon_etrago_carrier" __table_args__ = { "schema": "grid", @@ -626,7 +615,6 @@ class EgonPfHvStorage(Base): "comment" ] = "Added battery and pumped hydro storage units" - __tablename__ = "egon_etrago_storage" __table_args__ = { "schema": "grid", @@ -675,7 +663,6 @@ class EgonPfHvStorageTimeseries(Base): contributor_list = contributors(["cb"]) contributor_list[0]["comment"] = "Added metadata" - __tablename__ = "egon_etrago_storage_timeseries" __table_args__ = { "schema": "grid", @@ -814,7 +801,6 @@ class EgonPfHvTransformer(Base): contributor_list[0]["comment"] = "Added transformes from osmTGmod tables" contributor_list[1]["comment"] = "Added meta data" - __tablename__ = "egon_etrago_transformer" __table_args__ = { "schema": "grid", @@ -901,7 +887,7 @@ def create_tables(): db.execute_sql("CREATE SCHEMA IF NOT EXISTS grid;") engine = db.engine() - ##################### drop tables with old names ######################### + # drop tables with old names ######################### db.execute_sql( """ DROP TABLE IF EXISTS grid.egon_pf_hv_bus;""" @@ -1117,7 +1103,8 @@ def insert_carriers(): def check_carriers(): - """Check if any eTraGo table has carriers not included in the carrier table. + """Check if any eTraGo table has carriers not included + in the carrier table. Raises ------ @@ -1125,7 +1112,7 @@ def check_carriers(): used in any eTraGo table. """ carriers = db.select_dataframe( - f""" + """ SELECT name FROM grid.egon_etrago_carrier """ ) @@ -1145,8 +1132,8 @@ def check_carriers(): if len(unknown_carriers) > 0: msg = ( - "The eTraGo tables contain carriers, that are not included in the " - "carrier table:\n" + "The eTraGo tables contain carriers, that are not included " + "in the carrier table:\n" ) for table, carriers in unknown_carriers.items(): carriers = [str(c) for c in carriers] diff --git a/src/egon/data/datasets/gas_areas.py b/src/egon/data/datasets/gas_areas.py index 69921838d..250245297 100755 --- a/src/egon/data/datasets/gas_areas.py +++ b/src/egon/data/datasets/gas_areas.py @@ -2,22 +2,22 @@ The central module containing code to create CH4 and H2 voronoi polygons """ + import datetime import json -import pandas as pd from geoalchemy2.types import Geometry from sqlalchemy import BigInteger, Column, Text from sqlalchemy.ext.declarative import declarative_base +import pandas as pd -from egon.data import db, config +from egon.data import config, db from egon.data.datasets import Dataset, wrapped_partial from egon.data.datasets.generate_voronoi import get_voronoi_geodataframe from egon.data.metadata import ( context, contributors, license_egon_data_odbl, - meta_metadata, sources, ) @@ -39,7 +39,7 @@ class GasAreaseGon2035(Dataset): *Resulting tables* * :py:class:`EgonPfHvGasVoronoi ` - """ + """ # noqa: E501 #: name: str = "GasAreaseGon2035" @@ -62,17 +62,23 @@ class GasAreaseGon100RE(Dataset): :py:func:`voronoi_egon100RE` function. *Dependencies* - * :py:class:`EtragoSetup ` - * :py:class:`HydrogenBusEtrago ` - * :py:class:`HydrogenGridEtrago ` - * :py:class:`Vg250 ` - * :py:class:`GasNodesAndPipes ` - * :py:class:`GasAreaseGon2035 ` + * :py:class: + `EtragoSetup ` + * :py:class: + `HydrogenBusEtrago ` + * :py:class: + `HydrogenGridEtrago ` + * :py:class: + `Vg250 ` + * :py:class: + `GasNodesAndPipes ` + * :py:class: + `GasAreaseGon2035 ` *Resulting tables* * :py:class:`EgonPfHvGasVoronoi ` - """ + """ # noqa: E501 #: name: str = "GasAreaseGon100RE" @@ -96,7 +102,6 @@ class EgonPfHvGasVoronoi(Base): Class definition of table grid.egon_gas_voronoi """ - source_list = [ sources()["openstreetmap"], sources()["SciGRID_gas"], @@ -158,7 +163,6 @@ class EgonPfHvGasVoronoi(Base): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Create json dump meta_json = "'" + json.dumps(meta, indent=4, ensure_ascii=False) + "'" @@ -263,18 +267,16 @@ def create_voronoi(scn_name, carrier): """, geom_col="geometry", ).to_crs(epsg=4326) - - + if isinstance(carrier, str): - if carrier == "H2": - carriers = ["H2", "H2_grid"] - else: - carriers = [carrier] + if carrier == "H2": + carriers = ["H2", "H2_grid"] + else: + carriers = [carrier] else: carriers = carrier - + carrier_strings = "', '".join(carriers) - db.execute_sql( f""" @@ -292,11 +294,10 @@ def create_voronoi(scn_name, carrier): AND carrier IN ('{carrier_strings}'); """, ).to_crs(epsg=4326) - - + if len(buses) == 0: return - + # generate voronois # For some scenarios it is defined that there is only 1 bus (e.g. gas). It # means that there will be just 1 voronoi covering the entire german @@ -317,7 +318,7 @@ def create_voronoi(scn_name, carrier): # Insert data to db gdf.set_crs(epsg=4326).to_postgis( - f"egon_gas_voronoi", + "egon_gas_voronoi", engine, schema="grid", index=False, @@ -338,7 +339,7 @@ class GasAreas(Dataset): *Resulting tables* * :py:class:`EgonPfHvGasVoronoi ` - """ + """ # noqa: E501 #: name: str = "GasAreas" @@ -357,9 +358,13 @@ class GasAreas(Dataset): for scn_name in config.settings()["egon-data"]["--scenarios"]: if "status" in scn_name: - tasks += (wrapped_partial( - voronoi_status, scn_name=scn_name, postfix=f"_{scn_name[-4:]}" - ),) + tasks += ( + wrapped_partial( + voronoi_status, + scn_name=scn_name, + postfix=f"_{scn_name[-4:]}", + ), + ) def __init__(self, dependencies): super().__init__( diff --git a/src/egon/data/datasets/heat_demand/__init__.py b/src/egon/data/datasets/heat_demand/__init__.py index 8604bf890..f8ced5f81 100644 --- a/src/egon/data/datasets/heat_demand/__init__.py +++ b/src/egon/data/datasets/heat_demand/__init__.py @@ -35,44 +35,27 @@ from egon.data import db, subprocess from egon.data.datasets import Dataset -from egon.data.datasets.scenario_parameters import ( - get_sector_parameters, -) -from egon.data.metadata import ( - context, - license_ccby, - meta_metadata, - sources, -) +from egon.data.datasets.scenario_parameters import get_sector_parameters +from egon.data.metadata import context, license_ccby, sources import egon.data.config - +# Around line 49 class HeatDemandImport(Dataset): - """ Insert the annual heat demand per census cell for each scenario This dataset downloads the heat demand raster data for private households - and CTS from Peta 5.0.1 (https://s-eenergies-open-data-euf.hub.arcgis.com/maps/d7d18b63250240a49eb81db972aa573e/about) + and CTS from Peta 5.0.1 + (https://s-eenergies-open-data-euf.hub.arcgis.com/maps/d7d18b63250240a49eb81db972aa573e/about) and stores it into files in the working directory. The data from Peta 5.0.1 represents the status quo of the year 2015. To model future heat demands, the data is scaled to meet target values from external sources. These target values are defined for each scenario - in :py:class:`ScenarioParameters `. - - *Dependencies* - * :py:class:`ScenarioParameters ` - * :py:class:`Vg250 ` - * :py:class:`ZensusVg250 ` - - *Resulting tables* - * :py:class:`demand.egon_peta_heat ` is created and filled - + in :py:class:`ScenarioParameters + `. """ - - #: name: str = "heat-demands" #: version: str = "0.0.4" @@ -384,7 +367,8 @@ def future_heat_demand_germany(scenario_name): elif scenario_name == "status2019": heat_parameters = get_sector_parameters("heat", scenario=scenario_name) - # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 + # Calculate reduction share based on final energy demand and overall + # demand from Peta for 2015 res_hd_reduction = ( heat_parameters["DE_demand_residential_TJ"] / 3600 / 443.788483 ) @@ -392,25 +376,33 @@ def future_heat_demand_germany(scenario_name): heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 ) elif scenario_name == "status2023": - heat_parameters = get_sector_parameters("heat", scenario=scenario_name) # currently data for 2019 is used + heat_parameters = get_sector_parameters( + "heat", scenario=scenario_name + ) # currently data for 2019 is used # see scenario_paramters/__init__ for this. - # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 + # Calculate reduction share based on final energy demand and overall + # demand from Peta for 2015 res_hd_reduction = ( - heat_parameters["DE_demand_residential_TJ"] / 3600 / 443.788483 # TODO status2023 can values stay same? + heat_parameters["DE_demand_residential_TJ"] + / 3600 + / 443.788483 # TODO status2023 can values stay same? ) ser_hd_reduction = ( - heat_parameters["DE_demand_service_TJ"] / 3600 / 226.588158 # TODO status2023 can values stay same? + heat_parameters["DE_demand_service_TJ"] + / 3600 + / 226.588158 # TODO status2023 can values stay same? ) elif scenario_name == "eGon100RE": heat_parameters = get_sector_parameters("heat", scenario=scenario_name) - # Calculate reduction share based on final energy demand and overall demand from Peta for 2015 - res_hd_reduction = ( - heat_parameters["DE_demand_residential_MWh"] / (443.788483 * 1e6) + # Calculate reduction share based on final energy demand and overall + # demand from Peta for 2015 + res_hd_reduction = heat_parameters["DE_demand_residential_MWh"] / ( + 443.788483 * 1e6 ) - ser_hd_reduction = ( - heat_parameters["DE_demand_service_MWh"] / (226.588158 * 1e6) + ser_hd_reduction = heat_parameters["DE_demand_service_MWh"] / ( + 226.588158 * 1e6 ) else: heat_parameters = get_sector_parameters("heat", scenario=scenario_name) @@ -425,7 +417,7 @@ def future_heat_demand_germany(scenario_name): # Open, read and adjust the cutout heat demand distributions for Germany # https://rasterio.readthedocs.io/en/latest/topics/writing.html - # https://gis.stackexchange.com/questions/338282/applying-equation-to-a-numpy-array-while-preserving-tiff-metadata-coordinates + # https://gis.stackexchange.com/questions/338282/applying-equation-to-a-numpy-array-while-preserving-tiff-metadata-coordinates # noqa: E501 # Write an array as a raster band to a new 16-bit file. For # the new file's profile, the profile of the source is adjusted. @@ -529,8 +521,9 @@ def heat_demand_to_db_table(): db.execute_sql("DELETE FROM demand.egon_peta_heat;") for source in sources: - if not "2015" in source.stem: - # Create a temporary table and fill the final table using the sql script + if "2015" not in source.stem: + # Create a temporary table and fill the final table using + # the sql script rasters = f"heat_demand_rasters_{source.stem.lower()}" import_rasters = subprocess.run( ["raster2pgsql", "-e", "-s", "3035", "-I", "-C", "-F", "-a"] @@ -541,11 +534,11 @@ def heat_demand_to_db_table(): with engine.begin() as connection: print( f'CREATE TEMPORARY TABLE "{rasters}"' - ' ("rid" serial PRIMARY KEY,"rast" raster,"filename" text);' + ' ("rid" serial PRIMARY KEY,"rast" raster,"filename" text);' # noqa: E501 ) connection.execute( f'CREATE TEMPORARY TABLE "{rasters}"' - ' ("rid" serial PRIMARY KEY,"rast" raster,"filename" text);' + ' ("rid" serial PRIMARY KEY,"rast" raster,"filename" text);' # noqa: E501 ) connection.execute(import_rasters) connection.execute(f'ANALYZE "{rasters}"') @@ -581,21 +574,26 @@ def adjust_residential_heat_to_zensus(scenario): # Select overall residential heat demand overall_demand = db.select_dataframe( - f"""SELECT SUM(demand) as overall_demand - FROM demand.egon_peta_heat - WHERE scenario = {'scenario'} and sector = 'residential' - """ + f""" + SELECT SUM(demand) as overall_demand + FROM demand.egon_peta_heat + WHERE scenario = {scenario} + AND sector = 'residential' + """ ).overall_demand[0] # Select heat demand in populated cells df = db.select_dataframe( - f"""SELECT * - FROM demand.egon_peta_heat - WHERE scenario = {'scenario'} and sector = 'residential' + f""" + SELECT * + FROM demand.egon_peta_heat + WHERE scenario = {scenario} + AND sector = 'residential' AND zensus_population_id IN ( SELECT id FROM society.destatis_zensus_population_per_ha_inside_germany - )""", + ) + """, index_col="id", ) @@ -686,14 +684,14 @@ def add_metadata(): { "fields": ["zensus_population_id"], "reference": { - "resource": "society.destatis_zensus_population_per_ha", + "resource": "society.destatis_zensus_population_per_ha", # noqa: E501 "fields": ["id"], }, }, { "fields": ["scenario"], "reference": { - "resource": "scenario.egon_scenario_parameters", + "resource": "scenario.egon_scenario_parameters", # noqa: E501 "fields": ["name"], }, }, @@ -719,7 +717,6 @@ def add_metadata(): "comment": "Updated metadata", }, ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" diff --git a/src/egon/data/datasets/heat_demand_timeseries/__init__.py b/src/egon/data/datasets/heat_demand_timeseries/__init__.py index 4926ee5ab..42f32c7ae 100644 --- a/src/egon/data/datasets/heat_demand_timeseries/__init__.py +++ b/src/egon/data/datasets/heat_demand_timeseries/__init__.py @@ -7,20 +7,10 @@ from sqlalchemy import ARRAY, Column, Float, Integer, String, Text from sqlalchemy.ext.declarative import declarative_base -import geopandas as gpd import numpy as np import pandas as pd from egon.data import config, db -import egon.data.datasets.era5 as era - -try: - from disaggregator import temporal -except ImportError as e: - pass - -from math import ceil - from egon.data.datasets import Dataset from egon.data.datasets.heat_demand_timeseries.daily import ( daily_demand_shares_per_climate_zone, @@ -30,12 +20,7 @@ from egon.data.datasets.heat_demand_timeseries.service_sector import ( CTS_demand_scale, ) -from egon.data.metadata import ( - context, - license_egon_data_odbl, - meta_metadata, - sources, -) +from egon.data.metadata import context, license_egon_data_odbl, sources Base = declarative_base() @@ -87,7 +72,6 @@ def create_timeseries_for_building(building_id, scenario): ------- pandas.DataFrame Hourly heat demand timeseries in MW for the selected building - """ return db.select_dataframe( @@ -159,7 +143,6 @@ def create_district_heating_profile(scenario, area_id): pd.DataFrame An hourly heat demand timeseries in MW for the selected district heating grid. - """ start_time = datetime.now() @@ -245,8 +228,9 @@ def create_district_heating_profile(scenario, area_id): def create_district_heating_profile_python_like(scenario="eGon2035"): """Creates profiles for all district heating grids in one scenario. - Similar to create_district_heating_profile but faster and needs more RAM. - The results are directly written into the database. + + Similar to ``create_district_heating_profile`` but faster and needs + more RAM. The results are directly written into the database. Parameters ---------- @@ -256,7 +240,6 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): Returns ------- None. - """ start_time = datetime.now() @@ -384,18 +367,20 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): annual_demand.area_id == area ].demand_total.sum() ) / ( - annual_demand[annual_demand.area_id == area].demand_total.sum() + annual_demand[ + annual_demand.area_id == area + ].demand_total.sum() ) - assert ( - abs(diff) < 0.04 - ), f"""Deviation of residential heat demand time - series for district heating grid {str(area)} is {diff}""" + assert abs(diff) < 0.04, ( + "Deviation of residential heat demand time series for " + f"district heating grid {str(area)} is {diff}" + ) if abs(diff) > 0.03: warnings.warn( - f"""Deviation of residential heat demand time - series for district heating grid {str(area)} is {diff}""" + "Deviation of residential heat demand time series for " + f"district heating grid {str(area)} is {diff}" ) hh = np.concatenate( @@ -431,7 +416,7 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): entry = EgonTimeseriesDistrictHeating( area_id=int(area), scenario=scenario, - dist_aggregated_mw=(hh).tolist(), + dist_aggregated_mw=hh.tolist(), ) elif not cts.empty: entry = EgonTimeseriesDistrictHeating( @@ -450,7 +435,8 @@ def create_district_heating_profile_python_like(scenario="eGon2035"): session.commit() print( - f"Time to create time series for district heating scenario {scenario}" + "Time to create time series for district heating scenario " + f"{scenario}" ) print(datetime.now() - start_time) @@ -538,7 +524,7 @@ def calulate_peak_load(df, scenario): df.groupby("building_id") .max()[range(24)] .max(axis=1) - .mul(1000000) + .mul(1_000_000) .astype(int) .reset_index() ) @@ -670,7 +656,7 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): start_time = datetime.now() idp_df = db.select_dataframe( - f""" + """ SELECT index, idp FROM demand.egon_heat_idp_pool """, index_col="index", @@ -730,7 +716,8 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): session = sessionmaker(bind=db.engine())() print( - f"Time to create overhead for time series for district heating scenario {scenario}" + "Time to create overhead for time series for district heating " + f"scenario {scenario}" ) print(datetime.now() - start_time) @@ -748,7 +735,8 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): UNNEST (selected_idp_profiles) WITH ORDINALITY as selected_idp WHERE a.zensus_population_id NOT IN ( - SELECT zensus_population_id FROM demand.egon_map_zensus_district_heating_areas + SELECT zensus_population_id FROM + demand.egon_map_zensus_district_heating_areas WHERE scenario = '{scenario}' ) AND a.zensus_population_id IN ( @@ -794,8 +782,8 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): ) / (annual_demand[annual_demand.bus_id == grid].demand_total.sum()) assert abs(diff) < 0.03, ( - "Deviation of residential heat demand time series for mv" - f" grid {grid} is {diff}" + "Deviation of residential heat demand time series for mv " + f"grid {grid} is {diff}" ) if not (slice_df[hour].empty or cts.empty): @@ -808,7 +796,7 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): entry = EgonEtragoTimeseriesIndividualHeating( bus_id=int(grid), scenario=scenario, - dist_aggregated_mw=(hh).tolist(), + dist_aggregated_mw=hh.tolist(), ) elif not cts.empty: entry = EgonEtragoTimeseriesIndividualHeating( @@ -822,7 +810,8 @@ def create_individual_heating_profile_python_like(scenario="eGon2035"): session.commit() print( - f"Time to create time series for district heating scenario {scenario}" + "Time to create time series for district heating scenario " + f"{scenario}" ) print(datetime.now() - start_time) @@ -1079,13 +1068,12 @@ def store_national_profiles(): def export_etrago_cts_heat_profiles(): - """Export heat cts load profiles at mv substation level - to etrago-table in the database + """Export heat cts load profiles at MV substation level to the + etrago-table in the database. Returns ------- None. - """ # Calculate cts heat profiles at substation @@ -1179,7 +1167,6 @@ def metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Add metadata as a comment to the table @@ -1192,41 +1179,57 @@ def metadata(): class HeatTimeSeries(Dataset): """ - Chooses heat demand profiles for each residential and CTS building + Chooses heat demand profiles for each residential and CTS building. + + This dataset creates heat demand profiles in an hourly resolution. + Time series for CTS buildings are created using the SLP-gas method + implemented in the demandregio disaggregator with the function + :py:func:`export_etrago_cts_heat_profiles` and stored in the database. - This dataset creates heat demand profiles in an hourly resoultion. - Time series for CTS buildings are created using the SLP-gas method implemented - in the demandregio disagregator with the function :py:func:`export_etrago_cts_heat_profiles` - and stored in the database. - Time series for residential buildings are created based on a variety of synthetical created - individual demand profiles that are part of :py:class:`DataBundle `. - This method is desribed within the functions and in this publication: + Time series for residential buildings are created based on a variety + of synthetic individual demand profiles that are part of + :py:class:`DataBundle + `. + The method is described within the functions and in this publication: C. Büttner, J. Amme, J. Endres, A. Malla, B. Schachler, I. Cußmann, Open modeling of electricity and heat demand curves for all residential buildings in Germany, Energy Informatics 5 (1) (2022) 21. doi:10.1186/s42162-022-00201-y. - *Dependencies* - * :py:class:`DataBundle ` - * :py:class:`DemandRegio ` - * :py:class:`HeatDemandImport ` - * :py:class:`DistrictHeatingAreas ` + * :py:class:`DataBundle + ` + * :py:class:`DemandRegio + ` + * :py:class:`HeatDemandImport + ` + * :py:class:`DistrictHeatingAreas + ` * :py:class:`Vg250 ` - * :py:class:`ZensusMvGridDistricts ` - * :py:func:`hh_demand_buildings_setup ` + * :py:class:`ZensusMvGridDistricts + ` + * :py:func:`hh_demand_buildings_setup + ` * :py:class:`WeatherData ` - *Resulting tables* - * :py:class:`demand.egon_timeseries_district_heating ` is created and filled - * :py:class:`demand.egon_etrago_heat_cts ` is created and filled - * :py:class:`demand.egon_heat_timeseries_selected_profiles ` is created and filled - * :py:class:`demand.egon_daily_heat_demand_per_climate_zone ` + * :py:class:`demand.egon_timeseries_district_heating + ` is created and filled + * :py:class:`demand.egon_etrago_heat_cts + ` is created and filled - * :py:class:`boundaries.egon_map_zensus_climate_zones ` is created and filled - + * :py:class:`demand.egon_heat_timeseries_selected_profiles + ` + is created and filled + * :py:class:`demand.egon_daily_heat_demand_per_climate_zone + ` is created and filled + * :py:class:`boundaries.egon_map_zensus_climate_zones + ` is created and filled """ #: diff --git a/src/egon/data/datasets/heat_supply/__init__.py b/src/egon/data/datasets/heat_supply/__init__.py index 713ff9789..4d02d99f0 100644 --- a/src/egon/data/datasets/heat_supply/__init__.py +++ b/src/egon/data/datasets/heat_supply/__init__.py @@ -1,6 +1,4 @@ -"""The central module containing all code dealing with heat supply data - -""" +"""The central module containing all code dealing with heat supply data""" import datetime import json @@ -26,9 +24,7 @@ from egon.data.metadata import ( context, generate_resource_fields_from_sqla_model, - license_ccby, license_egon_data_odbl, - meta_metadata, sources, ) @@ -83,9 +79,8 @@ def district_heating(): Returns ------- None. - """ - sources = config.datasets()["heat_supply"]["sources"] + sources_cfg = config.datasets()["heat_supply"]["sources"] targets = config.datasets()["heat_supply"]["targets"] db.execute_sql( @@ -107,8 +102,7 @@ def district_heating(): if_exists="append", ) - - # Do not check data for status quo as is it not listed in the table + # Do not check data for status quo as it is not listed in the table if "status" not in scenario: # Compare target value with sum of distributed heat supply df_check = db.select_dataframe( @@ -117,8 +111,8 @@ def district_heating(): (SUM(a.capacity) - b.capacity) / SUM(a.capacity) as deviation FROM {targets['district_heating_supply']['schema']}. {targets['district_heating_supply']['table']} a, - {sources['scenario_capacities']['schema']}. - {sources['scenario_capacities']['table']} b + {sources_cfg['scenario_capacities']['schema']}. + {sources_cfg['scenario_capacities']['table']} b WHERE a.scenario = '{scenario}' AND b.scenario_name = '{scenario}' AND b.carrier = CONCAT('urban_central_', a.carrier) @@ -126,11 +120,10 @@ def district_heating(): """ ) # If the deviation is > 1%, throw an error - assert ( - df_check.deviation.abs().max() < 1 - ), f"""Unexpected deviation between target value and distributed - heat supply: {df_check} - """ + assert df_check.deviation.abs().max() < 1, ( + "Unexpected deviation between target value and distributed " + f"heat supply: {df_check}" + ) # Add gas boilers as conventional backup capacities backup = backup_gas_boilers(scenario) @@ -142,7 +135,6 @@ def district_heating(): if_exists="append", ) - # Insert resistive heaters which are not available in status quo if "status" not in scenario: backup_rh = backup_resistive_heaters(scenario) @@ -162,7 +154,6 @@ def individual_heating(): Returns ------- None. - """ targets = config.datasets()["heat_supply"]["targets"] @@ -180,7 +171,9 @@ def individual_heating(): distribution_level = "national" supply = cascade_heat_supply_indiv( - scenario, distribution_level=distribution_level, plotting=False + scenario, + distribution_level=distribution_level, + plotting=False, ) supply["scenario"] = scenario @@ -199,7 +192,6 @@ def metadata(): Returns ------- None. - """ fields = generate_resource_fields_from_sqla_model( @@ -273,7 +265,6 @@ def metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Add metadata as a comment to the table @@ -309,7 +300,9 @@ def metadata(): "name": "supply.egon_individual_heating", "title": "eGon heat supply for individual supplied buildings", "id": "WILL_BE_SET_AT_PUBLICATION", - "description": "Heat supply technologies for individual supplied buildings", + "description": ( + "Heat supply technologies for individual supplied buildings" + ), "language": ["EN"], "publicationDate": datetime.date.today().isoformat(), "context": context(), @@ -352,7 +345,6 @@ def metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Add metadata as a comment to the table @@ -365,27 +357,33 @@ def metadata(): class HeatSupply(Dataset): """ - Select and store heat supply technologies for inidvidual and district heating - - This dataset distributes heat supply technologies to each district heating grid - and individual supplies buildings per medium voltage grid district. - National installed capacities are predefined from external sources within - :py:class:`ScenarioCapacities `. - The further distribution is done using a cascade that follows a specific order of supply technologies - and the heat demand. + Select and store heat supply technologies for inidvidual and + district heating + This dataset distributes heat supply technologies to each district + heating grid and individual supplies buildings per medium voltage + grid district. National installed capacities are predefined from + external sources within :py:class:`ScenarioCapacities + `. + The further distribution is done using a cascade that follows a + specific order of supply technologies and the heat demand. *Dependencies* - * :py:class:`DataBundle ` - * :py:class:`DistrictHeatingAreas ` - * :py:class:`ZensusMvGridDistricts ` + * :py:class:`DataBundle + ` + * :py:class:`DistrictHeatingAreas + ` + * :py:class:`ZensusMvGridDistricts + ` * :py:class:`Chp ` - *Resulting tables* - * :py:class:`demand.egon_district_heating ` is created and filled - * :py:class:`demand.egon_individual_heating ` is created and filled - + * :py:class:`demand.egon_district_heating + ` is + created and filled + * :py:class:`demand.egon_individual_heating + ` is + created and filled """ #: @@ -408,13 +406,12 @@ def __init__(self, dependencies): ), ) + class GeothermalPotentialGermany(Dataset): def __init__(self, dependencies): super().__init__( name="GeothermalPotentialGermany", version="0.0.2", dependencies=dependencies, - tasks=( - potential_germany, - ), + tasks=(potential_germany,), ) diff --git a/src/egon/data/datasets/osm/__init__.py b/src/egon/data/datasets/osm/__init__.py index a63227f7c..78d79ff6b 100644 --- a/src/egon/data/datasets/osm/__init__.py +++ b/src/egon/data/datasets/osm/__init__.py @@ -27,7 +27,6 @@ context, generate_resource_fields_from_db_table, license_odbl, - meta_metadata, ) import egon.data.config import egon.data.subprocess as subprocess @@ -138,7 +137,7 @@ def add_metadata(): # Extract spatial extend and date (spatial_extend, osm_data_date) = re.compile( - "^([\\w-]*).*-(\\d+)$" + r"^([\w-]*).*-(\d+)$" ).findall(Path(input_filename).name.split(".")[0])[0] osm_data_date = datetime.datetime.strptime( osm_data_date, "%y%m%d" @@ -155,11 +154,10 @@ def add_metadata(): "title": f"OpenStreetMap (OSM) - Germany - {table_suffix}", "id": "WILL_BE_SET_AT_PUBLICATION", "description": ( - "OpenStreetMap is a free, editable map of the" - " whole world that is being built by volunteers" - " largely from scratch and released with" - " an open-content license.\n\n" - "The OpenStreetMap data here is the result of an PostgreSQL " + "OpenStreetMap is a free, editable map of the whole world " + "that is being built by volunteers largely from scratch and " + "released with an open-content license.\n\n" + "The OpenStreetMap data here is the result of a PostgreSQL " "database import using osm2pgsql with a custom style file." ), "language": ["en-EN", "de-DE"], @@ -185,7 +183,7 @@ def add_metadata(): "title": "OpenStreetMap Data Extracts (Geofabrik)", "description": ( "Full data extract of OpenStreetMap data for defined " - "spatial extent at ''referenceDate''" + "spatial extent at 'referenceDate'." ), "path": f"{osm_url}", "licenses": licenses, @@ -225,7 +223,6 @@ def add_metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(meta) + "'" @@ -305,16 +302,22 @@ class OpenStreetMap(Dataset): * :py:func:`Setup ` *Resulting Tables* - * openstreetmap.osm_line is created and filled (table has no associated python class) - * openstreetmap.osm_nodes is created and filled (table has no associated python class) - * openstreetmap.osm_point is created and filled (table has no associated python class) - * openstreetmap.osm_polygon is created and filled (table has no associated python class) - * openstreetmap.osm_rels is created and filled (table has no associated python class) - * openstreetmap.osm_roads is created and filled (table has no associated python class) - * openstreetmap.osm_ways is created and filled (table has no associated python class) + * openstreetmap.osm_line is created and filled (table has no + associated python class) + * openstreetmap.osm_nodes is created and filled (table has no + associated python class) + * openstreetmap.osm_point is created and filled (table has no + associated python class) + * openstreetmap.osm_polygon is created and filled (table has no + associated python class) + * openstreetmap.osm_rels is created and filled (table has no + associated python class) + * openstreetmap.osm_roads is created and filled (table has no + associated python class) + * openstreetmap.osm_ways is created and filled (table has no + associated python class) See documentation section :ref:`osm-ref` for more information. - """ #: diff --git a/src/egon/data/datasets/power_plants/mastr_db_classes.py b/src/egon/data/datasets/power_plants/mastr_db_classes.py index a3c340bbb..1379deecd 100644 --- a/src/egon/data/datasets/power_plants/mastr_db_classes.py +++ b/src/egon/data/datasets/power_plants/mastr_db_classes.py @@ -2,7 +2,6 @@ import json from geoalchemy2 import Geometry -from omi.dialects import get_dialect from sqlalchemy import ( Boolean, Column, @@ -20,8 +19,6 @@ contributors, generate_resource_fields_from_db_table, license_dedl, - meta_metadata, - meta_metadata, sources, ) @@ -52,7 +49,9 @@ class EgonPowerPlantsPv(Base): status = Column(String, nullable=True) # EinheitBetriebsstatus commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum - decommissioning_date = Column(DateTime, nullable=True) # DatumEndgueltigeStilllegung + decommissioning_date = Column( + DateTime, nullable=True + ) # DatumEndgueltigeStilllegung postcode = Column(String(5), nullable=True) # Postleitzahl city = Column(String(50), nullable=True) # Ort municipality = Column(String, nullable=True) # Gemeinde @@ -99,7 +98,9 @@ class EgonPowerPlantsWind(Base): status = Column(String, nullable=True) # EinheitBetriebsstatus commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum - decommissioning_date = Column(DateTime, nullable=True) # DatumEndgueltigeStilllegung + decommissioning_date = Column( + DateTime, nullable=True + ) # DatumEndgueltigeStilllegung postcode = Column(String(5), nullable=True) # Postleitzahl city = Column(String(50), nullable=True) # Ort municipality = Column(String, nullable=True) # Gemeinde @@ -131,7 +132,9 @@ class EgonPowerPlantsBiomass(Base): status = Column(String, nullable=True) # EinheitBetriebsstatus commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum - decommissioning_date = Column(DateTime, nullable=True) # DatumEndgueltigeStilllegung + decommissioning_date = Column( + DateTime, nullable=True + ) # DatumEndgueltigeStilllegung postcode = Column(String(5), nullable=True) # Postleitzahl city = Column(String(50), nullable=True) # Ort municipality = Column(String, nullable=True) # Gemeinde @@ -162,7 +165,9 @@ class EgonPowerPlantsHydro(Base): status = Column(String, nullable=True) # EinheitBetriebsstatus commissioning_date = Column(DateTime, nullable=True) # Inbetriebnahmedatum - decommissioning_date = Column(DateTime, nullable=True) # DatumEndgueltigeStilllegung + decommissioning_date = Column( + DateTime, nullable=True + ) # DatumEndgueltigeStilllegung postcode = Column(String(5), nullable=True) # Postleitzahl city = Column(String(50), nullable=True) # Ort municipality = Column(String, nullable=True) # Gemeinde @@ -382,9 +387,7 @@ def add_metadata(): "Data from Marktstammdatenregister (MaStR) data using " "the data dump from 2022-11-17 for eGon-data." ), - "path": ( - f"https://zenodo.org/record/{deposit_id_mastr}" - ), + "path": (f"https://zenodo.org/record/{deposit_id_mastr}"), "licenses": [license_dedl(attribution="© Amme, Jonathan")], }, sources()["egon-data"], @@ -413,7 +416,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github." @@ -441,9 +443,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", diff --git a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py index 3d9616de3..5c289d60b 100644 --- a/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py +++ b/src/egon/data/datasets/power_plants/pv_rooftop_buildings.py @@ -16,7 +16,6 @@ from geoalchemy2 import Geometry from loguru import logger from numpy.random import RandomState, default_rng -from omi.dialects import get_dialect from pyproj.crs.crs import CRS from sqlalchemy import BigInteger, Column, Float, Integer, String from sqlalchemy.dialects.postgresql import HSTORE @@ -39,8 +38,6 @@ generate_resource_fields_from_db_table, license_dedl, license_odbl, - meta_metadata, - meta_metadata, sources, ) @@ -358,6 +355,7 @@ class OsmBuildingsFiltered(Base): Class definition of table openstreetmap.osm_buildings_filtered. """ + __tablename__ = "osm_buildings_filtered" __table_args__ = {"schema": "openstreetmap"} @@ -760,9 +758,9 @@ def allocate_pv( assert len(assigned_buildings) == len(assigned_buildings.gens_id.unique()) - q_mastr_gdf.loc[assigned_buildings.gens_id, "building_id"] = ( - assigned_buildings.index - ) + q_mastr_gdf.loc[ + assigned_buildings.gens_id, "building_id" + ] = assigned_buildings.index assigned_gens = q_mastr_gdf.loc[~q_mastr_gdf.building_id.isna()] @@ -985,6 +983,7 @@ class Vg250Lan(Base): Class definition of table boundaries.vg250_lan. """ + __tablename__ = "vg250_lan" __table_args__ = {"schema": "boundaries"} @@ -1168,7 +1167,7 @@ def cap_per_bus_id( WHERE carrier = 'solar_rooftop' AND scenario = '{scenario}' GROUP BY bus_id - """ + """ # noqa: E501 df = db.select_dataframe(sql, index_col="bus_id") @@ -1180,7 +1179,7 @@ def cap_per_bus_id( FROM {targets['generators']['schema']}.{targets['generators']['table']} WHERE carrier = 'solar_rooftop' AND scn_name = '{scenario}' - """ + """ # noqa: E501 df = db.select_dataframe(sql, index_col="bus_id") df = df.loc[df.control != "Slack"] @@ -2131,6 +2130,7 @@ class EgonPowerPlantPvRoofBuilding(Base): Class definition of table supply.egon_power_plants_pv_roof_building. """ + __tablename__ = "egon_power_plants_pv_roof_building" __table_args__ = {"schema": "supply"} @@ -2210,9 +2210,7 @@ def add_metadata(): "Data from Marktstammdatenregister (MaStR) data using " "the data dump from 2022-11-17 for eGon-data." ), - "path": ( - f"https://zenodo.org/record/{deposit_id_mastr}" - ), + "path": (f"https://zenodo.org/record/{deposit_id_mastr}"), "licenses": [license_dedl(attribution="© Amme, Jonathan")], }, sources()["openstreetmap"], @@ -2240,7 +2238,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github." @@ -2268,9 +2265,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", @@ -2406,15 +2401,13 @@ def pv_rooftop_to_buildings(): mastr_gdf = load_mastr_data() - status_quo = "status2023" # FIXME: Hard coded + status_quo = "status2023" # FIXME: Hard coded ts = pd.Timestamp( config.datasets()["mastr_new"][f"{status_quo}_date_max"], tz="UTC" ) - mastr_gdf = mastr_gdf.loc[ - mastr_gdf.commissioning_date <= ts - ] + mastr_gdf = mastr_gdf.loc[mastr_gdf.commissioning_date <= ts] buildings_gdf = load_building_data() @@ -2438,7 +2431,8 @@ def pv_rooftop_to_buildings(): continue elif "status" in scenario: ts = pd.Timestamp( - config.datasets()["mastr_new"][f"{scenario}_date_max"], tz="UTC" + config.datasets()["mastr_new"][f"{scenario}_date_max"], + tz="UTC", ) scenario_buildings_gdf = scenario_buildings_gdf.loc[ diff --git a/src/egon/data/datasets/renewable_feedin.py b/src/egon/data/datasets/renewable_feedin.py index 2858c77cb..883d11809 100644 --- a/src/egon/data/datasets/renewable_feedin.py +++ b/src/egon/data/datasets/renewable_feedin.py @@ -1,10 +1,12 @@ """ -Central module containing all code dealing with processing era5 weather data. +Central module containing all code dealing with processing ERA5 +weather data. """ import datetime import json import time + from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd @@ -13,27 +15,27 @@ from egon.data import db from egon.data.datasets import Dataset -from egon.data.datasets.era5 import EgonEra5Cells, EgonRenewableFeedIn, import_cutout -from egon.data.datasets.scenario_parameters import get_sector_parameters -from egon.data.metadata import ( - context, - license_ccby, - meta_metadata, - sources, +from egon.data.datasets.era5 import ( + EgonEra5Cells, + EgonRenewableFeedIn, + import_cutout, ) +from egon.data.datasets.scenario_parameters import get_sector_parameters from egon.data.datasets.zensus_vg250 import DestatisZensusPopulationPerHa +from egon.data.metadata import context, license_ccby, sources import egon.data.config class RenewableFeedin(Dataset): """ - Calculate possible feedin time series for renewable energy generators + Calculate possible feed-in time series for renewable generators. - This dataset calculates possible feedin timeseries for fluctuation renewable generators - and coefficient of performance time series for heat pumps. Relevant input is the - downloaded weather data. Parameters for the time series calcultaion are also defined by - representative types of pv plants and wind turbines that are selected within this dataset. - The resulting profiles are stored in the database. + This dataset calculates feed-in time series for fluctuating renewable + generators and coefficient-of-performance series for heat pumps. + Relevant input is the downloaded weather data. Parameters for the + calculation are defined by representative types of PV plants and wind + turbines selected in this dataset. The resulting profiles are stored in + the database. *Dependencies* @@ -42,7 +44,8 @@ class RenewableFeedin(Dataset): * :py:class:`ZensusVg250 ` *Resulting tables* - * :py:class:`supply.egon_era5_renewable_feedin ` is filled + * :py:class:`supply.egon_era5_renewable_feedin + ` is filled """ @@ -85,7 +88,7 @@ class MapZensusWeatherCell(Base): def weather_cells_in_germany(geom_column="geom"): - """Get weather cells which intersect with Germany + """Get weather cells which intersect with Germany. Returns ------- @@ -100,15 +103,17 @@ def weather_cells_in_germany(geom_column="geom"): f"""SELECT w_id, geom_point, geom FROM {cfg['weather_cells']['schema']}. {cfg['weather_cells']['table']} - WHERE ST_Intersects('SRID=4326; - POLYGON((5 56, 15.5 56, 15.5 47, 5 47, 5 56))', geom)""", + WHERE ST_Intersects( + 'SRID=4326;\n POLYGON((5 56, 15.5 56, 15.5 47, 5 47, 5 56))', + geom + )""", geom_col=geom_column, index_col="w_id", ) def offshore_weather_cells(geom_column="geom"): - """Get weather cells which intersect with Germany + """Get weather cells which intersect with Germany. Returns ------- @@ -123,20 +128,21 @@ def offshore_weather_cells(geom_column="geom"): f"""SELECT w_id, geom_point, geom FROM {cfg['weather_cells']['schema']}. {cfg['weather_cells']['table']} - WHERE ST_Intersects('SRID=4326; - POLYGON((5.5 55.5, 14.5 55.5, 14.5 53.5, 5.5 53.5, 5.5 55.5))', - geom)""", + WHERE ST_Intersects( + 'SRID=4326;\n POLYGON((5.5 55.5, 14.5 55.5, 14.5 53.5, 5.5 53.5, 5.5 55.5))', + geom + )""", # noqa: E501 geom_col=geom_column, index_col="w_id", ) def federal_states_per_weather_cell(): - """Assings a federal state to each weather cell in Germany. + """Assign a federal state to each weather cell in Germany. - Sets the federal state to the weather celss using the centroid. - Weather cells at the borders whoes centroid is not inside Germany - are assinged to the closest federal state. + Sets the federal state to the weather cells using the centroid. Weather + cells at the borders whose centroid is not inside Germany are assigned to + the closest federal state. Returns ------- @@ -147,7 +153,7 @@ def federal_states_per_weather_cell(): cfg = egon.data.config.datasets()["renewable_feedin"]["sources"] - # Select weather cells and ferear states from database + # Select weather cells and federal states from database weather_cells = weather_cells_in_germany(geom_column="geom_point") federal_states = db.select_geodataframe( @@ -191,7 +197,7 @@ def federal_states_per_weather_cell(): def turbine_per_weather_cell(): - """Assign wind onshore turbine types to weather cells + """Assign wind onshore turbine types to weather cells. Returns ------- @@ -232,12 +238,12 @@ def turbine_per_weather_cell(): def feedin_per_turbine(): - """Calculate feedin timeseries per turbine type and weather cell + """Calculate feed-in time series per turbine type and weather cell. Returns ------- gdf : GeoPandas.GeoDataFrame - Feed-in timeseries per turbine type and weather cell + Feed-in time series per turbine type and weather cell """ @@ -246,7 +252,7 @@ def feedin_per_turbine(): gdf = gpd.GeoDataFrame(geometry=cutout.grid.geometry, crs=4326) - # Calculate feedin-timeseries for E-141 + # Calculate feed-in time series for E-141 # source: # https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library turbine_e141 = { @@ -290,7 +296,7 @@ def feedin_per_turbine(): gdf["E-141"] = ts_e141.to_pandas().transpose().values.tolist() - # Calculate feedin-timeseries for E-126 + # Calculate feed-in time series for E-126 # source: # https://openenergy-platform.org/dataedit/view/supply/wind_turbine_library turbine_e126 = { @@ -338,7 +344,7 @@ def feedin_per_turbine(): def wind(): - """Insert feed-in timeseries for wind onshore turbines to database + """Insert feed-in time series for wind onshore turbines to database. Returns ------- @@ -352,10 +358,10 @@ def wind(): weather_cells = turbine_per_weather_cell() weather_cells = weather_cells[weather_cells.wind_turbine.notnull()] - # Calculate feedin timeseries per turbine and weather cell + # Calculate feed-in time series per turbine and weather cell timeseries_per_turbine = feedin_per_turbine() - # Join weather cells and feedin-timeseries + # Join weather cells and feed-in time series timeseries = gpd.sjoin(weather_cells, timeseries_per_turbine)[ ["E-141", "E-126"] ] @@ -368,7 +374,7 @@ def wind(): data={"weather_year": weather_year, "carrier": "wind_onshore"}, ) - # Insert feedin for selected turbine per weather cell + # Insert feed-in for selected turbine per weather cell for turbine in ["E-126", "E-141"]: idx = weather_cells.index[ (weather_cells.wind_turbine == turbine) @@ -393,7 +399,7 @@ def wind(): def wind_offshore(): - """Insert feed-in timeseries for wind offshore turbines to database + """Insert feed-in time series for wind offshore to database. Returns ------- @@ -401,7 +407,7 @@ def wind_offshore(): """ - # Get offshore weather cells arround Germany + # Get offshore weather cells around Germany weather_cells = offshore_weather_cells() # Select weather data for German coast @@ -410,7 +416,7 @@ def wind_offshore(): # Select weather year from cutout weather_year = cutout.name.split("-")[2] - # Calculate feedin timeseries + # Calculate feed-in time series ts_wind_offshore = cutout.wind( "Vestas_V164_7MW_offshore", per_unit=True, @@ -422,7 +428,7 @@ def wind_offshore(): def pv(): - """Insert feed-in timeseries for pv plants to database + """Insert feed-in time series for PV plants to database. Returns ------- @@ -439,7 +445,7 @@ def pv(): # Select weather year from cutout weather_year = cutout.name.split("-")[1] - # Calculate feedin timeseries + # Calculate feed-in time series ts_pv = cutout.pv( "CSi", orientation={"slope": 35.0, "azimuth": 180.0}, @@ -452,7 +458,7 @@ def pv(): def solar_thermal(): - """Insert feed-in timeseries for pv plants to database + """Insert feed-in time series for solar thermal to database. Returns ------- @@ -469,7 +475,7 @@ def solar_thermal(): # Select weather year from cutout weather_year = cutout.name.split("-")[1] - # Calculate feedin timeseries + # Calculate feed-in time series ts_solar_thermal = cutout.solar_thermal( clearsky_model="simple", orientation={"slope": 45.0, "azimuth": 180.0}, @@ -486,15 +492,15 @@ def heat_pump_cop(): """ Calculate coefficient of performance for heat pumps according to T. Brown et al: "Synergies of sector coupling and transmission - reinforcement in a cost-optimised, highlyrenewable European energy system", - 2018, p. 8 + reinforcement in a cost-optimised, highly renewable European energy + system", 2018, p. 8 Returns ------- None. """ - # Assume temperature of heating system to 55°C according to Brown et. al + # Assume temperature of heating system to 55°C according to Brown et al. t_sink = 55 carrier = "heat_pump_cop" @@ -511,7 +517,7 @@ def heat_pump_cop(): # Select weather year from cutout weather_year = cutout.name.split("-")[1] - # Calculate feedin timeseries + # Calculate temperature series temperature = cutout.temperature( shapes=weather_cells.to_crs(4326).geom ).transpose() @@ -520,8 +526,8 @@ def heat_pump_cop(): delta_t = t_sink - t_source - # Calculate coefficient of performance for air sourced heat pumps - # according to Brown et. al + # Calculate coefficient of performance for air-sourced heat pumps + # according to Brown et al. cop = 6.81 - 0.121 * delta_t + 0.00063 * delta_t**2 df = pd.DataFrame( @@ -550,12 +556,12 @@ def heat_pump_cop(): def insert_feedin(data, carrier, weather_year): - """Insert feedin data into database + """Insert feed-in data into database. Parameters ---------- data : xarray.core.dataarray.DataArray - Feedin timeseries data + Feed-in time series data carrier : str Name of energy carrier weather_year : int @@ -579,11 +585,11 @@ def insert_feedin(data, carrier, weather_year): data={"weather_year": weather_year, "carrier": carrier}, ) - # Convert solar thermal data from W/m^2 to MW/(1000m^2) = kW/m^2 + # Convert solar thermal from W/m^2 to MW/(1000m^2) = kW/m^2 if carrier == "solar_thermal": data *= 1e-3 - # Insert feedin into DataFrame + # Insert feed-in into DataFrame df.feedin = data.values.tolist() # Delete existing rows for carrier @@ -604,7 +610,7 @@ def insert_feedin(data, carrier, weather_year): def mapping_zensus_weather(): - """Perform mapping between era5 weather cell and zensus grid""" + """Perform mapping between ERA5 weather cell and Zensus grid.""" with db.session_scope() as session: cells_query = session.query( @@ -649,7 +655,7 @@ def mapping_zensus_weather(): def add_metadata(): - """Add metdata to supply.egon_era5_renewable_feedin + """Add metadata to supply.egon_era5_renewable_feedin. Returns ------- @@ -678,7 +684,7 @@ def add_metadata(): "unit": "none", }, { - "description": "Weather-dependent feedin timeseries", + "description": "Weather-dependent feed-in time series", "name": "feedin", "type": "array", "unit": "p.u.", @@ -687,9 +693,9 @@ def add_metadata(): meta = { "name": "supply.egon_era5_renewable_feedin", - "title": "eGon feedin timeseries for RES", + "title": "eGon feed-in time series for RES", "id": "WILL_BE_SET_AT_PUBLICATION", - "description": "Weather-dependent feedin timeseries for RES", + "description": "Weather-dependent feed-in time series for RES", "language": ["EN"], "publicationDate": datetime.date.today().isoformat(), "context": context(), @@ -705,9 +711,16 @@ def add_metadata(): ], "licenses": [ license_ccby( - "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " - "© Copernicus Climate Change Service (C3S) Climate Data Store " - "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + ( + "© Bundesamt für Kartographie und Geodäsie 2020 " + "(Daten verändert); " + "© Copernicus Climate Change Service (C3S) " + "Climate Data Store " + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, " + "Julian Endres, Carlos Epia, Stephan Günther, " + "Ulf Müller, Amélia Nadal, Guido Pleßmann, " + "Francesco Witte" + ) ) ], "contributors": [ @@ -734,7 +747,6 @@ def add_metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Create json dump diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index 8c8820e51..5dfc33670 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -1,5 +1,5 @@ -"""The central module containing all code dealing with importing data from -Netzentwicklungsplan 2035, Version 2031, Szenario C +"""The central module for importing data from the +Netzentwicklungsplan 2035, Version 2031, Szenario C. """ from pathlib import Path @@ -20,7 +20,6 @@ context, generate_resource_fields_from_sqla_model, license_ccby, - meta_metadata, sources, ) @@ -126,18 +125,18 @@ def insert_capacities_status_quo(scenario: str) -> None: ) rural_heat_capacity = { - # Rural heat capacity for 2019 according to NEP 2035, version 2021 + # Rural heat capacity for 2019 according to NEP 2035, v2021 "status2019": 1e6 * 5e-3, - # Rural heat capacity for 2023 according to NEP 2037, version 2023 + # Rural heat capacity for 2023 according to NEP 2037, v2023. # 1.2 Mio. for 2020 - # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ - # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # https://www.netzentwicklungsplan.de/sites/default/files/ + # 2023-07/NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 # and 3 kW per heat pump - # https://www.netzentwicklungsplan.de/sites/default/files/2022-11/ - # NEP_2035_V2021_2_Entwurf_Teil1.pdf#page=33 + # https://www.netzentwicklungsplan.de/sites/default/files/ + # 2022-11/NEP_2035_V2021_2_Entwurf_Teil1.pdf#page=33 # plus 0.15 Mio. 2021 and 0.24 Mio. in 2022 # https://www.enercity.de/magazin/unsere-welt/waermepumpen-boom - # plus 0.2 Mio. in H1 2023 -> Assumption 2023: 2 * 0.2 Mio = 0.4 Mio. + # plus 0.2 Mio. in H1 2023 -> Assumption 2023: 2 * 0.2 = 0.4 Mio. "status2023": (1.2 + 0.15 + 0.24 + 0.4) * 1e6 * 3e-3, }[scenario] @@ -165,8 +164,8 @@ def insert_capacities_status_quo(scenario: str) -> None: # MW for Germany "status2019": 600, # 1.3 GW in 2020/2021 - # https://www.netzentwicklungsplan.de/sites/default/files/2023-07/ - # NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 + # https://www.netzentwicklungsplan.de/sites/default/files/ + # 2023-07/NEP_2037_2045_V2023_2_Entwurf_Teil1_1.pdf#page=25 # Installed quantity 2020: 272,000 # Installed quantity 2023: 1,197,000 # https://www.photovoltaik.eu/solarspeicher/ @@ -192,8 +191,8 @@ def insert_capacities_status_quo(scenario: str) -> None: def insert_capacities_per_federal_state_nep(): - """Inserts installed capacities per federal state accordning to - NEP 2035 (version 2021), scenario 2035 C + """Insert installed capacities per federal state according to + NEP 2035 (version 2021), scenario 2035 C. Returns ------- @@ -218,7 +217,7 @@ def insert_capacities_per_federal_state_nep(): """ ) - # read-in installed capacities per federal state of germany + # Read installed capacities per federal state of Germany target_file = ( Path(".") / "data_bundle_egon_data" @@ -255,15 +254,13 @@ def insert_capacities_per_federal_state_nep(): # List federal state with an assigned wind offshore capacity index_list = list(df_windoff_fs.index.values) - # Overwrite capacities in df_windoff with more accurate values from - # df_windoff_fs - + # Overwrite capacities in df_windoff with values from df_windoff_fs for state in index_list: df.at["Wind offshore", state] = ( df_windoff_fs.at[state, "C 2035"] / 1000 ) - # sort NEP-carriers: + # Sort NEP-carriers rename_carrier = { "Wind onshore": "wind_onshore", "Wind offshore": "wind_offshore", @@ -282,13 +279,14 @@ def insert_capacities_per_federal_state_nep(): "KWK < 10 MW": "small_chp", } # 'Elektromobilitaet gesamt': 'transport', - # 'Elektromobilitaet privat': 'transport'} + # 'Elektromobilitaet privat': 'transport' # nuts1 to federal state in Germany map_nuts = pd.read_sql( f""" SELECT DISTINCT ON (nuts) gen, nuts - FROM {sources['boundaries']['schema']}.{sources['boundaries']['table']} + FROM {sources['boundaries']['schema']}. + {sources['boundaries']['table']} """, engine, index_col="gen", @@ -305,8 +303,7 @@ def insert_capacities_per_federal_state_nep(): for bl in map_nuts.index: data = pd.DataFrame(df[bl]) - # if distribution to federal states is not provided, - # use data from draft of scenario report + # If distribution to states is not provided, use scenario draft data for c in scaled_carriers: data.loc[c, bl] = ( df_draft.loc[c, bl] @@ -314,7 +311,7 @@ def insert_capacities_per_federal_state_nep(): * df.loc[c, "Summe"] ) - # split hydro into run of river and reservoir + # Split hydro into run of river and reservoir # according to draft of scenario report if data.loc["Lauf- und Speicherwasser", bl] > 0: for c in ["Speicherwasser", "Laufwasser"]: @@ -330,31 +327,28 @@ def insert_capacities_per_federal_state_nep(): data["nuts"] = map_nuts.nuts[bl] data["scenario_name"] = "eGon2035" - # According to NEP, each heatpump has 5kW_el installed capacity - # source: Entwurf des Szenariorahmens NEP 2035, version 2021, page 47 + # Each heat pump has 5kW_el installed capacity (NEP 2035 v2021 p.47) data.loc[data.carrier == "residential_rural_heat_pump", bl] *= 5e-6 data.loc[ - data.carrier == "residential_rural_heat_pump", "component" + data.carrier == "residential_rural_heat_pump", + "component", ] = "link" data = data.rename(columns={bl: "capacity"}) - # convert GW to MW + # Convert GW to MW data.capacity *= 1e3 insert_data = pd.concat([insert_data, data]) - # Get aggregated capacities from nep's power plant list for certain carrier - + # Get capacities from NEP's power plant list for certain carriers carriers = ["oil", "other_non_renewable", "pumped_hydro"] - capacities_list = aggr_nep_capacities(carriers) # Filter by carrier updated = insert_data[insert_data["carrier"].isin(carriers)] - # Merge to replace capacities for carriers "oil", "other_non_renewable" and - # "pumped_hydro" + # Merge to replace capacities for these carriers updated = ( updated.merge(capacities_list, on=["carrier", "nuts"], how="left") .fillna(0) @@ -377,7 +371,7 @@ def insert_capacities_per_federal_state_nep(): index=insert_data.index, ) - # Add district heating data accordning to energy and full load hours + # Add district heating data according to energy and full load hours district_heating_input() @@ -408,8 +402,7 @@ def population_share(): def aggr_nep_capacities(carriers): - """Aggregates capacities from NEP power plants list by carrier and federal - state + """Aggregate capacities from NEP list by carrier and federal state. Returns ------- @@ -417,12 +410,12 @@ def aggr_nep_capacities(carriers): Dataframe with capacities per federal state and carrier """ - # Get list of power plants from nep + # Get list of power plants from NEP nep_capacities = insert_nep_list_powerplants(export=False)[ ["federal_state", "carrier", "c2035_capacity"] ] - # Sum up capacities per federal state and carrier + # Sum capacities per federal state and carrier capacities_list = ( nep_capacities.groupby(["federal_state", "carrier"])["c2035_capacity"] .sum() @@ -436,7 +429,7 @@ def aggr_nep_capacities(carriers): # Include NUTS code capacities_list["nuts"] = capacities_list.federal_state.map(nuts_mapping()) - # Drop entries for foreign plants with nan values and federal_state column + # Drop entries for foreign plants with NaN values; remove federal_state capacities_list = capacities_list.dropna(subset=["nuts"]).drop( columns=["federal_state"] ) @@ -445,7 +438,7 @@ def aggr_nep_capacities(carriers): def map_carrier(): - """Map carriers from NEP and Marktstammdatenregister to carriers from eGon + """Map carriers from NEP/MaStR to eGon carriers. Returns ------- @@ -479,20 +472,18 @@ def map_carrier(): def insert_nep_list_powerplants(export=True): - """Insert list of conventional powerplants attached to the approval - of the scenario report by BNetzA + """Insert list of conventional power plants from BNetzA approval. Parameters ---------- export : bool - Choose if nep list should be exported to the data - base. The default is True. - If export=False a data frame will be returned + Choose if NEP list should be exported to the database. Default True. + If export=False a data frame will be returned. Returns ------- kw_liste_nep : pandas.DataFrame - List of conventional power plants from nep if export=False + List of conventional power plants from NEP if export=False """ sources = config.datasets()["scenario_input"]["sources"] @@ -501,7 +492,7 @@ def insert_nep_list_powerplants(export=True): # Connect to local database engine = db.engine() - # Read-in data from csv-file + # Read data from csv-file target_file = ( Path(".") / "data_bundle_egon_data" @@ -536,7 +527,7 @@ def insert_nep_list_powerplants(export=True): } ) - # Cut data to federal state if in testmode + # Cut data to federal state if in test mode boundary = config.settings()["egon-data"]["--dataset-boundary"] if boundary != "Everything": map_states = { @@ -570,7 +561,8 @@ def insert_nep_list_powerplants(export=True): "b2040_capacity", ]: kw_liste_nep.loc[ - kw_liste_nep[kw_liste_nep.federal_state.isnull()].index, col + kw_liste_nep[kw_liste_nep.federal_state.isnull()].index, + col, ] *= population_share() kw_liste_nep["carrier"] = map_carrier()[kw_liste_nep.carrier_nep].values @@ -588,7 +580,7 @@ def insert_nep_list_powerplants(export=True): def district_heating_input(): - """Imports data for district heating networks in Germany + """Import data for district heating networks in Germany. Returns ------- @@ -598,7 +590,7 @@ def district_heating_input(): sources = config.datasets()["scenario_input"]["sources"] - # import data to dataframe + # Import data to dataframe file = ( Path(".") / "data_bundle_egon_data" @@ -606,11 +598,13 @@ def district_heating_input(): / sources["eGon2035"]["capacities"] ) df = pd.read_excel( - file, sheet_name="Kurzstudie_KWK", dtype={"Wert": float} + file, + sheet_name="Kurzstudie_KWK", + dtype={"Wert": float}, ) df.set_index(["Energietraeger", "Name"], inplace=True) - # Scale values to population share in testmode + # Scale values to population share in test mode if config.settings()["egon-data"]["--dataset-boundary"] != "Everything": df.loc[ pd.IndexSlice[:, "Fernwaermeerzeugung"], "Wert" @@ -620,37 +614,49 @@ def district_heating_input(): engine = db.engine() session = sessionmaker(bind=engine)() - # insert heatpumps and resistive heater as link + # Insert heat pumps and resistive heaters as link for c in ["Grosswaermepumpe", "Elektrodenheizkessel"]: entry = EgonScenarioCapacities( component="link", scenario_name="eGon2035", nuts="DE", - carrier="urban_central_" - + ("heat_pump" if c == "Grosswaermepumpe" else "resistive_heater"), - capacity=df.loc[(c, "Fernwaermeerzeugung"), "Wert"] - * 1e6 - / df.loc[(c, "Volllaststunden"), "Wert"] - / df.loc[(c, "Wirkungsgrad"), "Wert"], + carrier=( + "urban_central_" + + ( + "heat_pump" + if c == "Grosswaermepumpe" + else "resistive_heater" + ) + ), + capacity=( + df.loc[(c, "Fernwaermeerzeugung"), "Wert"] + * 1e6 + / df.loc[(c, "Volllaststunden"), "Wert"] + / df.loc[(c, "Wirkungsgrad"), "Wert"] + ), ) session.add(entry) - # insert solar- and geothermal as generator + # Insert solar- and geothermal as generator for c in ["Geothermie", "Solarthermie"]: entry = EgonScenarioCapacities( component="generator", scenario_name="eGon2035", nuts="DE", - carrier="urban_central_" - + ( - "solar_thermal_collector" - if c == "Solarthermie" - else "geo_thermal" + carrier=( + "urban_central_" + + ( + "solar_thermal_collector" + if c == "Solarthermie" + else "geo_thermal" + ) + ), + capacity=( + df.loc[(c, "Fernwaermeerzeugung"), "Wert"] + * 1e6 + / df.loc[(c, "Volllaststunden"), "Wert"] ), - capacity=df.loc[(c, "Fernwaermeerzeugung"), "Wert"] - * 1e6 - / df.loc[(c, "Volllaststunden"), "Wert"], ) session.add(entry) @@ -659,7 +665,7 @@ def district_heating_input(): def insert_data_nep(): - """Overall function for importing scenario input data for eGon2035 scenario + """Overall function for importing scenario input data for eGon2035. Returns ------- @@ -673,7 +679,7 @@ def insert_data_nep(): def eGon100_capacities(): - """Inserts installed capacities for the eGon100 scenario + """Insert installed capacities for the eGon100 scenario. Returns ------- @@ -684,7 +690,7 @@ def eGon100_capacities(): sources = config.datasets()["scenario_input"]["sources"] targets = config.datasets()["scenario_input"]["targets"] - # read-in installed capacities + # Read installed capacities cwd = Path(".") if config.settings()["egon-data"]["--run-pypsa-eur"]: @@ -713,8 +719,15 @@ def eGon100_capacities(): ) df = pd.read_csv(target_file, delimiter=",", skiprows=3) - df.columns = ["component", "country", "carrier", "p_nom_2025", - "p_nom_2030", "p_nom_2035", "p_nom_2045"] + df.columns = [ + "component", + "country", + "carrier", + "p_nom_2025", + "p_nom_2030", + "p_nom_2035", + "p_nom_2045", + ] df.set_index("carrier", inplace=True) @@ -723,7 +736,7 @@ def eGon100_capacities(): # Drop country column df.drop("country", axis=1, inplace=True) - # Drop copmponents which will be optimized in eGo + # Drop components which will be optimized in eGo unused_carrier = [ "BEV charger", "DAC", @@ -767,8 +780,9 @@ def eGon100_capacities(): df.index = df.index.str.replace(" ", "_") # Aggregate offshore wind - df.loc["wind_offshore"] = df[df.index.str.startswith( - "offwind")].sum(numeric_only=True) + df.loc["wind_offshore"] = df[df.index.str.startswith("offwind")].sum( + numeric_only=True + ) df.loc["wind_offshore", "component"] = "generators" df = df.drop(df.index[df.index.str.startswith("offwind")]) @@ -793,36 +807,38 @@ def eGon100_capacities(): df.loc["solar", "component"] = "generators" df = df.drop(["solar-hsat"]) - # Aggregate technologies with and without carbon_capture (CC) - for carrier in ["urban_central_gas_CHP", "urban_central_solid_biomass_CHP"]: + # Aggregate tech with and without carbon_capture (CC) + for carrier in [ + "urban_central_gas_CHP", + "urban_central_solid_biomass_CHP", + ]: df.loc[ - carrier, - ["p_nom_2025", "p_nom_2030", "p_nom_2035", "p_nom_2045"]] += df.loc[ - f"{carrier}_CC", - ["p_nom_2025", "p_nom_2030", "p_nom_2035", "p_nom_2045"]] + carrier, ["p_nom_2025", "p_nom_2030", "p_nom_2035", "p_nom_2045"] + ] += df.loc[ + f"{carrier}_CC", + ["p_nom_2025", "p_nom_2030", "p_nom_2035", "p_nom_2045"], + ] df = df.drop([f"{carrier}_CC"]) # Aggregate urban decentral and rural heat supply - for merge_carrier in df.index[ - df.index.str.startswith("urban_decentral")]: - - # Add capacity of urban decentral to coresponding rural technology - df.loc[ - merge_carrier.replace("urban_decentral", "rural")] += df.loc[ - merge_carrier] + for merge_carrier in df.index[df.index.str.startswith("urban_decentral")]: + # Add capacity of urban decentral to corresponding rural technology + df.loc[merge_carrier.replace("urban_decentral", "rural")] += df.loc[ + merge_carrier + ] # Avoid summing up of component names df.loc[ - merge_carrier.replace("urban_decentral", "rural"), - "component"] = df.loc[ - merge_carrier, "component"] + merge_carrier.replace("urban_decentral", "rural"), "component" + ] = df.loc[merge_carrier, "component"] # Drop urban decentral technology df = df.drop(merge_carrier) # Aggregate rural air and rural ground heat pump - df.loc["rural_heat_pump"] = df.loc[ - "rural_air_heat_pump"] + df.loc["rural_ground_heat_pump"] + df.loc["rural_heat_pump"] = ( + df.loc["rural_air_heat_pump"] + df.loc["rural_ground_heat_pump"] + ) df.loc["rural_heat_pump", "component"] = "links" df = df.drop(["rural_air_heat_pump", "rural_ground_heat_pump"]) @@ -835,7 +851,7 @@ def eGon100_capacities(): "urban_central_solar_thermal": ( "urban_central_solar_thermal_collector" ), - "home_battery": "battery" + "home_battery": "battery", }, inplace=True, ) @@ -846,10 +862,22 @@ def eGon100_capacities(): # Insert target capacities for all years for year in ["2025", "2030", "2035", "2045"]: df_year = df.rename( - {f"p_nom_{year}": "capacity", "index": "carrier"}, axis="columns" + {f"p_nom_{year}": "capacity", "index": "carrier"}, + axis="columns", + ) + df_year.drop( + df_year.columns[ + ~df_year.columns.isin( + [ + "carrier", + "component", + "capacity", + ] + ) + ], + axis="columns", + inplace=True, ) - df_year.drop(df_year.columns[~df_year.columns.isin( - ["carrier", 'component', "capacity"])], axis="columns", inplace=True) if year == "2045": df_year["scenario_name"] = "eGon100RE" @@ -861,7 +889,8 @@ def eGon100_capacities(): db.execute_sql( f""" DELETE FROM - {targets['scenario_capacities']['schema']}.{targets['scenario_capacities']['table']} + {targets['scenario_capacities']['schema']}. + {targets['scenario_capacities']['table']} WHERE scenario_name='{df_year["scenario_name"].unique()[0]}' """ ) @@ -874,8 +903,9 @@ def eGon100_capacities(): index=False, ) + def add_metadata(): - """Add metdata to supply.egon_scenario_capacities + """Add metadata to supply.egon_scenario_capacities. Returns ------- @@ -883,7 +913,7 @@ def add_metadata(): """ - # Import column names and datatypes + # Import column names and data types fields = pd.DataFrame( generate_resource_fields_from_sqla_model(EgonScenarioCapacities) ).set_index("name") @@ -929,10 +959,16 @@ def add_metadata(): ], "licenses": [ license_ccby( - "© Übertragungsnetzbetreiber; " - "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " - "© Statistische Ämter des Bundes und der Länder 2014; " - "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + ( + "© Übertragungsnetzbetreiber; " + "© Bundesamt für Kartographie und Geodäsie 2020 " + "(Daten verändert); " + "© Statistische Ämter des Bundes und der Länder 2014; " + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, " + "Julian Endres, Carlos Epia, Stephan Günther, " + "Ulf Müller, Amélia Nadal, Guido Pleßmann, " + "Francesco Witte" + ) ) ], "contributors": [ @@ -959,7 +995,6 @@ def add_metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Create json dump @@ -972,6 +1007,7 @@ def add_metadata(): EgonScenarioCapacities.__table__.name, ) + tasks = (create_table,) scenarios = config.settings()["egon-data"]["--scenarios"] @@ -982,8 +1018,9 @@ def add_metadata(): if "status" in scenario: tasks += ( wrapped_partial( - insert_capacities_status_quo, scenario=scenario, - postfix=f"_{scenario[-2:]}" + insert_capacities_status_quo, + scenario=scenario, + postfix=f"_{scenario[-2:]}", ), ) status_quo = True @@ -1001,10 +1038,12 @@ class ScenarioCapacities(Dataset): """ Create and fill table with installed generation capacities in Germany - This dataset creates and fills a table with the installed generation capacities in - Germany in a lower spatial resolution (either per federal state or on national level). - This data is coming from external sources (e.g. German grid developement plan for scenario eGon2035). - The table is in downstream datasets used to define target values for the installed capacities. + This dataset creates and fills a table with the installed generation + capacities in Germany in a lower spatial resolution (either per federal + state or on national level). This data is coming from external sources + (e.g. German grid development plan for scenario eGon2035). The table is + used by downstream datasets to define target values for the installed + capacities. *Dependencies* @@ -1012,12 +1051,17 @@ class ScenarioCapacities(Dataset): * :py:class:`PypsaEurSec ` * :py:class:`Vg250 ` * :py:class:`DataBundle ` - * :py:class:`ZensusPopulation ` + * :py:class:`ZensusPopulation + ` *Resulting tables* - * :py:class:`supply.egon_scenario_capacities ` is created and filled - * :py:class:`supply.egon_nep_2021_conventional_powerplants ` is created and filled + * :py:class:`supply.egon_scenario_capacities + ` + is created and filled + * :py:class:`supply.egon_nep_2021_conventional_powerplants + ` + is created and filled """ @@ -1033,4 +1077,3 @@ def __init__(self, dependencies): dependencies=dependencies, tasks=tasks, ) - diff --git a/src/egon/data/datasets/storages/home_batteries.py b/src/egon/data/datasets/storages/home_batteries.py index 13df23ca5..06a7473e9 100644 --- a/src/egon/data/datasets/storages/home_batteries.py +++ b/src/egon/data/datasets/storages/home_batteries.py @@ -38,7 +38,6 @@ from loguru import logger from numpy.random import RandomState -from omi.dialects import get_dialect from sqlalchemy import Column, Float, Integer, String from sqlalchemy.ext.declarative import declarative_base import numpy as np @@ -52,7 +51,6 @@ generate_resource_fields_from_db_table, license_dedl, license_odbl, - meta_metadata, sources, ) @@ -108,10 +106,10 @@ def allocate_home_batteries_to_buildings(): WHERE carrier = 'home_battery' AND scenario = '{scenario}'; """ - cbat_pbat_ratio = get_sector_parameters( - "electricity", scenario - )["efficiency"]["battery"]["max_hours"] - + cbat_pbat_ratio = get_sector_parameters("electricity", scenario)[ + "efficiency" + ]["battery"]["max_hours"] + home_batteries_df = db.select_dataframe(sql) home_batteries_df = home_batteries_df.assign( @@ -270,9 +268,7 @@ def add_metadata(): "Data from Marktstammdatenregister (MaStR) data using " "the data dump from 2022-11-17 for eGon-data." ), - "path": ( - f"https://zenodo.org/record/{deposit_id_mastr}" - ), + "path": (f"https://zenodo.org/record/{deposit_id_mastr}"), "licenses": [license_dedl(attribution="© Amme, Jonathan")], }, sources()["openstreetmap"], @@ -306,7 +302,6 @@ def add_metadata(): } ], "review": {"path": "", "badge": ""}, - "metaMetadata": meta_metadata(), "_comment": { "metadata": ( "Metadata documentation and explanation (https://github.com/" @@ -334,9 +329,7 @@ def add_metadata(): }, } - dialect = get_dialect(f"oep-v{meta_metadata()['metadataVersion'][4:7]}")() - - meta = dialect.compile_and_render(dialect.parse(json.dumps(meta))) + meta = json.dumps(meta) db.submit_comment( f"'{json.dumps(meta)}'", diff --git a/src/egon/data/datasets/vg250/__init__.py b/src/egon/data/datasets/vg250/__init__.py index 92741a67d..3d0e01ac8 100644 --- a/src/egon/data/datasets/vg250/__init__.py +++ b/src/egon/data/datasets/vg250/__init__.py @@ -11,11 +11,10 @@ from pathlib import Path from urllib.request import urlretrieve -import time import datetime -import codecs import json import os +import time from geoalchemy2 import Geometry import geopandas as gpd @@ -23,12 +22,8 @@ from egon.data import db from egon.data.config import settings from egon.data.datasets import Dataset +from egon.data.metadata import context, licenses_datenlizenz_deutschland import egon.data.config -from egon.data.metadata import ( - context, - meta_metadata, - licenses_datenlizenz_deutschland, -) def download_files(): @@ -38,7 +33,6 @@ def download_files(): Data is downloaded from source specified in *datasets.yml* in section *vg250/original_data/source/url* and saved to file specified in *vg250/original_data/target/file*. - """ data_config = egon.data.config.datasets() vg250_config = data_config["vg250"]["original_data"] @@ -61,7 +55,6 @@ def to_postgres(): Creates schema boundaries if it does not yet exist. Newly creates all tables specified as keys in *datasets.yml* in section *vg250/processed/file_table_map*. - """ # Get information from data configuration file @@ -79,16 +72,19 @@ def to_postgres(): for filename, table in vg250_processed["file_table_map"].items(): # Open files and read .shp (within .zip) with geopandas data = gpd.read_file( - f"zip://{zip_file}!vg250_01-01.geo84.shape.ebenen/" - f"vg250_ebenen_0101/{filename}" + "zip://" + f"{zip_file}!vg250_01-01.geo84.shape.ebenen/" + "vg250_ebenen_0101/" + f"{filename}" ) boundary = settings()["egon-data"]["--dataset-boundary"] if boundary != "Everything": # read-in borders of federal state Schleswig-Holstein data_sta = gpd.read_file( - f"zip://{zip_file}!vg250_01-01.geo84.shape.ebenen/" - f"vg250_ebenen_0101/VG250_LAN.shp" + "zip://" + f"{zip_file}!vg250_01-01.geo84.shape.ebenen/" + "vg250_ebenen_0101/VG250_LAN.shp" ).query(f"GEN == '{boundary}'") data_sta.BEZ = "Bundesrepublik" data_sta.NUTS = "DE" @@ -107,7 +103,7 @@ def to_postgres(): # Drop table before inserting data db.execute_sql( - f"DROP TABLE IF EXISTS " + "DROP TABLE IF EXISTS " f"{vg250_processed['schema']}.{table} CASCADE;" ) @@ -123,7 +119,7 @@ def to_postgres(): db.execute_sql( f"ALTER TABLE {vg250_processed['schema']}.{table} " - f"ADD PRIMARY KEY (id);" + "ADD PRIMARY KEY (id);" ) # Add index on geometry column @@ -145,47 +141,63 @@ def add_metadata(): }, "vg250_lan": { "title": "BKG - Verwaltungsgebiete 1:250.000 - Länder (LAN)", - "description": "Landesgrenzen der Bundesländer in der " - "Bundesrepublik Deutschland", + "description": ( + "Landesgrenzen der Bundesländer in der " + "Bundesrepublik Deutschland" + ), }, "vg250_rbz": { - "title": "BKG - Verwaltungsgebiete 1:250.000 - Regierungsbezirke " - "(RBZ)", - "description": "Grenzen der Regierungsbezirke in der " - "Bundesrepublik Deutschland", + "title": ( + "BKG - Verwaltungsgebiete 1:250.000 - " + "Regierungsbezirke (RBZ)" + ), + "description": ( + "Grenzen der Regierungsbezirke in der " + "Bundesrepublik Deutschland" + ), }, "vg250_krs": { "title": "BKG - Verwaltungsgebiete 1:250.000 - Kreise (KRS)", - "description": "Grenzen der Landkreise in der " - "Bundesrepublik Deutschland", + "description": ( + "Grenzen der Landkreise in der " "Bundesrepublik Deutschland" + ), }, "vg250_vwg": { - "title": "BKG - Verwaltungsgebiete 1:250.000 - " - "Verwaltungsgemeinschaften (VWG)", - "description": "Grenzen der Verwaltungsgemeinschaften in der " - "Bundesrepublik Deutschland", + "title": ( + "BKG - Verwaltungsgebiete 1:250.000 - " + "Verwaltungsgemeinschaften (VWG)" + ), + "description": ( + "Grenzen der Verwaltungsgemeinschaften in der " + "Bundesrepublik Deutschland" + ), }, "vg250_gem": { "title": "BKG - Verwaltungsgebiete 1:250.000 - Gemeinden (GEM)", - "description": "Grenzen der Gemeinden in der " - "Bundesrepublik Deutschland", + "description": ( + "Grenzen der Gemeinden in der " "Bundesrepublik Deutschland" + ), }, } licenses = [ licenses_datenlizenz_deutschland( - attribution="© Bundesamt für Kartographie und Geodäsie " - "2020 (Daten verändert)" + attribution=( + "© Bundesamt für Kartographie und Geodäsie 2020 " + "(Daten verändert)" + ) ) ] vg250_source = { "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", - "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der " - "hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden " - "mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der " - "Verwaltungseinheit sowie die spezifische Bezeichnung der " - "Verwaltungsebene des jeweiligen Landes.", + "description": ( + "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der " + "hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden " + "mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der " + "Verwaltungseinheit sowie die spezifische Bezeichnung der " + "Verwaltungsebene des jeweiligen Landes." + ), "path": vg250_config["original_data"]["source"]["url"], "licenses": licenses, } @@ -248,10 +260,11 @@ def add_metadata(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } - meta_json = "'" + json.dumps(meta) + "'" + meta_json = ( + "'" + json.dumps(meta) + "'" + ) # TODO @jh: Find me and remove db.submit_comment( meta_json, vg250_config["processed"]["schema"], table @@ -261,10 +274,12 @@ def add_metadata(): def nuts_mview(): """ Creates MView boundaries.vg250_lan_nuts_id. - """ db.execute_sql_script( - os.path.join(os.path.dirname(__file__), "vg250_lan_nuts_id_mview.sql") + os.path.join( + os.path.dirname(__file__), + "vg250_lan_nuts_id_mview.sql", + ) ) @@ -273,8 +288,8 @@ def cleaning_and_preperation(): Creates tables and MViews with cleaned and corrected geometry data. The following table is created: - * boundaries.vg250_gem_clean where municipalities (Gemeinden) that are fragmented - are cleaned from ringholes + * boundaries.vg250_gem_clean where municipalities (Gemeinden) that + are fragmented are cleaned from ringholes The following MViews are created: * boundaries.vg250_gem_hole @@ -285,18 +300,19 @@ def cleaning_and_preperation(): * boundaries.vg250_sta_invalid_geometry * boundaries.vg250_sta_tiny_buffer * boundaries.vg250_sta_union - """ db.execute_sql_script( - os.path.join(os.path.dirname(__file__), "cleaning_and_preparation.sql") + os.path.join( + os.path.dirname(__file__), + "cleaning_and_preparation.sql", + ) ) def vg250_metadata_resources_fields(): """ Returns metadata string for VG250 tables. - """ return [ @@ -451,7 +467,10 @@ def vg250_metadata_resources_fields(): "unit": "none", }, { - "description": "Seat of the administration (territorial code, deprecated column)", + "description": ( + "Seat of the administration (territorial code, " + "deprecated column)" + ), "name": "sdv_rs", "type": "string", "unit": "none", @@ -475,8 +494,8 @@ class Vg250(Dataset): """ Obtains and processes VG250 data and writes it to database. - Original data is downloaded using :py:func:`download_files` function and written - to database using :py:func:`to_postgres` function. + Original data is downloaded using :py:func:`download_files` + and written to database using :py:func:`to_postgres`. *Dependencies* No dependencies @@ -488,25 +507,26 @@ class Vg250(Dataset): * :py:func:`boundaries.vg250_rbz ` is created and filled * :py:func:`boundaries.vg250_sta ` is created and filled * :py:func:`boundaries.vg250_vwg ` is created and filled - * :py:func:`boundaries.vg250_lan_nuts_id ` is created and filled - * :py:func:`boundaries.vg250_gem_hole ` is created and - filled - * :py:func:`boundaries.vg250_gem_valid ` is created and - filled - * :py:func:`boundaries.vg250_krs_area ` is created and - filled - * :py:func:`boundaries.vg250_lan_union ` is created and - filled - * :py:func:`boundaries.vg250_sta_bbox ` is created and - filled - * :py:func:`boundaries.vg250_sta_invalid_geometry ` is - created and filled - * :py:func:`boundaries.vg250_sta_tiny_buffer ` is - created and filled - * :py:func:`boundaries.vg250_sta_union ` is - created and filled - + * :py:func:`boundaries.vg250_lan_nuts_id ` is created + and filled + * :py:func:`boundaries.vg250_gem_hole ` + is created and filled + * :py:func:`boundaries.vg250_gem_valid ` + is created and filled + * :py:func:`boundaries.vg250_krs_area ` + is created and filled + * :py:func:`boundaries.vg250_lan_union ` + is created and filled + * :py:func:`boundaries.vg250_sta_bbox ` + is created and filled + * :py:func:`boundaries.vg250_sta_invalid_geometry + ` is created and filled + * :py:func:`boundaries.vg250_sta_tiny_buffer + ` is created and filled + * :py:func:`boundaries.vg250_sta_union ` + is created and filled """ + filename = egon.data.config.datasets()["vg250"]["original_data"]["source"][ "url" ] diff --git a/src/egon/data/datasets/zensus_vg250.py b/src/egon/data/datasets/zensus_vg250.py index e5d39906e..e1b2d53e4 100755 --- a/src/egon/data/datasets/zensus_vg250.py +++ b/src/egon/data/datasets/zensus_vg250.py @@ -3,16 +3,7 @@ import time from geoalchemy2 import Geometry -from sqlalchemy import ( - BigInteger, - Column, - Float, - Integer, - SmallInteger, - String, - func, - select, -) +from sqlalchemy import BigInteger, Column, Float, Integer, SmallInteger, String from sqlalchemy.ext.declarative import declarative_base import geopandas as gpd import pandas as pd @@ -22,10 +13,8 @@ from egon.data.datasets.vg250 import vg250_metadata_resources_fields from egon.data.metadata import ( context, - generate_resource_fields_from_sqla_model, license_ccby, licenses_datenlizenz_deutschland, - meta_metadata, sources, ) import egon.data.config @@ -183,20 +172,28 @@ def map_zensus_vg250(): local_engine = db.engine() db.execute_sql( - f"""DELETE FROM - {cfg['targets']['map']['schema']}.{cfg['targets']['map']['table']}""" + f""" + DELETE FROM + {cfg['targets']['map']['schema']}. + {cfg['targets']['map']['table']} + """ ) gdf = db.select_geodataframe( - f"""SELECT * FROM + f""" + SELECT * FROM {cfg['sources']['zensus_population']['schema']}. - {cfg['sources']['zensus_population']['table']}""", + {cfg['sources']['zensus_population']['table']} + """, geom_col="geom_point", ) gdf_boundaries = db.select_geodataframe( - f"""SELECT * FROM {cfg['sources']['vg250_municipalities']['schema']}. - {cfg['sources']['vg250_municipalities']['table']}""", + f""" + SELECT * FROM + {cfg['sources']['vg250_municipalities']['schema']}. + {cfg['sources']['vg250_municipalities']['table']} + """, geom_col="geometry", epsg=3035, ) @@ -264,7 +261,10 @@ def inside_germany(): # Create new table db.execute_sql( f""" - DROP TABLE IF EXISTS {DestatisZensusPopulationPerHaInsideGermany.__table__.schema}.{DestatisZensusPopulationPerHaInsideGermany.__table__.name} CASCADE; + DROP TABLE IF EXISTS + {DestatisZensusPopulationPerHaInsideGermany.__table__.schema}. + {DestatisZensusPopulationPerHaInsideGermany.__table__.name} + CASCADE; """ ) DestatisZensusPopulationPerHaInsideGermany.__table__.create( @@ -290,7 +290,7 @@ def inside_germany(): ) # Insert above queried data into new table - insert = DestatisZensusPopulationPerHaInsideGermany.__table__.insert().from_select( + insert = DestatisZensusPopulationPerHaInsideGermany.__table__.insert().from_select( # noqa: E501 ( DestatisZensusPopulationPerHaInsideGermany.id, DestatisZensusPopulationPerHaInsideGermany.grid_id, @@ -325,16 +325,18 @@ def population_in_municipalities(): ) gem["area_ha"] = gem.area / 10000 - gem["area_km2"] = gem.area / 1000000 population = db.select_dataframe( - """SELECT id, population, vg250_municipality_id + """ + SELECT id, population, vg250_municipality_id FROM society.destatis_zensus_population_per_ha INNER JOIN boundaries.egon_map_zensus_vg250 ON ( - society.destatis_zensus_population_per_ha.id = - boundaries.egon_map_zensus_vg250.zensus_population_id) - WHERE population > 0""" + society.destatis_zensus_population_per_ha.id = + boundaries.egon_map_zensus_vg250.zensus_population_id + ) + WHERE population > 0 + """ ) gem["population_total"] = ( @@ -359,7 +361,8 @@ def add_metadata_zensus_inside_ger(): """ Create metadata JSON for DestatisZensusPopulationPerHaInsideGermany - Creates a metdadata JSON string and writes it to the database table comment + Creates a metdadata JSON string and writes it to the database table + comment """ schema_table = ".".join( [ @@ -373,8 +376,8 @@ def add_metadata_zensus_inside_ger(): "title": "DESTATIS - Zensus 2011 - Population per hectar", "id": "WILL_BE_SET_AT_PUBLICATION", "description": ( - "National census in Germany in 2011 with the bounds on Germanys " - "borders." + "National census in Germany in 2011 with the bounds on " + "Germanys borders." ), "language": ["en-EN", "de-DE"], "publicationDate": datetime.date.today().isoformat(), @@ -396,55 +399,69 @@ def add_metadata_zensus_inside_ger(): }, "sources": [ { - "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des " - "Zensus 2011 zum Download", + "title": ( + "Statistisches Bundesamt (Destatis) - Ergebnisse des " + "Zensus 2011 zum Download" + ), "description": ( - "Als Download bieten wir Ihnen auf dieser Seite " - "zusätzlich zur Zensusdatenbank CSV- und " - "teilweise Excel-Tabellen mit umfassenden " - "Personen-, Haushalts- und Familien- sowie " - "Gebäude- und Wohnungsmerkmalen. Die " - "Ergebnisse liegen auf Bundes-, Länder-, Kreis- " - "und Gemeindeebene vor. Außerdem sind einzelne " - "Ergebnisse für Gitterzellen verfügbar." + "Als Download bieten wir Ihnen auf dieser Seite zusä-" + "tzlich zur Zensusdatenbank CSV- und teilweise " + "Excel-Tabellen mit umfassenden Personen-, Haushalts- " + "und Familien- " + "sowie Gebäude- und Wohnungsmerkmalen. Die Ergebnisse " + "liegen auf Bundes-, Länder-, Kreis- und Gemeindeebene " + "vor. Außerdem sind einzelne Ergebnisse für Gitterzellen " + "verfügbar." + ), + "path": ( + "https://www.zensus2011.de/DE/Home/Aktuelles/" + "DemografischeGrunddaten.html" ), - "path": "https://www.zensus2011.de/DE/Home/Aktuelles/" - "DemografischeGrunddaten.html", "licenses": [ licenses_datenlizenz_deutschland( - attribution="© Statistische Ämter des Bundes und der " - "Länder 2014" + attribution=( + "© Statistische Ämter des Bundes und der Länder " + "2014" + ) ) ], }, { - "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "title": ( + "Dokumentation - Zensus 2011 - Methoden und Verfahren" + ), "description": ( - "Diese Publikation beschreibt ausführlich die " - "Methoden und Verfahren des registergestützten " - "Zensus 2011; von der Datengewinnung und " - "-aufbereitung bis hin zur Ergebniserstellung" - " und Geheimhaltung. Der vorliegende Band wurde " - "von den Statistischen Ämtern des Bundes und " + "Diese Publikation beschreibt ausführlich die Methoden " + "und " + "Verfahren des registergestützten Zensus 2011; von der " + "Datengewinnung und -aufbereitung bis hin zur " + "Ergebniserstellung und Geheimhaltung. Der vorliegende " + "Band wurde von den Statistischen Ämtern des Bundes und " "der Länder im Juni 2015 veröffentlicht." ), - "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Be" - "voelkerung/Zensus/ZensusBuLaMethodenVerfahren51211051" - "19004.pdf?__blob=publicationFile", + "path": ( + "https://www.destatis.de/DE/Publikationen/Thematisch/" + "Bevoelkerung/Zensus/" + "ZensusBuLaMethodenVerfahren5121105119004.pdf" + "?__blob=publicationFile" + ), "licenses": [ licenses_datenlizenz_deutschland( - attribution="© Statistisches Bundesamt, Wiesbaden " - "2015 (im Auftrag der " - "Herausgebergemeinschaft)" + attribution=( + "© Statistisches Bundesamt, Wiesbaden 2015 " + "(im Auftrag der Herausgebergemeinschaft)" + ) ) ], }, ], "licenses": [ licenses_datenlizenz_deutschland( - attribution="© Statistische Ämter des Bundes und der Länder " - "2014; © Statistisches Bundesamt, Wiesbaden 2015 " - "(Daten verändert)" + attribution=( + "© Statistische Ämter des Bundes und der Länder 2014; " + "© Statistisches Bundesamt, Wiesbaden 2015 " + "(Daten verändert)" + ) ) ], "contributors": [ @@ -486,7 +503,7 @@ def add_metadata_zensus_inside_ger(): }, { "name": "population", - "description": "Number of registred residents", + "description": ("Number of registred residents"), "type": "integer", "unit": "resident", }, @@ -509,7 +526,6 @@ def add_metadata_zensus_inside_ger(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(metadata) + "'" @@ -525,7 +541,8 @@ def add_metadata_vg250_gem_pop(): """ Create metadata JSON for Vg250GemPopulation - Creates a metdadata JSON string and writes it to the database table comment + Creates a metdadata JSON string and writes it to the database table + comment """ vg250_config = egon.data.config.datasets()["vg250"] schema_table = ".".join( @@ -537,18 +554,22 @@ def add_metadata_vg250_gem_pop(): licenses = [ licenses_datenlizenz_deutschland( - attribution="© Bundesamt für Kartographie und Geodäsie " - "2020 (Daten verändert)" + attribution=( + "© Bundesamt für Kartographie und Geodäsie 2020 " + "(Daten verändert)" + ) ) ] vg250_source = { "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", - "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der " - "hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden " - "mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der " - "Verwaltungseinheit sowie die spezifische Bezeichnung der " - "Verwaltungsebene des jeweiligen Landes.", + "description": ( + "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der " + "hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden " + "mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der " + "Verwaltungseinheit sowie die spezifische Bezeichnung der " + "Verwaltungsebene des jeweiligen Landes." + ), "path": vg250_config["original_data"]["source"]["url"], "licenses": licenses, } @@ -648,7 +669,6 @@ def add_metadata_vg250_gem_pop(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } meta_json = "'" + json.dumps(metadata) + "'" @@ -709,9 +729,13 @@ def add_metadata_vg250_zensus(): ], "licenses": [ license_ccby( - "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert); " + "© Bundesamt für Kartographie und Geodäsie 2020 " + "(Daten verändert); " "© Statistische Ämter des Bundes und der Länder 2014 " - "© Jonathan Amme, Clara Büttner, Ilka Cußmann, Julian Endres, Carlos Epia, Stephan Günther, Ulf Müller, Amélia Nadal, Guido Pleßmann, Francesco Witte", + "© Jonathan Amme, Clara Büttner, Ilka Cußmann, " + "Julian Endres, Carlos Epia, Stephan Günther, " + "Ulf Müller, Amélia Nadal, Guido Pleßmann, " + "Francesco Witte" ) ], "contributors": [ @@ -738,7 +762,6 @@ def add_metadata_vg250_zensus(): "dialect": {"delimiter": None, "decimalSeparator": "."}, } ], - "metaMetadata": meta_metadata(), } # Create json dump From c44e6939492c6e983db517302fad4040edd3d33b Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 21:13:13 +0200 Subject: [PATCH 09/32] #1177 oem v2 update: changes are not relevant just make the current implementation a bit more stable until it is teared down --- src/egon/data/metadata/__init__.py | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index 042d61580..1add3ff74 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -1,5 +1,7 @@ +from copy import deepcopy + from geoalchemy2 import Geometry -from omi.base import get_metadata_specification +from omi.base import MetadataSpecification, get_metadata_specification from omi.validation import parse_metadata, validate_metadata from sqlalchemy import MetaData, Table from sqlalchemy.dialects.postgresql.base import ischema_names @@ -12,7 +14,35 @@ # Easy access to oemetadata schema, template and example Dicts # Uses the oemetadata version specified in settings module -OEMetaData = get_metadata_specification(settings.OEMETADATA_VERSION) +OEMetaData: MetadataSpecification = get_metadata_specification( + settings.OEMETADATA_VERSION +) + + +def oem_datapackage_template() -> dict: + """ + Provides a clean interface to the oemetadata template. + + the template is used to create a new metadata. It provides + all oemetadata properties, including nested fields with empty + -> "" values assigned + + Returns + ------- + dict + OEP metadata conform template for new metadata + """ + + template: dict + + if OEMetaData.template: + template: dict = deepcopy(OEMetaData.template) + + if OEMetaData.example: + template["@context"] = OEMetaData.example["@context"] + template["metaMetadata"] = OEMetaData.example["metaMetadata"] + + return template def context(): From 8597954a1c5bdbf3b1aa26c69026a9b3d84fbbf2 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 21:13:58 +0200 Subject: [PATCH 10/32] =?UTF-8?q?#1177=20oem=20v2=20update:=20Add=20module?= =?UTF-8?q?=20readme=C2=B4s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/egon/data/metadata/README.md | 23 +++++++++++ src/egon/data/metadata/overlays/README.md | 49 +++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/egon/data/metadata/README.md create mode 100644 src/egon/data/metadata/overlays/README.md diff --git a/src/egon/data/metadata/README.md b/src/egon/data/metadata/README.md new file mode 100644 index 000000000..4dda823ef --- /dev/null +++ b/src/egon/data/metadata/README.md @@ -0,0 +1,23 @@ +# egon-data Metadata + +The metadata module provides all metadata related functionality used to create/update metadata for all datasets used in the eGon-data pipeline. Functionality defined will be imported into the dataset task. + +The metadata module is/offers: + + - the central place to configuring the metadata generation + - it provides a metadata-template system which applies information like the project context description to all dataset resources by default to ease handling redundant information. + - Metadata can be setup using YAML files or dict data structures when working in python code using the OMI package + - A central functionality to upload metadata to the internal database as SQL comment on table <---## We could also use a jsonb column to avoid parsing metadata from string + - Provides a simple way to fill the metadata structure with content while keep oemetadata specification compliance + - Validate metadata against the json schema specification + - Create a single datapackage which lists all metadata from its resources as defined in a dataset dependency list. It represents the dataset which is then either patly or fully published on the OEP. + - Store generated metadata + +The metadata module is used in dataset modules: + + - To add individual metadata elements using the predefined structure offered by the metadata module + - Add information about the data model derived from the technical schema as implemented in the database reading form sqlalchemy definitions + +Additional use cases + + - Well described metadata is used in the process to upload datasets to the OEP diff --git a/src/egon/data/metadata/overlays/README.md b/src/egon/data/metadata/overlays/README.md new file mode 100644 index 000000000..d72587b1f --- /dev/null +++ b/src/egon/data/metadata/overlays/README.md @@ -0,0 +1,49 @@ +# Description + +This directory is called “overlays” because it stored content for oemetadata generation oemetadata. The overlays job is to layer human-authored bits (titles, context, licenses, field hints) on top of what we auto-extract from the DB—rather than being the single source of truth. That mental model is base metadata → overlay additional metadata. Overlay can override what is added in previous steps. + +## Notes + +Much of the functionality does not have to be implemented in egon-data as it is general purpose and should be added to omi. Currently we have a conflict with the python version. + +Also omi provides functionality to add template metadata. In the example below you see an entry for the context. In most cases the Project will be the same for all datasets this is when templates come in handy as we can apply them to all datasets in one go. If you specify the context here then it will override the template entry for that specific dataset. This is why the overlay can also be viewed as individual metadata. + +## Usage + +For each dataset used in the pipeline there should be metadata created. If we can provide additional metadata which cant be auto inferred from data sources then we use this directory and add yaml files. + +A reasonable "minimal" yaml file would be what you see below. Keep in mind that this minimal content is not what is required. Here minimal refers more to what would be at least useful to add. + +Example for zensus_vg250.yaml + +```yaml +name: "society.destatis_zensus_population_per_ha_inside_germany" +title: "DESTATIS – Zensus 2011 – Population per hectare (inside Germany)" +description: > + National census in Germany in 2011 filtered to German borders and cells with population > 0. +language: ["de-DE", "en-EN"] +context: + homepage: "https://ego-n.org/" + documentation: "https://egon-data.readthedocs.io/en/latest/" + sourceCode: "https://github.com/openego/eGon-data" + contact: "https://ego-n.org/partners/" +spatial: + extent: "Germany" + resolution: "1 ha" +temporal: + referenceDate: "2011-12-31" +sources: + - title: "Statistisches Bundesamt (Destatis) – Ergebnisse des Zensus 2011 zum Download" + path: "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html" +licenses: + - name: "DL-DE-BY-2.0" + title: "Datenlizenz Deutschland – Namensnennung – Version 2.0" + path: "https://www.govdata.de/dl-de/by-2-0" +contributors: + - title: "Guido Pleßmann" + email: "http://github.com/gplssm" + comment: "Imported data" + - title: "Jonathan Amme" + email: "http://github.com/nesnoj" + comment: "Metadata extended" +``` From 59213791c271a79d6e30e7ca0c915c6c894a02cc Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 13 Aug 2025 21:19:07 +0200 Subject: [PATCH 11/32] #1177 #1305 oem v2 update; - Builder to generate, create and customize oemetadata for egon-data datasets - Implement draft builder tool -> will be enhanced further by using mixins to reduce class complexity - The oemetadataBuilder will become a more complex module and spit into a resource and datapackage builder --- src/egon/data/metadata/oemetadata.py | 407 +++++++++++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 src/egon/data/metadata/oemetadata.py diff --git a/src/egon/data/metadata/oemetadata.py b/src/egon/data/metadata/oemetadata.py new file mode 100644 index 000000000..667753a69 --- /dev/null +++ b/src/egon/data/metadata/oemetadata.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Dict, List, Optional +import datetime as dt +import json + +from geoalchemy2 import Geometry +from omi.base import MetadataSpecification, get_metadata_specification +from omi.validation import validate_metadata # parse_metadata +from sqlalchemy import MetaData, Table, inspect +from sqlalchemy.dialects.postgresql.base import ischema_names +from sqlalchemy.engine import Engine +import yaml # PyYAML + +# ---- Optional: your project settings/hooks +# from egon.data import db, logger +from egon.data.metadata import settings + +# Geometry awareness for reflection +ischema_names["geometry"] = Geometry # generic +# You can add specific geometry columns later per-table via kwargs + + +def _today() -> str: + return dt.date.today().isoformat() + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Deep merge with 'override wins', recursively. + Lists are replaced (not merged) by default to avoid subtle duplication. + """ + out = deepcopy(base) + for k, v in override.items(): + if isinstance(v, dict) and isinstance(out.get(k), dict): + out[k] = _deep_merge(out[k], v) + else: + out[k] = deepcopy(v) + return out + + +def _sqlatype_to_oem_type(sa_type: str) -> str: + """ + Map SQLAlchemy reflected type string -> OEM v2 field.type + Keep it simple and deterministic; adjust as needed. + """ + t = sa_type.lower() + # geometry + if "geometry" in t: + return "geometry" + # integers + if any(x in t for x in ["int", "serial", "bigint", "smallint"]): + return "integer" + # floats / numeric + if any(x in t for x in ["float", "double", "numeric", "real", "decimal"]): + return "number" + # booleans + if "bool" in t: + return "boolean" + # timestamp/date/time + if "timestamp" in t or "timestamptz" in t: + return "datetime" + if t == "date": + return "date" + if t == "time": + return "time" + # text-ish + if any( + x in t for x in ["text", "char", "string", "uuid", "json", "jsonb"] + ): + return "string" + # fallback + return "string" + + +@dataclass +class ResourceField: + """ + Minimal implementation of oemetadata v2 resource structure. + Eases usage in Python. + """ + + name: str + description: Optional[str] = None + type: str = "string" + unit: Optional[str] = None + nullable: Optional[bool] = None + + def to_dict(self) -> dict: + d = { + "name": self.name, + "type": self.type, + } + # include optional keys only when provided + if self.description is not None: + d["description"] = self.description + if self.unit is not None: + d["unit"] = self.unit + if self.nullable is not None: + d["nullable"] = self.nullable + return d + + +class OEMetadataBuilder: + """ + Single, reusable builder for OEP oemetadata v2 using omi as source of truth. + + Typical flow: + builder = OEMetadataBuilder().from_template() + .apply_yaml("dataset_meta.yaml") + .auto_resource_from_table(engine, "schema", "table", geom_cols=["geom"]) + .set_basic(name="schema.table", title="...", description="...") + .finalize() + payload = builder.as_json() # validated JSON string + builder.save_as_table_comment(db_engine, "schema", "table") # optional + """ # noqa: E501 + + def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: + self.spec: MetadataSpecification = get_metadata_specification(version) + self._meta: Dict[str, Any] = {} + self._validated: bool = False + + # ---- Required steps + + def from_template(self) -> "OEMetadataBuilder": + """ + Start from omi's template plus selected bits from example + (context/metaMetadata). + Ensures keys exist (empty strings/structures as per spec). + """ + tpl = deepcopy(self.spec.template) if self.spec.template else {} + if self.spec.example: + # Copy @context + metaMetadata if present in example + if "@context" in self.spec.example: + tpl["@context"] = deepcopy(self.spec.example["@context"]) + if "metaMetadata" in self.spec.example: + tpl["metaMetadata"] = deepcopy( + self.spec.example["metaMetadata"] + ) + self._meta = tpl["resources"][0] + self._validated = False + return self + + def apply_yaml( + self, yaml_path: str | None = None, yaml_text: str | None = None + ) -> "OEMetadataBuilder": + """ + Merge user-provided YAML overrides into the current metadata object. + You can allow either a file path or a YAML string (for testing). + """ + if yaml_path: + with open(yaml_path, "r", encoding="utf-8") as fh: + override = yaml.safe_load(fh) or {} + elif yaml_text: + override = yaml.safe_load(yaml_text) or {} + else: + override = {} + + self._meta = _deep_merge(self._meta, override) + self._validated = False + return self + + def set_basic( + self, + name: str, + title: Optional[str] = None, + description: Optional[str] = None, + language: Optional[List[str]] = None, + publication_date: Optional[str] = None, + dataset_id: Optional[str] = None, + ) -> "OEMetadataBuilder": + """ + Convenience setter for common top-level fields. + """ + if publication_date is None: + publication_date = _today() + patch = { + "name": name, + "publicationDate": publication_date, + } + if title is not None: + patch["title"] = title + if description is not None: + patch["description"] = description + if language is not None: + patch["language"] = language + if dataset_id is not None: + patch["id"] = dataset_id + + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + def set_context(self, context_obj: dict) -> "OEMetadataBuilder": + self._meta = _deep_merge(self._meta, {"context": context_obj}) + self._validated = False + return self + + def set_spatial( + self, + extent: Optional[str] = None, + resolution: Optional[str] = None, + location: Optional[Any] = None, + ) -> "OEMetadataBuilder": + patch = {"spatial": {}} + if location is not None: + patch["spatial"]["location"] = location + if extent is not None: + patch["spatial"]["extent"] = extent + if resolution is not None: + patch["spatial"]["resolution"] = resolution + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + def set_temporal( + self, + reference_date: Optional[str] = None, + timeseries: Optional[dict] = None, + ) -> "OEMetadataBuilder": + patch = {"temporal": {}} + if reference_date is not None: + # NOTE: your older code used 'referenceDate' vs + # 'reference_date' in places. + # OEM v2 uses 'referenceDate' (camelCase). Keep consistent here: + patch["temporal"]["referenceDate"] = reference_date + if timeseries is not None: + patch["temporal"]["timeseries"] = timeseries + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + # ---- Sources, licenses, contributors + + def add_source(self, source: dict) -> "OEMetadataBuilder": + self._meta.setdefault("sources", []) + self._meta["sources"].append(source) + self._validated = False + return self + + def add_license(self, lic: dict) -> "OEMetadataBuilder": + self._meta.setdefault("licenses", []) + self._meta["licenses"].append(lic) + self._validated = False + return self + + def add_contributor(self, contributor: dict) -> "OEMetadataBuilder": + self._meta.setdefault("contributors", []) + self._meta["contributors"].append(contributor) + self._validated = False + return self + + # ---- Resources + + def auto_resource_from_table( + self, + engine: Engine, + schema: str, + table: str, + *, + resource_name: Optional[str] = None, + format_: str = "PostgreSQL", + encoding: str = "UTF-8", + primary_key: Optional[List[str]] = None, + foreign_keys: Optional[List[dict]] = None, + geom_cols: Optional[List[str]] = None, + dialect: Optional[dict] = None, + overwrite_existing: bool = False, + ) -> "OEMetadataBuilder": + """ + Introspect a DB table and create a single tabular data resource entry. + + - Maps SQLA types to OEM types + - Marks 'nullable' where possible + - Recognizes geometry columns (if given in geom_cols) as 'geometry' + + If overwrite_existing=False and a resource already exists with the same + name, it will be left as-is (you could add a flag to update instead). + """ + if geom_cols is None: + geom_cols = ["geom", "geometry", "geom_point", "geom_polygon"] + + # reflect + meta = MetaData() + tbl = Table(table, meta, schema=schema, autoload_with=engine) + + fields: List[ResourceField] = [] + for col in tbl.columns: + sa_t = str(col.type) + # if explicitly geometry by name, treat as geometry + col_type = ( + "geometry" + if col.name in geom_cols + else _sqlatype_to_oem_type(sa_t) + ) + fields.append( + ResourceField( + name=col.name, + description=None, + type=col_type, + unit=None, + nullable=col.nullable, + ) + ) + + if not resource_name: + resource_name = f"{schema}.{table}" + + resource = { + "name": resource_name, + # TODO: @jh-RLI The OEP will set this, + # consider if local usage is important + "path": None, + "type": "table", + "format": format_, + "encoding": encoding, + "schema": { + "fields": [f.to_dict() for f in fields], + "primaryKey": primary_key + or self._best_guess_pk(engine, schema, table), + "foreignKeys": foreign_keys or [], + }, + "dialect": dialect or {"delimiter": None, "decimalSeparator": "."}, + } + + # install resources array + self._meta.setdefault("resources", []) + if overwrite_existing: + self._meta["resources"] = [ + r + for r in self._meta["resources"] + if r.get("name") != resource_name + ] + # only add if not present + if not any( + r.get("name") == resource_name for r in self._meta["resources"] + ): + self._meta["resources"].append(resource) + + self._validated = False + return self + + def _best_guess_pk( + self, engine: Engine, schema: str, table: str + ) -> List[str]: + """ + Try to read PK columns via SQLAlchemy inspector, fallback to + ['id'] if found, else []. + """ + insp = inspect(engine) + pk = insp.get_pk_constraint(table, schema=schema) + cols = pk.get("constrained_columns") if pk else None + if cols: + return cols + # common fallback + columns = [c["name"] for c in insp.get_columns(table, schema=schema)] + return ["id"] if "id" in columns else [] + + # ---- Finalize/validate/serialize + + def finalize(self, license_check: bool = False) -> "OEMetadataBuilder": + """ + Make minimal guarantees & validate with omi. + """ + # Fill sane defaults if missing + # self._meta.setdefault("publicationDate", _today()) + self._meta.setdefault("language", ["en-EN"]) + + # TODO: @jh-RLI might be expensive + # parse + validate with omi + # parse_metadata expects string; serialize & round-trip to normalize + # text = json.dumps(self._meta, ensure_ascii=False) + # parsed = parse_metadata(text) + + # You can toggle license checks if you are mid-migration: + validate_metadata(self._meta, check_license=license_check) + + # Reassign parsed (it may normalize the structure) + # self._meta = parsed + self._validated = True + return self + + def as_dict(self) -> dict: + if not self._validated: + self.finalize() + return deepcopy(self._meta) + + def as_json(self) -> str: + return json.dumps(self.as_dict(), ensure_ascii=False) + + # ---- Optional convenience: store as comment on a table + + def save_as_table_comment( + self, engine: Engine, schema: str, table: str + ) -> None: + """ + Store metadata JSON as a COMMENT ON TABLE ... (PostgreSQL). + """ + payload = self.as_json().replace( + "'", "''" + ) # escape single-quotes for SQL literal + full = f"{schema}.{table}" + sql = f"COMMENT ON TABLE {full} IS '{payload}';" + with engine.begin() as conn: + conn.exec_driver_sql(sql) From d28875605bbafffe24cc8a06a57a96a1e863f775 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 10:58:07 +0200 Subject: [PATCH 12/32] #1177: fix syntax --- .../emobility/heavy_duty_transport/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/emobility/heavy_duty_transport/__init__.py b/src/egon/data/datasets/emobility/heavy_duty_transport/__init__.py index 0d0863f36..26ace28c7 100644 --- a/src/egon/data/datasets/emobility/heavy_duty_transport/__init__.py +++ b/src/egon/data/datasets/emobility/heavy_duty_transport/__init__.py @@ -12,6 +12,7 @@ * Mapping demand to H2 buses and writing to DB """ + from pathlib import Path import csv @@ -39,8 +40,10 @@ def create_tables(): """ - Drops existing :py:class:`demand.egon_heavy_duty_transport_voronoi ` is extended - table and creates new one. + Drops existing :py:class: + `demand.egon_heavy_duty_transport_voronoi + ` + is extended table and creates new one. """ engine = db.engine() @@ -103,11 +106,13 @@ class HeavyDutyTransport(Dataset): The config of this dataset can be found in *datasets.yml* in section *mobility_hgv*. - """ + """ # noqa: E501 + #: name: str = "HeavyDutyTransport" #: version: str = "0.0.2" + def __init__(self, dependencies): super().__init__( name=self.name, From 9d56161f37a37fdfcf89ce22f3d1b548b39e2dbc Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 11:02:38 +0200 Subject: [PATCH 13/32] #1177: py3.10 migration note and unused imports --- src/egon/data/datasets/loadarea/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/loadarea/__init__.py b/src/egon/data/datasets/loadarea/__init__.py index 0c970aa93..ccaee6d0e 100644 --- a/src/egon/data/datasets/loadarea/__init__.py +++ b/src/egon/data/datasets/loadarea/__init__.py @@ -5,17 +5,20 @@ import os -from airflow.operators.postgres_operator import PostgresOperator +# TODO: part of PY 3.10 update ... maybe part of another PR +# from airflow.providers.postgres.operators.postgres import PostgresOperator from geoalchemy2.types import Geometry from sqlalchemy import Column, Float, Integer, String from sqlalchemy.dialects.postgresql import HSTORE from sqlalchemy.ext.declarative import declarative_base -import importlib_resources as resources from egon.data import db from egon.data.datasets import Dataset import egon.data.config +# import importlib_resources as resources + + # will be later imported from another file ### Base = declarative_base() @@ -24,6 +27,7 @@ class OsmPolygonUrban(Base): """ Class definition of table openstreetmap.osm_landuse. """ + __tablename__ = "osm_landuse" __table_args__ = {"schema": "openstreetmap"} id = Column(Integer, primary_key=True) @@ -92,7 +96,7 @@ class LoadArea(Dataset): * voltage levels 4-7 * only demand from ind. sites+osm located in LA! - """ + """ # noqa: E501 #: name: str = "LoadArea" From 3bca467d6fcefb4be8acf3cb80859adb7b708819 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:31:40 +0200 Subject: [PATCH 14/32] #1177 oem v2 update: add builder module description --- .../data/metadata/oemetadataBuilder/README.md | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/egon/data/metadata/oemetadataBuilder/README.md diff --git a/src/egon/data/metadata/oemetadataBuilder/README.md b/src/egon/data/metadata/oemetadataBuilder/README.md new file mode 100644 index 000000000..6efaafa0d --- /dev/null +++ b/src/egon/data/metadata/oemetadataBuilder/README.md @@ -0,0 +1,73 @@ +# oemetadataBuilder + +This module implements functionality to create new oemetadata documents as part of other processing steps. This design is useful to integrate oemetadata creation into a pipeline like egon-data. + +Note: Code will be moved to the `omi` package once python version is updates in egon-data and initial developments are done. This makes a lot of sense as the Builder implements a generic way of creating metadata and the use case is relevant in general. One great and highly relevant contribution from Regon to the OpenEnergyFamily. + +## Design + +The main concept we implement is metadata for datasets which can be managed and created in a modular and developer friendly way. The datasets definition meant here is more general then the egon-data dataset usage as multiple data resources can be part of that dataset. A egon-data dataset will implement the resource builder which can generate oemetadata for that single resource. Some of this metadata derived and automatically generated then some information which applies to more then one resource is added using templates and dataset-individual metadata can added using overlays where YAML files are used to specify metadata elements. The builder can apply all this layers of metadata information and "stack" them together into n resource metadata objects - as described one for each dataset. This part is inspired by the current metadata implementation where metadata is generated as part of ech the dataset task in the airflow pipeline. + +The second feature the builder implements is the dataset composition. It is named "package" as naming it dataset could cause some confusion. It is a class which manages all resources and applies them into the oemetadata v2 structure. It will be used in a new task in the pipeline which generates a single metadata document. + +Mixins are used as a common modularization pattern to keep the builder classes cleaner. + +## Usage + +In any dataset use: + +```python +def add_metadata_vg250_zensus(): + engine: Engine = db.engine() + schema, table = "boundaries", "egon_map_zensus_vg250" + yaml_path = "metadata/overlay/zensus/egon_map_zensus_vg250.yaml" + + (OEMetadataBuilder() + .from_template() + .apply_yaml(yaml_path) + .set_basic(name=f"{schema}.{table}") + .auto_resource_from_table( + engine, schema, table, + geom_cols=["zensus_geom"], + ) + .apply_field_hints_from_yaml() + .finalize(license_check=False) + .save_as_table_comment(engine, schema, table) + ) +``` + +OPTIONAL: in airflow pipeline we could use a helper like this one when strong metadata + +```python +# airflow callable +def write_oemetadata(engine: Engine, schema: str, table: str, yaml_path: str): + (OEMetadataBuilder() + .from_template() + .apply_yaml(yaml_path) + .set_basic(name=f"{schema}.{table}") + .auto_resource_from_table(engine, schema, table) + .apply_field_hints_from_yaml() + .finalize(license_check=False) + .save_as_table_comment(engine, schema, table) + ) +``` + +Then to build the package like the code below (not all implemented yet): + +```python +from egon.data.metadata.registry import iter_resources + +pack = OEMetadataPackage().set_root( + name="egon-datapackage-2025-08-13", + title="eGon data release", + description="All resources for this run", + id_="https://example.org/egon/datapackage/2025-08-13" +) + +for res in iter_resources(): + pack.add_resource(res) + +pack.finalize() +json_text = pack.as_json() + +``` From daee9805d15bd8876d7ec8876f03f9aee5817e71 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:34:36 +0200 Subject: [PATCH 15/32] #1177 oem v2 update: Add builder class to create / manage a datapackage with multiple resources --- .../metadata/oemetadataBuilder/package.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/egon/data/metadata/oemetadataBuilder/package.py diff --git a/src/egon/data/metadata/oemetadataBuilder/package.py b/src/egon/data/metadata/oemetadataBuilder/package.py new file mode 100644 index 000000000..e1ddaca71 --- /dev/null +++ b/src/egon/data/metadata/oemetadataBuilder/package.py @@ -0,0 +1,105 @@ +from copy import deepcopy +from typing import Any, Dict +import json + +from omi.base import get_metadata_specification +from omi.validation import validate_metadata # parse_metadata +from sqlalchemy.engine import Engine + +from egon.data.metadata import settings + + +class OEMetadataPackage: + def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: + self.spec = get_metadata_specification(version) + self._doc: Dict[str, Any] = { + "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/production/oemetadata/latest/context.json", # noqa: E501 + "name": "", + "title": "", + "description": "", + "@id": "", + "resources": [], + "metaMetadata": { + "metadataVersion": version, + "metadataLicense": {"name": "CC0-1.0"}, + }, + } + self._validated = False + + def set_root( + self, + *, + name: str, + title: str = "", + description: str = "", + id_: str = "", + ) -> "OEMetadataPackage": + self._doc["name"] = name + self._doc["title"] = title + self._doc["description"] = description + self._doc["@id"] = id_ + self._validated = False + return self + + def add_resource( + self, + resource: dict, + *, + dedupe_by: str = "name", + overwrite: bool = True, + ) -> "OEMetadataPackage": + if dedupe_by and overwrite: + self._doc["resources"] = [ + r + for r in self._doc["resources"] + if r.get(dedupe_by) != resource.get(dedupe_by) + ] + self._doc["resources"].append(deepcopy(resource)) + self._validated = False + return self + + def add_from_full_document( + self, full_doc: dict, *, take_root_if_empty: bool = False + ) -> "OEMetadataPackage": + # Optionally fill root if still empty + if take_root_if_empty and not self._doc["name"]: + for k in ("name", "title", "description", "@id"): + if k in full_doc: + self._doc[k] = full_doc[k] + for r in full_doc.get("resources", []): + self.add_resource(r) + return self + + def add_from_table_comment( + self, engine: Engine, schema: str, table: str + ) -> "OEMetadataPackage": + sql = """ + SELECT obj_description((quote_ident(%s)||'.'||quote_ident(%s))::regclass, 'pg_class') AS comment + """ # noqa: E501 + with engine.begin() as conn: + comment = conn.exec_driver_sql(sql, (schema, table)).scalar() + if not comment: + return self + try: + full_doc = json.loads(comment) + except Exception: + return self + # Optionally validate the doc before merging + try: + validate_metadata(full_doc, check_license=False) + except Exception: + pass + return self.add_from_full_document(full_doc) + + def finalize(self, *, license_check: bool = True) -> "OEMetadataPackage": + validate_metadata(self._doc, check_license=license_check) + self._validated = True + return self + + def as_dict(self) -> dict: + if not self._validated: + self.finalize() + return deepcopy(self._doc) + + def as_json(self) -> str: + return json.dumps(self.as_dict(), ensure_ascii=False) From e75a0ca8b9de7f0210ae118c21318ae56cf19d2a Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:35:17 +0200 Subject: [PATCH 16/32] #1177 oem v2 update: refactor oemetadata builder into more modular approach -> becomes resource builder to create metadata for single data resources --- .../metadata/oemetadataBuilder/resource.py | 411 ++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 src/egon/data/metadata/oemetadataBuilder/resource.py diff --git a/src/egon/data/metadata/oemetadataBuilder/resource.py b/src/egon/data/metadata/oemetadataBuilder/resource.py new file mode 100644 index 000000000..402234d7f --- /dev/null +++ b/src/egon/data/metadata/oemetadataBuilder/resource.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Dict, List, Optional +import datetime as dt +import json + +from geoalchemy2 import Geometry +from omi.base import MetadataSpecification, get_metadata_specification +from omi.validation import validate_metadata # parse_metadata +from sqlalchemy import MetaData, Table, inspect +from sqlalchemy.dialects.postgresql.base import ischema_names +from sqlalchemy.engine import Engine +import yaml # PyYAML + +# ---- Optional: your project settings/hooks +# from egon.data import db, logger +from egon.data.metadata import settings + +# Geometry awareness for reflection +ischema_names["geometry"] = Geometry # generic +# You can add specific geometry columns later per-table via kwargs + + +def _today() -> str: + return dt.date.today().isoformat() + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Deep merge with 'override wins', recursively. + Lists are replaced (not merged) by default to avoid subtle duplication. + """ + out = deepcopy(base) + for k, v in override.items(): + if isinstance(v, dict) and isinstance(out.get(k), dict): + out[k] = _deep_merge(out[k], v) + else: + out[k] = deepcopy(v) + return out + + +def _sqlatype_to_oem_type(sa_type: str) -> str: + """ + Map SQLAlchemy reflected type string -> OEM v2 field.type + Keep it simple and deterministic; adjust as needed. + """ + t = sa_type.lower() + # geometry + if "geometry" in t: + return "geometry" + # integers + if any(x in t for x in ["int", "serial", "bigint", "smallint"]): + return "integer" + # floats / numeric + if any(x in t for x in ["float", "double", "numeric", "real", "decimal"]): + return "number" + # booleans + if "bool" in t: + return "boolean" + # timestamp/date/time + if "timestamp" in t or "timestamptz" in t: + return "datetime" + if t == "date": + return "date" + if t == "time": + return "time" + # text-ish + if any( + x in t for x in ["text", "char", "string", "uuid", "json", "jsonb"] + ): + return "string" + # fallback + return "string" + + +@dataclass +class ResourceField: + """ + Minimal implementation of oemetadata v2 resource structure. + Eases usage in Python. + """ + + name: str + description: Optional[str] = None + type: str = "string" + unit: Optional[str] = None + nullable: Optional[bool] = None + + def to_dict(self) -> dict: + d = { + "name": self.name, + "type": self.type, + } + # include optional keys only when provided + if self.description is not None: + d["description"] = self.description + if self.unit is not None: + d["unit"] = self.unit + if self.nullable is not None: + d["nullable"] = self.nullable + return d + + +class OEMetadataResourceBuilder: + """ + Single, reusable builder for OEP oemetadata v2 using omi as source of truth. + + Typical flow: + builder = OEMetadataBuilder().from_template() + .apply_yaml("dataset_meta.yaml") + .auto_resource_from_table(engine, "schema", "table", geom_cols=["geom"]) + .set_basic(name="schema.table", title="...", description="...") + .finalize() + payload = builder.as_json() # validated JSON string + builder.save_as_table_comment(db_engine, "schema", "table") # optional + """ # noqa: E501 + + def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: + self.spec: MetadataSpecification = get_metadata_specification(version) + self._meta: Dict[str, Any] = {} + self._validated: bool = False + + # ---- Required steps + + def from_template(self) -> "OEMetadataResourceBuilder": + """ + Start from omi's template plus selected bits from example + (context/metaMetadata). + Ensures keys exist (empty strings/structures as per spec). + """ + tpl = deepcopy(self.spec.template) if self.spec.template else {} + if self.spec.example: + # Copy @context + metaMetadata if present in example + if "@context" in self.spec.example: + tpl["@context"] = deepcopy(self.spec.example["@context"]) + if "metaMetadata" in self.spec.example: + tpl["metaMetadata"] = deepcopy( + self.spec.example["metaMetadata"] + ) + self._meta = tpl["resources"][0] + self._validated = False + return self + + def apply_yaml( + self, yaml_path: str | None = None, yaml_text: str | None = None + ) -> "OEMetadataResourceBuilder": + """ + Merge user-provided YAML overrides into the current metadata object. + You can allow either a file path or a YAML string (for testing). + """ + if yaml_path: + with open(yaml_path, "r", encoding="utf-8") as fh: + override = yaml.safe_load(fh) or {} + elif yaml_text: + override = yaml.safe_load(yaml_text) or {} + else: + override = {} + + self._meta = _deep_merge(self._meta, override) + self._validated = False + return self + + def set_basic( + self, + name: str, + title: Optional[str] = None, + description: Optional[str] = None, + language: Optional[List[str]] = None, + publication_date: Optional[str] = None, + dataset_id: Optional[str] = None, + ) -> "OEMetadataResourceBuilder": + """ + Convenience setter for common top-level fields. + """ + if publication_date is None: + publication_date = _today() + patch = { + "name": name, + "publicationDate": publication_date, + } + if title is not None: + patch["title"] = title + if description is not None: + patch["description"] = description + if language is not None: + patch["language"] = language + if dataset_id is not None: + patch["id"] = dataset_id + + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + def set_context(self, context_obj: dict) -> "OEMetadataResourceBuilder": + self._meta = _deep_merge(self._meta, {"context": context_obj}) + self._validated = False + return self + + def set_spatial( + self, + extent: Optional[str] = None, + resolution: Optional[str] = None, + location: Optional[Any] = None, + ) -> "OEMetadataResourceBuilder": + patch = {"spatial": {}} + if location is not None: + patch["spatial"]["location"] = location + if extent is not None: + patch["spatial"]["extent"] = extent + if resolution is not None: + patch["spatial"]["resolution"] = resolution + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + def set_temporal( + self, + reference_date: Optional[str] = None, + timeseries: Optional[dict] = None, + ) -> "OEMetadataResourceBuilder": + patch = {"temporal": {}} + if reference_date is not None: + # NOTE: your older code used 'referenceDate' vs + # 'reference_date' in places. + # OEM v2 uses 'referenceDate' (camelCase). Keep consistent here: + patch["temporal"]["referenceDate"] = reference_date + if timeseries is not None: + patch["temporal"]["timeseries"] = timeseries + self._meta = _deep_merge(self._meta, patch) + self._validated = False + return self + + # ---- Sources, licenses, contributors + + def add_source(self, source: dict) -> "OEMetadataResourceBuilder": + self._meta.setdefault("sources", []) + self._meta["sources"].append(source) + self._validated = False + return self + + def add_license(self, lic: dict) -> "OEMetadataResourceBuilder": + self._meta.setdefault("licenses", []) + self._meta["licenses"].append(lic) + self._validated = False + return self + + def add_contributor( + self, contributor: dict + ) -> "OEMetadataResourceBuilder": + self._meta.setdefault("contributors", []) + self._meta["contributors"].append(contributor) + self._validated = False + return self + + # ---- Resources + + def auto_resource_from_table( + self, + engine: Engine, + schema: str, + table: str, + *, + resource_name: Optional[str] = None, + format_: str = "PostgreSQL", + encoding: str = "UTF-8", + primary_key: Optional[List[str]] = None, + foreign_keys: Optional[List[dict]] = None, + geom_cols: Optional[List[str]] = None, + dialect: Optional[dict] = None, + overwrite_existing: bool = False, + ) -> "OEMetadataResourceBuilder": + """ + Introspect a DB table and create a single tabular data resource entry. + + - Maps SQLA types to OEM types + - Marks 'nullable' where possible + - Recognizes geometry columns (if given in geom_cols) as 'geometry' + + If overwrite_existing=False and a resource already exists with the same + name, it will be left as-is (you could add a flag to update instead). + """ + if geom_cols is None: + geom_cols = ["geom", "geometry", "geom_point", "geom_polygon"] + + # reflect + meta = MetaData() + tbl = Table(table, meta, schema=schema, autoload_with=engine) + + fields: List[ResourceField] = [] + for col in tbl.columns: + sa_t = str(col.type) + # if explicitly geometry by name, treat as geometry + col_type = ( + "geometry" + if col.name in geom_cols + else _sqlatype_to_oem_type(sa_t) + ) + fields.append( + ResourceField( + name=col.name, + description=None, + type=col_type, + unit=None, + nullable=col.nullable, + ) + ) + + if not resource_name: + resource_name = f"{schema}.{table}" + + resource = { + "name": resource_name, + # TODO: @jh-RLI The OEP will set this, + # consider if local usage is important + "path": None, + "type": "table", + "format": format_, + "encoding": encoding, + "schema": { + "fields": [f.to_dict() for f in fields], + "primaryKey": primary_key + or self._best_guess_pk(engine, schema, table), + "foreignKeys": foreign_keys or [], + }, + "dialect": dialect or {"delimiter": None, "decimalSeparator": "."}, + } + + # install resources array + self._meta.setdefault("resources", []) + if overwrite_existing: + self._meta["resources"] = [ + r + for r in self._meta["resources"] + if r.get("name") != resource_name + ] + # only add if not present + if not any( + r.get("name") == resource_name for r in self._meta["resources"] + ): + self._meta["resources"].append(resource) + + self._validated = False + return self + + def _best_guess_pk( + self, engine: Engine, schema: str, table: str + ) -> List[str]: + """ + Try to read PK columns via SQLAlchemy inspector, fallback to + ['id'] if found, else []. + """ + insp = inspect(engine) + pk = insp.get_pk_constraint(table, schema=schema) + cols = pk.get("constrained_columns") if pk else None + if cols: + return cols + # common fallback + columns = [c["name"] for c in insp.get_columns(table, schema=schema)] + return ["id"] if "id" in columns else [] + + # ---- Finalize/validate/serialize + + def finalize( + self, license_check: bool = False + ) -> "OEMetadataResourceBuilder": + """ + Make minimal guarantees & validate with omi. + """ + # Fill sane defaults if missing + # self._meta.setdefault("publicationDate", _today()) + self._meta.setdefault("language", ["en-EN"]) + + # TODO: @jh-RLI might be expensive + # parse + validate with omi + # parse_metadata expects string; serialize & round-trip to normalize + # text = json.dumps(self._meta, ensure_ascii=False) + # parsed = parse_metadata(text) + + # You can toggle license checks if you are mid-migration: + validate_metadata(self._meta, check_license=license_check) + + # Reassign parsed (it may normalize the structure) + # self._meta = parsed + self._validated = True + return self + + def as_dict(self) -> dict: + if not self._validated: + self.finalize() + return deepcopy(self._meta) + + def as_json(self) -> str: + return json.dumps(self.as_dict(), ensure_ascii=False) + + # ---- Optional convenience: store as comment on a table + + def save_as_table_comment( + self, engine: Engine, schema: str, table: str + ) -> None: + """ + Store metadata JSON as a COMMENT ON TABLE ... (PostgreSQL). + """ + payload = self.as_json().replace( + "'", "''" + ) # escape single-quotes for SQL literal + full = f"{schema}.{table}" + sql = f"COMMENT ON TABLE {full} IS '{payload}';" + with engine.begin() as conn: + conn.exec_driver_sql(sql) From f800272d768ce1485fbf02fbb7f0862512dfce91 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:35:28 +0200 Subject: [PATCH 17/32] extend description --- src/egon/data/metadata/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/metadata/README.md b/src/egon/data/metadata/README.md index 4dda823ef..3abeaa79e 100644 --- a/src/egon/data/metadata/README.md +++ b/src/egon/data/metadata/README.md @@ -4,7 +4,7 @@ The metadata module provides all metadata related functionality used to create/u The metadata module is/offers: - - the central place to configuring the metadata generation + - the central place to configuring the metadata generation using the oemetadata builder - it provides a metadata-template system which applies information like the project context description to all dataset resources by default to ease handling redundant information. - Metadata can be setup using YAML files or dict data structures when working in python code using the OMI package - A central functionality to upload metadata to the internal database as SQL comment on table <---## We could also use a jsonb column to avoid parsing metadata from string From 1127cdffcfaec551a5c09893b8c256629f4efb7d Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:35:48 +0200 Subject: [PATCH 18/32] #1177 oem v2 update: refactor oemetadata builder --- src/egon/data/metadata/oemetadata.py | 407 --------------------------- 1 file changed, 407 deletions(-) delete mode 100644 src/egon/data/metadata/oemetadata.py diff --git a/src/egon/data/metadata/oemetadata.py b/src/egon/data/metadata/oemetadata.py deleted file mode 100644 index 667753a69..000000000 --- a/src/egon/data/metadata/oemetadata.py +++ /dev/null @@ -1,407 +0,0 @@ -from __future__ import annotations - -from copy import deepcopy -from dataclasses import dataclass -from typing import Any, Dict, List, Optional -import datetime as dt -import json - -from geoalchemy2 import Geometry -from omi.base import MetadataSpecification, get_metadata_specification -from omi.validation import validate_metadata # parse_metadata -from sqlalchemy import MetaData, Table, inspect -from sqlalchemy.dialects.postgresql.base import ischema_names -from sqlalchemy.engine import Engine -import yaml # PyYAML - -# ---- Optional: your project settings/hooks -# from egon.data import db, logger -from egon.data.metadata import settings - -# Geometry awareness for reflection -ischema_names["geometry"] = Geometry # generic -# You can add specific geometry columns later per-table via kwargs - - -def _today() -> str: - return dt.date.today().isoformat() - - -def _deep_merge(base: dict, override: dict) -> dict: - """ - Deep merge with 'override wins', recursively. - Lists are replaced (not merged) by default to avoid subtle duplication. - """ - out = deepcopy(base) - for k, v in override.items(): - if isinstance(v, dict) and isinstance(out.get(k), dict): - out[k] = _deep_merge(out[k], v) - else: - out[k] = deepcopy(v) - return out - - -def _sqlatype_to_oem_type(sa_type: str) -> str: - """ - Map SQLAlchemy reflected type string -> OEM v2 field.type - Keep it simple and deterministic; adjust as needed. - """ - t = sa_type.lower() - # geometry - if "geometry" in t: - return "geometry" - # integers - if any(x in t for x in ["int", "serial", "bigint", "smallint"]): - return "integer" - # floats / numeric - if any(x in t for x in ["float", "double", "numeric", "real", "decimal"]): - return "number" - # booleans - if "bool" in t: - return "boolean" - # timestamp/date/time - if "timestamp" in t or "timestamptz" in t: - return "datetime" - if t == "date": - return "date" - if t == "time": - return "time" - # text-ish - if any( - x in t for x in ["text", "char", "string", "uuid", "json", "jsonb"] - ): - return "string" - # fallback - return "string" - - -@dataclass -class ResourceField: - """ - Minimal implementation of oemetadata v2 resource structure. - Eases usage in Python. - """ - - name: str - description: Optional[str] = None - type: str = "string" - unit: Optional[str] = None - nullable: Optional[bool] = None - - def to_dict(self) -> dict: - d = { - "name": self.name, - "type": self.type, - } - # include optional keys only when provided - if self.description is not None: - d["description"] = self.description - if self.unit is not None: - d["unit"] = self.unit - if self.nullable is not None: - d["nullable"] = self.nullable - return d - - -class OEMetadataBuilder: - """ - Single, reusable builder for OEP oemetadata v2 using omi as source of truth. - - Typical flow: - builder = OEMetadataBuilder().from_template() - .apply_yaml("dataset_meta.yaml") - .auto_resource_from_table(engine, "schema", "table", geom_cols=["geom"]) - .set_basic(name="schema.table", title="...", description="...") - .finalize() - payload = builder.as_json() # validated JSON string - builder.save_as_table_comment(db_engine, "schema", "table") # optional - """ # noqa: E501 - - def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: - self.spec: MetadataSpecification = get_metadata_specification(version) - self._meta: Dict[str, Any] = {} - self._validated: bool = False - - # ---- Required steps - - def from_template(self) -> "OEMetadataBuilder": - """ - Start from omi's template plus selected bits from example - (context/metaMetadata). - Ensures keys exist (empty strings/structures as per spec). - """ - tpl = deepcopy(self.spec.template) if self.spec.template else {} - if self.spec.example: - # Copy @context + metaMetadata if present in example - if "@context" in self.spec.example: - tpl["@context"] = deepcopy(self.spec.example["@context"]) - if "metaMetadata" in self.spec.example: - tpl["metaMetadata"] = deepcopy( - self.spec.example["metaMetadata"] - ) - self._meta = tpl["resources"][0] - self._validated = False - return self - - def apply_yaml( - self, yaml_path: str | None = None, yaml_text: str | None = None - ) -> "OEMetadataBuilder": - """ - Merge user-provided YAML overrides into the current metadata object. - You can allow either a file path or a YAML string (for testing). - """ - if yaml_path: - with open(yaml_path, "r", encoding="utf-8") as fh: - override = yaml.safe_load(fh) or {} - elif yaml_text: - override = yaml.safe_load(yaml_text) or {} - else: - override = {} - - self._meta = _deep_merge(self._meta, override) - self._validated = False - return self - - def set_basic( - self, - name: str, - title: Optional[str] = None, - description: Optional[str] = None, - language: Optional[List[str]] = None, - publication_date: Optional[str] = None, - dataset_id: Optional[str] = None, - ) -> "OEMetadataBuilder": - """ - Convenience setter for common top-level fields. - """ - if publication_date is None: - publication_date = _today() - patch = { - "name": name, - "publicationDate": publication_date, - } - if title is not None: - patch["title"] = title - if description is not None: - patch["description"] = description - if language is not None: - patch["language"] = language - if dataset_id is not None: - patch["id"] = dataset_id - - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - def set_context(self, context_obj: dict) -> "OEMetadataBuilder": - self._meta = _deep_merge(self._meta, {"context": context_obj}) - self._validated = False - return self - - def set_spatial( - self, - extent: Optional[str] = None, - resolution: Optional[str] = None, - location: Optional[Any] = None, - ) -> "OEMetadataBuilder": - patch = {"spatial": {}} - if location is not None: - patch["spatial"]["location"] = location - if extent is not None: - patch["spatial"]["extent"] = extent - if resolution is not None: - patch["spatial"]["resolution"] = resolution - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - def set_temporal( - self, - reference_date: Optional[str] = None, - timeseries: Optional[dict] = None, - ) -> "OEMetadataBuilder": - patch = {"temporal": {}} - if reference_date is not None: - # NOTE: your older code used 'referenceDate' vs - # 'reference_date' in places. - # OEM v2 uses 'referenceDate' (camelCase). Keep consistent here: - patch["temporal"]["referenceDate"] = reference_date - if timeseries is not None: - patch["temporal"]["timeseries"] = timeseries - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - # ---- Sources, licenses, contributors - - def add_source(self, source: dict) -> "OEMetadataBuilder": - self._meta.setdefault("sources", []) - self._meta["sources"].append(source) - self._validated = False - return self - - def add_license(self, lic: dict) -> "OEMetadataBuilder": - self._meta.setdefault("licenses", []) - self._meta["licenses"].append(lic) - self._validated = False - return self - - def add_contributor(self, contributor: dict) -> "OEMetadataBuilder": - self._meta.setdefault("contributors", []) - self._meta["contributors"].append(contributor) - self._validated = False - return self - - # ---- Resources - - def auto_resource_from_table( - self, - engine: Engine, - schema: str, - table: str, - *, - resource_name: Optional[str] = None, - format_: str = "PostgreSQL", - encoding: str = "UTF-8", - primary_key: Optional[List[str]] = None, - foreign_keys: Optional[List[dict]] = None, - geom_cols: Optional[List[str]] = None, - dialect: Optional[dict] = None, - overwrite_existing: bool = False, - ) -> "OEMetadataBuilder": - """ - Introspect a DB table and create a single tabular data resource entry. - - - Maps SQLA types to OEM types - - Marks 'nullable' where possible - - Recognizes geometry columns (if given in geom_cols) as 'geometry' - - If overwrite_existing=False and a resource already exists with the same - name, it will be left as-is (you could add a flag to update instead). - """ - if geom_cols is None: - geom_cols = ["geom", "geometry", "geom_point", "geom_polygon"] - - # reflect - meta = MetaData() - tbl = Table(table, meta, schema=schema, autoload_with=engine) - - fields: List[ResourceField] = [] - for col in tbl.columns: - sa_t = str(col.type) - # if explicitly geometry by name, treat as geometry - col_type = ( - "geometry" - if col.name in geom_cols - else _sqlatype_to_oem_type(sa_t) - ) - fields.append( - ResourceField( - name=col.name, - description=None, - type=col_type, - unit=None, - nullable=col.nullable, - ) - ) - - if not resource_name: - resource_name = f"{schema}.{table}" - - resource = { - "name": resource_name, - # TODO: @jh-RLI The OEP will set this, - # consider if local usage is important - "path": None, - "type": "table", - "format": format_, - "encoding": encoding, - "schema": { - "fields": [f.to_dict() for f in fields], - "primaryKey": primary_key - or self._best_guess_pk(engine, schema, table), - "foreignKeys": foreign_keys or [], - }, - "dialect": dialect or {"delimiter": None, "decimalSeparator": "."}, - } - - # install resources array - self._meta.setdefault("resources", []) - if overwrite_existing: - self._meta["resources"] = [ - r - for r in self._meta["resources"] - if r.get("name") != resource_name - ] - # only add if not present - if not any( - r.get("name") == resource_name for r in self._meta["resources"] - ): - self._meta["resources"].append(resource) - - self._validated = False - return self - - def _best_guess_pk( - self, engine: Engine, schema: str, table: str - ) -> List[str]: - """ - Try to read PK columns via SQLAlchemy inspector, fallback to - ['id'] if found, else []. - """ - insp = inspect(engine) - pk = insp.get_pk_constraint(table, schema=schema) - cols = pk.get("constrained_columns") if pk else None - if cols: - return cols - # common fallback - columns = [c["name"] for c in insp.get_columns(table, schema=schema)] - return ["id"] if "id" in columns else [] - - # ---- Finalize/validate/serialize - - def finalize(self, license_check: bool = False) -> "OEMetadataBuilder": - """ - Make minimal guarantees & validate with omi. - """ - # Fill sane defaults if missing - # self._meta.setdefault("publicationDate", _today()) - self._meta.setdefault("language", ["en-EN"]) - - # TODO: @jh-RLI might be expensive - # parse + validate with omi - # parse_metadata expects string; serialize & round-trip to normalize - # text = json.dumps(self._meta, ensure_ascii=False) - # parsed = parse_metadata(text) - - # You can toggle license checks if you are mid-migration: - validate_metadata(self._meta, check_license=license_check) - - # Reassign parsed (it may normalize the structure) - # self._meta = parsed - self._validated = True - return self - - def as_dict(self) -> dict: - if not self._validated: - self.finalize() - return deepcopy(self._meta) - - def as_json(self) -> str: - return json.dumps(self.as_dict(), ensure_ascii=False) - - # ---- Optional convenience: store as comment on a table - - def save_as_table_comment( - self, engine: Engine, schema: str, table: str - ) -> None: - """ - Store metadata JSON as a COMMENT ON TABLE ... (PostgreSQL). - """ - payload = self.as_json().replace( - "'", "''" - ) # escape single-quotes for SQL literal - full = f"{schema}.{table}" - sql = f"COMMENT ON TABLE {full} IS '{payload}';" - with engine.begin() as conn: - conn.exec_driver_sql(sql) From c65d8b9161a9818f82b2d64c91b4920824167b80 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 12:40:34 +0200 Subject: [PATCH 19/32] #1177: enhance code --- src/egon/data/metadata/oemetadataBuilder/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/egon/data/metadata/oemetadataBuilder/package.py b/src/egon/data/metadata/oemetadataBuilder/package.py index e1ddaca71..bf7c08864 100644 --- a/src/egon/data/metadata/oemetadataBuilder/package.py +++ b/src/egon/data/metadata/oemetadataBuilder/package.py @@ -12,17 +12,15 @@ class OEMetadataPackage: def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: self.spec = get_metadata_specification(version) + self._doc: Dict[str, Any] = { - "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/production/oemetadata/latest/context.json", # noqa: E501 + "@context": self.spec.example["@context"], "name": "", "title": "", "description": "", "@id": "", "resources": [], - "metaMetadata": { - "metadataVersion": version, - "metadataLicense": {"name": "CC0-1.0"}, - }, + "metaMetadata": self.spec.example["metaMetadata"], } self._validated = False From dc649b35a80d814fea040e91c66f02830f56605a Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Fri, 15 Aug 2025 15:51:18 +0200 Subject: [PATCH 20/32] #1177: add oemetadata builder modules --- src/egon/data/metadata/oemetadataBuilder/__init__.py | 0 src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/egon/data/metadata/oemetadataBuilder/__init__.py create mode 100644 src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py diff --git a/src/egon/data/metadata/oemetadataBuilder/__init__.py b/src/egon/data/metadata/oemetadataBuilder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py b/src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py new file mode 100644 index 000000000..e69de29bb From d788f52abaf8f1793c2b384e12cc4fc209386b6f Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 15 Oct 2025 19:46:28 +0200 Subject: [PATCH 21/32] #1177: add regon attribution to metadata settings --- src/egon/data/metadata/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/metadata/settings.py b/src/egon/data/metadata/settings.py index 8023a2f6c..8e8d5877a 100644 --- a/src/egon/data/metadata/settings.py +++ b/src/egon/data/metadata/settings.py @@ -4,3 +4,4 @@ EGON_ATTRIBUTION: str = "© eGon development team" +REGON_ATTRIBUTION: str = "© ReGon development team" From 55d8359cb5ca712d350306d8e7ee5e107cf52676 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Tue, 4 Nov 2025 00:16:53 +0100 Subject: [PATCH 22/32] #1177 oemetadata update: prepare metadata for conversion --- .../results/boundaries.egon_map_zensus_climate_zones.json | 2 +- .../results/boundaries.egon_map_zensus_mvgd_buildings.json | 2 +- .../results/boundaries.egon_map_zensus_weather_cell.json | 2 +- .../results/demand.egon_building_electricity_peak_loads.json | 2 +- .../metadata/results/demand.egon_building_heat_peak_loads.json | 2 +- .../demand.egon_cts_electricity_demand_building_share.json | 2 +- .../results/demand.egon_cts_heat_demand_building_share.json | 2 +- .../results/demand.egon_daily_heat_demand_per_climate_zone.json | 2 +- .../metadata/results/demand.egon_etrago_electricity_cts.json | 2 +- .../results/demand.egon_etrago_electricity_households.json | 2 +- src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json | 2 +- src/egon/data/metadata/results/demand.egon_heat_idp_pool.json | 2 +- .../results/demand.egon_heat_timeseries_selected_profiles.json | 2 +- ...emand.egon_household_electricity_profile_in_census_cell.json | 2 +- .../demand.egon_household_electricity_profile_of_buildings.json | 2 +- .../metadata/results/demand.iee_household_load_profiles.json | 2 +- src/egon/data/metadata/results/grid.egon_mv_grid_district.json | 2 +- .../metadata/results/openstreetmap.osm_buildings_filtered.json | 2 +- .../metadata/results/openstreetmap.osm_buildings_synthetic.json | 2 +- .../results/society.destatis_zensus_population_per_ha.json | 2 +- ...on_destatis_zensus_apartment_building_population_per_ha.json | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json index f3f5a56e6..f9f4af178 100644 --- a/src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json +++ b/src/egon/data/metadata/results/boundaries.egon_map_zensus_climate_zones.json @@ -132,7 +132,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json index e09cf47aa..00281ce8d 100644 --- a/src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json +++ b/src/egon/data/metadata/results/boundaries.egon_map_zensus_mvgd_buildings.json @@ -152,7 +152,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json index 3026115aa..5431be2ad 100644 --- a/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json +++ b/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json @@ -121,7 +121,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json b/src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json index 19d7b55cc..6fe4a6303 100644 --- a/src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json +++ b/src/egon/data/metadata/results/demand.egon_building_electricity_peak_loads.json @@ -182,7 +182,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json b/src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json index 17f7dfee0..409900bc2 100644 --- a/src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json +++ b/src/egon/data/metadata/results/demand.egon_building_heat_peak_loads.json @@ -190,7 +190,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json b/src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json index c7a60b34c..7c11ef8da 100644 --- a/src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json +++ b/src/egon/data/metadata/results/demand.egon_cts_electricity_demand_building_share.json @@ -176,7 +176,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json b/src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json index 0f9228270..441ad4b99 100644 --- a/src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json +++ b/src/egon/data/metadata/results/demand.egon_cts_heat_demand_building_share.json @@ -162,7 +162,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json b/src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json index 06d950eda..0e39076de 100644 --- a/src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json +++ b/src/egon/data/metadata/results/demand.egon_daily_heat_demand_per_climate_zone.json @@ -144,7 +144,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json b/src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json index 5db7cd176..4eac590cc 100644 --- a/src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json +++ b/src/egon/data/metadata/results/demand.egon_etrago_electricity_cts.json @@ -178,7 +178,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json b/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json index b46c4b4ef..eb74fbc29 100644 --- a/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json +++ b/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json @@ -178,7 +178,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json b/src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json index 6fff2bd10..0f83fe036 100644 --- a/src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json +++ b/src/egon/data/metadata/results/demand.egon_etrago_heat_cts.json @@ -192,7 +192,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_heat_idp_pool.json b/src/egon/data/metadata/results/demand.egon_heat_idp_pool.json index 5f597c7f8..2f8f9ff2f 100644 --- a/src/egon/data/metadata/results/demand.egon_heat_idp_pool.json +++ b/src/egon/data/metadata/results/demand.egon_heat_idp_pool.json @@ -133,7 +133,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json b/src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json index ff1b05742..b58c2d80a 100644 --- a/src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json +++ b/src/egon/data/metadata/results/demand.egon_heat_timeseries_selected_profiles.json @@ -138,7 +138,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json index 3e0696b84..71878ccf0 100644 --- a/src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json +++ b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_in_census_cell.json @@ -199,7 +199,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json index f302d0a27..8fea6a0d6 100644 --- a/src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json +++ b/src/egon/data/metadata/results/demand.egon_household_electricity_profile_of_buildings.json @@ -151,7 +151,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/demand.iee_household_load_profiles.json b/src/egon/data/metadata/results/demand.iee_household_load_profiles.json index 093fc47dd..e8572e3e4 100644 --- a/src/egon/data/metadata/results/demand.iee_household_load_profiles.json +++ b/src/egon/data/metadata/results/demand.iee_household_load_profiles.json @@ -139,7 +139,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/grid.egon_mv_grid_district.json b/src/egon/data/metadata/results/grid.egon_mv_grid_district.json index a8eb56501..d25a19623 100644 --- a/src/egon/data/metadata/results/grid.egon_mv_grid_district.json +++ b/src/egon/data/metadata/results/grid.egon_mv_grid_district.json @@ -131,7 +131,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json b/src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json index 519c87941..938403bbe 100644 --- a/src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json +++ b/src/egon/data/metadata/results/openstreetmap.osm_buildings_filtered.json @@ -164,7 +164,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json b/src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json index 5957f61f3..640301d09 100644 --- a/src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json +++ b/src/egon/data/metadata/results/openstreetmap.osm_buildings_synthetic.json @@ -169,7 +169,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json b/src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json index 5e20057a6..ec223071b 100644 --- a/src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json +++ b/src/egon/data/metadata/results/society.destatis_zensus_population_per_ha.json @@ -123,7 +123,7 @@ "badge": "" }, "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", diff --git a/src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json b/src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json index a48513071..b6a5f983a 100644 --- a/src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json +++ b/src/egon/data/metadata/results/society.egon_destatis_zensus_apartment_building_population_per_ha.json @@ -131,7 +131,7 @@ } ], "metaMetadata": { - "metadataVersion": "OEP-1.4.1", + "metadataVersion": "OEP-1.5.2", "metadataLicense": { "name": "CC0-1.0", "title": "Creative Commons Zero v1.0 Universal", From 54106aae690dc307287af2027dd9bba6d5ea364b Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 5 Nov 2025 18:04:10 +0100 Subject: [PATCH 23/32] #1177: Update metadata to sql comment on table function: - It now expects a python dict or json string without extra quoting --- src/egon/data/db.py | 59 ++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/egon/data/db.py b/src/egon/data/db.py index 347019048..0d4a98260 100755 --- a/src/egon/data/db.py +++ b/src/egon/data/db.py @@ -1,6 +1,8 @@ from contextlib import contextmanager import codecs import functools +import json +import logging import os import time @@ -8,6 +10,7 @@ from sqlalchemy import create_engine, text from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import sessionmaker +from sqlalchemy.sql import quoted_name import geopandas as gpd import pandas as pd @@ -73,36 +76,38 @@ def execute_sql(sql_string): con.execute(text(sql_string)) -def submit_comment(json, schema, table): - """Add comment to table. - - We use `Open Energy Metadata `_ - standard for describing our data. Metadata is stored as JSON in the table - comment. - - Parameters - ---------- - json : str - JSON string reflecting comment - schema : str - The target table's database schema - table : str - Database table on which to put the given comment - """ - prefix_str = "COMMENT ON TABLE {0}.{1} IS ".format(schema, table) - - check_json_str = ( - "SELECT obj_description('{0}.{1}'::regclass)::json".format( - schema, table - ) +def submit_comment(metadata, schema, table): + # normalize to JSON text + json_str = ( + json.dumps(metadata, ensure_ascii=False) + if not isinstance(metadata, str) + else json.dumps(json.loads(metadata), ensure_ascii=False) ) - execute_sql(prefix_str + json + ";") + ident = f"{quoted_name(schema, True)}.{quoted_name(table, True)}" + stmt = text(f"COMMENT ON TABLE {ident} IS :comment") + + with engine().begin() as conn: + # fail early if table isn’t there + if ( + conn.execute( + text("SELECT to_regclass(:reg)"), + {"reg": f"{schema}.{table}"}, + ).scalar() + is None + ): + logging.warning("Table %s.%s does not exist", schema, table) + return None + + conn.execute(stmt, {"comment": json_str}) + + # ✅ validation (no cast directly on a bind) + conn.execute( + text("SELECT obj_description(to_regclass(:reg))::json"), + {"reg": f"{schema}.{table}"}, + ) - # Query table comment and cast it into JSON - # The query throws an error if JSON is invalid - execute_sql(check_json_str) + logging.info("Metadata comment for %s.%s stored.", schema, table) def execute_sql_script(script, encoding="utf-8-sig"): From ffc827880d7e6a2e040c8213f630d609cd72a5d3 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 6 Nov 2025 16:38:52 +0100 Subject: [PATCH 24/32] #1177: Remove extra quoting for JSON metadata strings, now handled in submit comment functionality - Also update path that reads the static oemetadata.json files stored in the metadata/results directory - Update metadataVersion to at least v1.5.2, omi does not support older version --- .../datasets/district_heating_areas/__init__.py | 4 ++-- src/egon/data/datasets/heat_demand/__init__.py | 2 +- src/egon/data/datasets/osm/__init__.py | 2 +- src/egon/data/datasets/renewable_feedin.py | 2 +- src/egon/data/datasets/scenario_capacities.py | 2 +- src/egon/data/datasets/vg250/__init__.py | 5 ++--- src/egon/data/datasets/zensus_vg250.py | 2 +- src/egon/data/metadata/__init__.py | 14 ++++++++++---- 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/egon/data/datasets/district_heating_areas/__init__.py b/src/egon/data/datasets/district_heating_areas/__init__.py index c8f58ab22..beb0af576 100644 --- a/src/egon/data/datasets/district_heating_areas/__init__.py +++ b/src/egon/data/datasets/district_heating_areas/__init__.py @@ -850,7 +850,7 @@ def add_metadata(): }, ], } - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) db.submit_comment(meta_json, "demand", "egon_district_heating_areas") @@ -955,7 +955,7 @@ def add_metadata(): }, ], } - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) db.submit_comment( meta_json, "demand", "egon_map_zensus_district_heating_areas" diff --git a/src/egon/data/datasets/heat_demand/__init__.py b/src/egon/data/datasets/heat_demand/__init__.py index cb1fa7103..1b928fc62 100644 --- a/src/egon/data/datasets/heat_demand/__init__.py +++ b/src/egon/data/datasets/heat_demand/__init__.py @@ -719,7 +719,7 @@ def add_metadata(): }, ], } - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) db.submit_comment(meta_json, "demand", "egon_peta_heat") diff --git a/src/egon/data/datasets/osm/__init__.py b/src/egon/data/datasets/osm/__init__.py index 78d79ff6b..1ae599f21 100644 --- a/src/egon/data/datasets/osm/__init__.py +++ b/src/egon/data/datasets/osm/__init__.py @@ -225,7 +225,7 @@ def add_metadata(): ], } - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) db.submit_comment(meta_json, "openstreetmap", table) diff --git a/src/egon/data/datasets/renewable_feedin.py b/src/egon/data/datasets/renewable_feedin.py index 54673fb3b..202391631 100644 --- a/src/egon/data/datasets/renewable_feedin.py +++ b/src/egon/data/datasets/renewable_feedin.py @@ -750,7 +750,7 @@ def add_metadata(): } # Create json dump - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) # Add metadata as a comment to the table db.submit_comment( diff --git a/src/egon/data/datasets/scenario_capacities.py b/src/egon/data/datasets/scenario_capacities.py index dc20abf8e..4e9003f42 100755 --- a/src/egon/data/datasets/scenario_capacities.py +++ b/src/egon/data/datasets/scenario_capacities.py @@ -1006,7 +1006,7 @@ def add_metadata(): } # Create json dump - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) # Add metadata as a comment to the table db.submit_comment( diff --git a/src/egon/data/datasets/vg250/__init__.py b/src/egon/data/datasets/vg250/__init__.py index 90065fa4b..6f8ba7b12 100644 --- a/src/egon/data/datasets/vg250/__init__.py +++ b/src/egon/data/datasets/vg250/__init__.py @@ -130,6 +130,7 @@ def to_postgres(): ) +# TODO @jh:Move to metadata module, as yaml def add_metadata(): """Writes metadata JSON string into table comment.""" # Prepare variables @@ -263,9 +264,7 @@ def add_metadata(): ], } - meta_json = ( - "'" + json.dumps(meta) + "'" - ) # TODO @jh: Find me and remove + meta_json = json.dumps(meta) # TODO @jh: Find me and remove db.submit_comment( meta_json, vg250_config["processed"]["schema"], table diff --git a/src/egon/data/datasets/zensus_vg250.py b/src/egon/data/datasets/zensus_vg250.py index 48bde2e9a..b7e8df888 100755 --- a/src/egon/data/datasets/zensus_vg250.py +++ b/src/egon/data/datasets/zensus_vg250.py @@ -768,7 +768,7 @@ def add_metadata_vg250_zensus(): } # Create json dump - meta_json = "'" + json.dumps(meta) + "'" + meta_json = json.dumps(meta) # Add metadata as a comment to the table db.submit_comment( diff --git a/src/egon/data/metadata/__init__.py b/src/egon/data/metadata/__init__.py index 1add3ff74..8b9505eca 100755 --- a/src/egon/data/metadata/__init__.py +++ b/src/egon/data/metadata/__init__.py @@ -967,20 +967,26 @@ def contributors(authorlist): def upload_json_metadata(): """Upload json metadata into db from zenodo""" - - for path in importlib_resources.files(__name__).glob("*.json"): + # TODO @jh-RLI: This reads from "converted", either read form results + # or remove this completely once metadata generation is done + # by OMI / oemetaBuilder + for path in ( + importlib_resources.files(__name__) + .joinpath("results", "converted") + .glob("*.json") + ): split = path.name.split(".") if len(split) != 3: continue schema = split[0] table = split[1] - + logger.info(f"Process metadata comment for {schema}.{table} to db.") with open(path, "r") as infile: obj = parse_metadata(infile.read()) # TODO @jh-RLI: Deactivate license check for now validate_metadata(obj, check_license=False) - metadata = f"'{obj}'" + metadata = obj db.submit_comment(metadata, schema, table) logger.info(f"Metadata comment for {schema}.{table} stored.") From 0ca40bfc7565863123f60d5fa71040a54384f4da Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 19 Nov 2025 12:55:41 +0100 Subject: [PATCH 25/32] #1177: move builder code to omi, remove from egon-data --- .../data/metadata/oemetadataBuilder/README.md | 73 ---- .../metadata/oemetadataBuilder/__init__.py | 0 .../oemetadataBuilder/mixins/__init__.py | 0 .../metadata/oemetadataBuilder/package.py | 103 ----- .../metadata/oemetadataBuilder/resource.py | 411 ------------------ 5 files changed, 587 deletions(-) delete mode 100644 src/egon/data/metadata/oemetadataBuilder/README.md delete mode 100644 src/egon/data/metadata/oemetadataBuilder/__init__.py delete mode 100644 src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py delete mode 100644 src/egon/data/metadata/oemetadataBuilder/package.py delete mode 100644 src/egon/data/metadata/oemetadataBuilder/resource.py diff --git a/src/egon/data/metadata/oemetadataBuilder/README.md b/src/egon/data/metadata/oemetadataBuilder/README.md deleted file mode 100644 index 6efaafa0d..000000000 --- a/src/egon/data/metadata/oemetadataBuilder/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# oemetadataBuilder - -This module implements functionality to create new oemetadata documents as part of other processing steps. This design is useful to integrate oemetadata creation into a pipeline like egon-data. - -Note: Code will be moved to the `omi` package once python version is updates in egon-data and initial developments are done. This makes a lot of sense as the Builder implements a generic way of creating metadata and the use case is relevant in general. One great and highly relevant contribution from Regon to the OpenEnergyFamily. - -## Design - -The main concept we implement is metadata for datasets which can be managed and created in a modular and developer friendly way. The datasets definition meant here is more general then the egon-data dataset usage as multiple data resources can be part of that dataset. A egon-data dataset will implement the resource builder which can generate oemetadata for that single resource. Some of this metadata derived and automatically generated then some information which applies to more then one resource is added using templates and dataset-individual metadata can added using overlays where YAML files are used to specify metadata elements. The builder can apply all this layers of metadata information and "stack" them together into n resource metadata objects - as described one for each dataset. This part is inspired by the current metadata implementation where metadata is generated as part of ech the dataset task in the airflow pipeline. - -The second feature the builder implements is the dataset composition. It is named "package" as naming it dataset could cause some confusion. It is a class which manages all resources and applies them into the oemetadata v2 structure. It will be used in a new task in the pipeline which generates a single metadata document. - -Mixins are used as a common modularization pattern to keep the builder classes cleaner. - -## Usage - -In any dataset use: - -```python -def add_metadata_vg250_zensus(): - engine: Engine = db.engine() - schema, table = "boundaries", "egon_map_zensus_vg250" - yaml_path = "metadata/overlay/zensus/egon_map_zensus_vg250.yaml" - - (OEMetadataBuilder() - .from_template() - .apply_yaml(yaml_path) - .set_basic(name=f"{schema}.{table}") - .auto_resource_from_table( - engine, schema, table, - geom_cols=["zensus_geom"], - ) - .apply_field_hints_from_yaml() - .finalize(license_check=False) - .save_as_table_comment(engine, schema, table) - ) -``` - -OPTIONAL: in airflow pipeline we could use a helper like this one when strong metadata - -```python -# airflow callable -def write_oemetadata(engine: Engine, schema: str, table: str, yaml_path: str): - (OEMetadataBuilder() - .from_template() - .apply_yaml(yaml_path) - .set_basic(name=f"{schema}.{table}") - .auto_resource_from_table(engine, schema, table) - .apply_field_hints_from_yaml() - .finalize(license_check=False) - .save_as_table_comment(engine, schema, table) - ) -``` - -Then to build the package like the code below (not all implemented yet): - -```python -from egon.data.metadata.registry import iter_resources - -pack = OEMetadataPackage().set_root( - name="egon-datapackage-2025-08-13", - title="eGon data release", - description="All resources for this run", - id_="https://example.org/egon/datapackage/2025-08-13" -) - -for res in iter_resources(): - pack.add_resource(res) - -pack.finalize() -json_text = pack.as_json() - -``` diff --git a/src/egon/data/metadata/oemetadataBuilder/__init__.py b/src/egon/data/metadata/oemetadataBuilder/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py b/src/egon/data/metadata/oemetadataBuilder/mixins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/egon/data/metadata/oemetadataBuilder/package.py b/src/egon/data/metadata/oemetadataBuilder/package.py deleted file mode 100644 index bf7c08864..000000000 --- a/src/egon/data/metadata/oemetadataBuilder/package.py +++ /dev/null @@ -1,103 +0,0 @@ -from copy import deepcopy -from typing import Any, Dict -import json - -from omi.base import get_metadata_specification -from omi.validation import validate_metadata # parse_metadata -from sqlalchemy.engine import Engine - -from egon.data.metadata import settings - - -class OEMetadataPackage: - def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: - self.spec = get_metadata_specification(version) - - self._doc: Dict[str, Any] = { - "@context": self.spec.example["@context"], - "name": "", - "title": "", - "description": "", - "@id": "", - "resources": [], - "metaMetadata": self.spec.example["metaMetadata"], - } - self._validated = False - - def set_root( - self, - *, - name: str, - title: str = "", - description: str = "", - id_: str = "", - ) -> "OEMetadataPackage": - self._doc["name"] = name - self._doc["title"] = title - self._doc["description"] = description - self._doc["@id"] = id_ - self._validated = False - return self - - def add_resource( - self, - resource: dict, - *, - dedupe_by: str = "name", - overwrite: bool = True, - ) -> "OEMetadataPackage": - if dedupe_by and overwrite: - self._doc["resources"] = [ - r - for r in self._doc["resources"] - if r.get(dedupe_by) != resource.get(dedupe_by) - ] - self._doc["resources"].append(deepcopy(resource)) - self._validated = False - return self - - def add_from_full_document( - self, full_doc: dict, *, take_root_if_empty: bool = False - ) -> "OEMetadataPackage": - # Optionally fill root if still empty - if take_root_if_empty and not self._doc["name"]: - for k in ("name", "title", "description", "@id"): - if k in full_doc: - self._doc[k] = full_doc[k] - for r in full_doc.get("resources", []): - self.add_resource(r) - return self - - def add_from_table_comment( - self, engine: Engine, schema: str, table: str - ) -> "OEMetadataPackage": - sql = """ - SELECT obj_description((quote_ident(%s)||'.'||quote_ident(%s))::regclass, 'pg_class') AS comment - """ # noqa: E501 - with engine.begin() as conn: - comment = conn.exec_driver_sql(sql, (schema, table)).scalar() - if not comment: - return self - try: - full_doc = json.loads(comment) - except Exception: - return self - # Optionally validate the doc before merging - try: - validate_metadata(full_doc, check_license=False) - except Exception: - pass - return self.add_from_full_document(full_doc) - - def finalize(self, *, license_check: bool = True) -> "OEMetadataPackage": - validate_metadata(self._doc, check_license=license_check) - self._validated = True - return self - - def as_dict(self) -> dict: - if not self._validated: - self.finalize() - return deepcopy(self._doc) - - def as_json(self) -> str: - return json.dumps(self.as_dict(), ensure_ascii=False) diff --git a/src/egon/data/metadata/oemetadataBuilder/resource.py b/src/egon/data/metadata/oemetadataBuilder/resource.py deleted file mode 100644 index 402234d7f..000000000 --- a/src/egon/data/metadata/oemetadataBuilder/resource.py +++ /dev/null @@ -1,411 +0,0 @@ -from __future__ import annotations - -from copy import deepcopy -from dataclasses import dataclass -from typing import Any, Dict, List, Optional -import datetime as dt -import json - -from geoalchemy2 import Geometry -from omi.base import MetadataSpecification, get_metadata_specification -from omi.validation import validate_metadata # parse_metadata -from sqlalchemy import MetaData, Table, inspect -from sqlalchemy.dialects.postgresql.base import ischema_names -from sqlalchemy.engine import Engine -import yaml # PyYAML - -# ---- Optional: your project settings/hooks -# from egon.data import db, logger -from egon.data.metadata import settings - -# Geometry awareness for reflection -ischema_names["geometry"] = Geometry # generic -# You can add specific geometry columns later per-table via kwargs - - -def _today() -> str: - return dt.date.today().isoformat() - - -def _deep_merge(base: dict, override: dict) -> dict: - """ - Deep merge with 'override wins', recursively. - Lists are replaced (not merged) by default to avoid subtle duplication. - """ - out = deepcopy(base) - for k, v in override.items(): - if isinstance(v, dict) and isinstance(out.get(k), dict): - out[k] = _deep_merge(out[k], v) - else: - out[k] = deepcopy(v) - return out - - -def _sqlatype_to_oem_type(sa_type: str) -> str: - """ - Map SQLAlchemy reflected type string -> OEM v2 field.type - Keep it simple and deterministic; adjust as needed. - """ - t = sa_type.lower() - # geometry - if "geometry" in t: - return "geometry" - # integers - if any(x in t for x in ["int", "serial", "bigint", "smallint"]): - return "integer" - # floats / numeric - if any(x in t for x in ["float", "double", "numeric", "real", "decimal"]): - return "number" - # booleans - if "bool" in t: - return "boolean" - # timestamp/date/time - if "timestamp" in t or "timestamptz" in t: - return "datetime" - if t == "date": - return "date" - if t == "time": - return "time" - # text-ish - if any( - x in t for x in ["text", "char", "string", "uuid", "json", "jsonb"] - ): - return "string" - # fallback - return "string" - - -@dataclass -class ResourceField: - """ - Minimal implementation of oemetadata v2 resource structure. - Eases usage in Python. - """ - - name: str - description: Optional[str] = None - type: str = "string" - unit: Optional[str] = None - nullable: Optional[bool] = None - - def to_dict(self) -> dict: - d = { - "name": self.name, - "type": self.type, - } - # include optional keys only when provided - if self.description is not None: - d["description"] = self.description - if self.unit is not None: - d["unit"] = self.unit - if self.nullable is not None: - d["nullable"] = self.nullable - return d - - -class OEMetadataResourceBuilder: - """ - Single, reusable builder for OEP oemetadata v2 using omi as source of truth. - - Typical flow: - builder = OEMetadataBuilder().from_template() - .apply_yaml("dataset_meta.yaml") - .auto_resource_from_table(engine, "schema", "table", geom_cols=["geom"]) - .set_basic(name="schema.table", title="...", description="...") - .finalize() - payload = builder.as_json() # validated JSON string - builder.save_as_table_comment(db_engine, "schema", "table") # optional - """ # noqa: E501 - - def __init__(self, version: str = settings.OEMETADATA_VERSION) -> None: - self.spec: MetadataSpecification = get_metadata_specification(version) - self._meta: Dict[str, Any] = {} - self._validated: bool = False - - # ---- Required steps - - def from_template(self) -> "OEMetadataResourceBuilder": - """ - Start from omi's template plus selected bits from example - (context/metaMetadata). - Ensures keys exist (empty strings/structures as per spec). - """ - tpl = deepcopy(self.spec.template) if self.spec.template else {} - if self.spec.example: - # Copy @context + metaMetadata if present in example - if "@context" in self.spec.example: - tpl["@context"] = deepcopy(self.spec.example["@context"]) - if "metaMetadata" in self.spec.example: - tpl["metaMetadata"] = deepcopy( - self.spec.example["metaMetadata"] - ) - self._meta = tpl["resources"][0] - self._validated = False - return self - - def apply_yaml( - self, yaml_path: str | None = None, yaml_text: str | None = None - ) -> "OEMetadataResourceBuilder": - """ - Merge user-provided YAML overrides into the current metadata object. - You can allow either a file path or a YAML string (for testing). - """ - if yaml_path: - with open(yaml_path, "r", encoding="utf-8") as fh: - override = yaml.safe_load(fh) or {} - elif yaml_text: - override = yaml.safe_load(yaml_text) or {} - else: - override = {} - - self._meta = _deep_merge(self._meta, override) - self._validated = False - return self - - def set_basic( - self, - name: str, - title: Optional[str] = None, - description: Optional[str] = None, - language: Optional[List[str]] = None, - publication_date: Optional[str] = None, - dataset_id: Optional[str] = None, - ) -> "OEMetadataResourceBuilder": - """ - Convenience setter for common top-level fields. - """ - if publication_date is None: - publication_date = _today() - patch = { - "name": name, - "publicationDate": publication_date, - } - if title is not None: - patch["title"] = title - if description is not None: - patch["description"] = description - if language is not None: - patch["language"] = language - if dataset_id is not None: - patch["id"] = dataset_id - - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - def set_context(self, context_obj: dict) -> "OEMetadataResourceBuilder": - self._meta = _deep_merge(self._meta, {"context": context_obj}) - self._validated = False - return self - - def set_spatial( - self, - extent: Optional[str] = None, - resolution: Optional[str] = None, - location: Optional[Any] = None, - ) -> "OEMetadataResourceBuilder": - patch = {"spatial": {}} - if location is not None: - patch["spatial"]["location"] = location - if extent is not None: - patch["spatial"]["extent"] = extent - if resolution is not None: - patch["spatial"]["resolution"] = resolution - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - def set_temporal( - self, - reference_date: Optional[str] = None, - timeseries: Optional[dict] = None, - ) -> "OEMetadataResourceBuilder": - patch = {"temporal": {}} - if reference_date is not None: - # NOTE: your older code used 'referenceDate' vs - # 'reference_date' in places. - # OEM v2 uses 'referenceDate' (camelCase). Keep consistent here: - patch["temporal"]["referenceDate"] = reference_date - if timeseries is not None: - patch["temporal"]["timeseries"] = timeseries - self._meta = _deep_merge(self._meta, patch) - self._validated = False - return self - - # ---- Sources, licenses, contributors - - def add_source(self, source: dict) -> "OEMetadataResourceBuilder": - self._meta.setdefault("sources", []) - self._meta["sources"].append(source) - self._validated = False - return self - - def add_license(self, lic: dict) -> "OEMetadataResourceBuilder": - self._meta.setdefault("licenses", []) - self._meta["licenses"].append(lic) - self._validated = False - return self - - def add_contributor( - self, contributor: dict - ) -> "OEMetadataResourceBuilder": - self._meta.setdefault("contributors", []) - self._meta["contributors"].append(contributor) - self._validated = False - return self - - # ---- Resources - - def auto_resource_from_table( - self, - engine: Engine, - schema: str, - table: str, - *, - resource_name: Optional[str] = None, - format_: str = "PostgreSQL", - encoding: str = "UTF-8", - primary_key: Optional[List[str]] = None, - foreign_keys: Optional[List[dict]] = None, - geom_cols: Optional[List[str]] = None, - dialect: Optional[dict] = None, - overwrite_existing: bool = False, - ) -> "OEMetadataResourceBuilder": - """ - Introspect a DB table and create a single tabular data resource entry. - - - Maps SQLA types to OEM types - - Marks 'nullable' where possible - - Recognizes geometry columns (if given in geom_cols) as 'geometry' - - If overwrite_existing=False and a resource already exists with the same - name, it will be left as-is (you could add a flag to update instead). - """ - if geom_cols is None: - geom_cols = ["geom", "geometry", "geom_point", "geom_polygon"] - - # reflect - meta = MetaData() - tbl = Table(table, meta, schema=schema, autoload_with=engine) - - fields: List[ResourceField] = [] - for col in tbl.columns: - sa_t = str(col.type) - # if explicitly geometry by name, treat as geometry - col_type = ( - "geometry" - if col.name in geom_cols - else _sqlatype_to_oem_type(sa_t) - ) - fields.append( - ResourceField( - name=col.name, - description=None, - type=col_type, - unit=None, - nullable=col.nullable, - ) - ) - - if not resource_name: - resource_name = f"{schema}.{table}" - - resource = { - "name": resource_name, - # TODO: @jh-RLI The OEP will set this, - # consider if local usage is important - "path": None, - "type": "table", - "format": format_, - "encoding": encoding, - "schema": { - "fields": [f.to_dict() for f in fields], - "primaryKey": primary_key - or self._best_guess_pk(engine, schema, table), - "foreignKeys": foreign_keys or [], - }, - "dialect": dialect or {"delimiter": None, "decimalSeparator": "."}, - } - - # install resources array - self._meta.setdefault("resources", []) - if overwrite_existing: - self._meta["resources"] = [ - r - for r in self._meta["resources"] - if r.get("name") != resource_name - ] - # only add if not present - if not any( - r.get("name") == resource_name for r in self._meta["resources"] - ): - self._meta["resources"].append(resource) - - self._validated = False - return self - - def _best_guess_pk( - self, engine: Engine, schema: str, table: str - ) -> List[str]: - """ - Try to read PK columns via SQLAlchemy inspector, fallback to - ['id'] if found, else []. - """ - insp = inspect(engine) - pk = insp.get_pk_constraint(table, schema=schema) - cols = pk.get("constrained_columns") if pk else None - if cols: - return cols - # common fallback - columns = [c["name"] for c in insp.get_columns(table, schema=schema)] - return ["id"] if "id" in columns else [] - - # ---- Finalize/validate/serialize - - def finalize( - self, license_check: bool = False - ) -> "OEMetadataResourceBuilder": - """ - Make minimal guarantees & validate with omi. - """ - # Fill sane defaults if missing - # self._meta.setdefault("publicationDate", _today()) - self._meta.setdefault("language", ["en-EN"]) - - # TODO: @jh-RLI might be expensive - # parse + validate with omi - # parse_metadata expects string; serialize & round-trip to normalize - # text = json.dumps(self._meta, ensure_ascii=False) - # parsed = parse_metadata(text) - - # You can toggle license checks if you are mid-migration: - validate_metadata(self._meta, check_license=license_check) - - # Reassign parsed (it may normalize the structure) - # self._meta = parsed - self._validated = True - return self - - def as_dict(self) -> dict: - if not self._validated: - self.finalize() - return deepcopy(self._meta) - - def as_json(self) -> str: - return json.dumps(self.as_dict(), ensure_ascii=False) - - # ---- Optional convenience: store as comment on a table - - def save_as_table_comment( - self, engine: Engine, schema: str, table: str - ) -> None: - """ - Store metadata JSON as a COMMENT ON TABLE ... (PostgreSQL). - """ - payload = self.as_json().replace( - "'", "''" - ) # escape single-quotes for SQL literal - full = f"{schema}.{table}" - sql = f"COMMENT ON TABLE {full} IS '{payload}';" - with engine.begin() as conn: - conn.exec_driver_sql(sql) From aee700707710c797bbafac3f3b352ecd4275cf88 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Thu, 20 Nov 2025 09:58:21 +0100 Subject: [PATCH 26/32] #1177: remove file from outdated idea --- src/egon/data/metadata/overlays/README.md | 49 ----------------------- 1 file changed, 49 deletions(-) delete mode 100644 src/egon/data/metadata/overlays/README.md diff --git a/src/egon/data/metadata/overlays/README.md b/src/egon/data/metadata/overlays/README.md deleted file mode 100644 index d72587b1f..000000000 --- a/src/egon/data/metadata/overlays/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Description - -This directory is called “overlays” because it stored content for oemetadata generation oemetadata. The overlays job is to layer human-authored bits (titles, context, licenses, field hints) on top of what we auto-extract from the DB—rather than being the single source of truth. That mental model is base metadata → overlay additional metadata. Overlay can override what is added in previous steps. - -## Notes - -Much of the functionality does not have to be implemented in egon-data as it is general purpose and should be added to omi. Currently we have a conflict with the python version. - -Also omi provides functionality to add template metadata. In the example below you see an entry for the context. In most cases the Project will be the same for all datasets this is when templates come in handy as we can apply them to all datasets in one go. If you specify the context here then it will override the template entry for that specific dataset. This is why the overlay can also be viewed as individual metadata. - -## Usage - -For each dataset used in the pipeline there should be metadata created. If we can provide additional metadata which cant be auto inferred from data sources then we use this directory and add yaml files. - -A reasonable "minimal" yaml file would be what you see below. Keep in mind that this minimal content is not what is required. Here minimal refers more to what would be at least useful to add. - -Example for zensus_vg250.yaml - -```yaml -name: "society.destatis_zensus_population_per_ha_inside_germany" -title: "DESTATIS – Zensus 2011 – Population per hectare (inside Germany)" -description: > - National census in Germany in 2011 filtered to German borders and cells with population > 0. -language: ["de-DE", "en-EN"] -context: - homepage: "https://ego-n.org/" - documentation: "https://egon-data.readthedocs.io/en/latest/" - sourceCode: "https://github.com/openego/eGon-data" - contact: "https://ego-n.org/partners/" -spatial: - extent: "Germany" - resolution: "1 ha" -temporal: - referenceDate: "2011-12-31" -sources: - - title: "Statistisches Bundesamt (Destatis) – Ergebnisse des Zensus 2011 zum Download" - path: "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html" -licenses: - - name: "DL-DE-BY-2.0" - title: "Datenlizenz Deutschland – Namensnennung – Version 2.0" - path: "https://www.govdata.de/dl-de/by-2-0" -contributors: - - title: "Guido Pleßmann" - email: "http://github.com/gplssm" - comment: "Imported data" - - title: "Jonathan Amme" - email: "http://github.com/nesnoj" - comment: "Metadata extended" -``` From c05ca1d3cbc00e7068f09f5ca0b1dc1ca22fa858 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:16:30 +0100 Subject: [PATCH 27/32] #1177: add metadata conversion output to gitignore add testing / zensus dags to gitignore --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index dfe58380d..74606aa18 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,10 @@ docs/_build # Mypy Cache .mypy_cache/ + +# oemetadata +/src/egon/data/metadata/results/converted + +# Dev Airflow DAGS for testing +/src/egon/data/airflow/dags/mini_dag_zensu.py +/src/egon/data/airflow/dags/pipeline_mini_dag_metadata.py From e1fe6715f6072ebbd5fcd2a9c9f4316ed307e180 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:21:08 +0100 Subject: [PATCH 28/32] 1177: Add converted to v2 and to yaml transformed metadata definitions --- .../datasets/egon-data.dataset.yaml | 6 + .../datasets/egon-data.template.yaml | 40 +++ ...gon_map_zensus_climate_zones.resource.yaml | 194 ++++++++++++ ...on_map_zensus_mvgd_buildings.resource.yaml | 206 +++++++++++++ ...lding_electricity_peak_loads.resource.yaml | 277 +++++++++++++++++ ...gon_building_heat_peak_loads.resource.yaml | 272 +++++++++++++++++ ...ricity_demand_building_share.resource.yaml | 263 ++++++++++++++++ ...s_heat_demand_building_share.resource.yaml | 215 +++++++++++++ ...heat_demand_per_climate_zone.resource.yaml | 190 ++++++++++++ ..._egon_etrago_electricity_cts.resource.yaml | 247 +++++++++++++++ ...trago_electricity_households.resource.yaml | 247 +++++++++++++++ .../demand_egon_etrago_heat_cts.resource.yaml | 282 ++++++++++++++++++ .../demand_egon_heat_idp_pool.resource.yaml | 165 ++++++++++ ...timeseries_selected_profiles.resource.yaml | 175 +++++++++++ ...icity_profile_in_census_cell.resource.yaml | 255 ++++++++++++++++ ...tricity_profile_of_buildings.resource.yaml | 182 +++++++++++ ..._iee_household_load_profiles.resource.yaml | 166 +++++++++++ .../grid_egon_mv_grid_district.resource.yaml | 185 ++++++++++++ ...etmap_osm_buildings_filtered.resource.yaml | 204 +++++++++++++ ...tmap_osm_buildings_synthetic.resource.yaml | 202 +++++++++++++ ...tis_zensus_population_per_ha.resource.yaml | 176 +++++++++++ ...t_building_population_per_ha.resource.yaml | 193 ++++++++++++ 22 files changed, 4342 insertions(+) create mode 100644 src/egon/data/metadata/dataset_metadata/datasets/egon-data.dataset.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/datasets/egon-data.template.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_climate_zones.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_mvgd_buildings.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_electricity_peak_loads.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_heat_peak_loads.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_electricity_demand_building_share.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_heat_demand_building_share.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_daily_heat_demand_per_climate_zone.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_cts.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_households.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_heat_cts.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_idp_pool.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_timeseries_selected_profiles.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_in_census_cell.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_of_buildings.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_iee_household_load_profiles.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/grid_egon_mv_grid_district.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_filtered.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_synthetic.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/society_destatis_zensus_population_per_ha.resource.yaml create mode 100644 src/egon/data/metadata/dataset_metadata/resources/egon-data/society_egon_destatis_zensus_apartment_building_population_per_ha.resource.yaml diff --git a/src/egon/data/metadata/dataset_metadata/datasets/egon-data.dataset.yaml b/src/egon/data/metadata/dataset_metadata/datasets/egon-data.dataset.yaml new file mode 100644 index 000000000..bdcb362b1 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/datasets/egon-data.dataset.yaml @@ -0,0 +1,6 @@ +version: OEMetadata-2.0 +dataset: + name: 'egon-data' + title: '' + description: '' + '@id': null diff --git a/src/egon/data/metadata/dataset_metadata/datasets/egon-data.template.yaml b/src/egon/data/metadata/dataset_metadata/datasets/egon-data.template.yaml new file mode 100644 index 000000000..d12b9fb52 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/datasets/egon-data.template.yaml @@ -0,0 +1,40 @@ +context: + grantNo: 03EI1002 + fundingAgency: Bundesministerium f\u00fcr Wirtschaft und Energie + sourceCode: https://github.com/openego/eGon-data + documentation: https://egon-data.readthedocs.io/en/latest/ + publisherLogo: https://ego-n.org/images/eGon_logo_noborder_transbg.svg + homepage: https://ego-n.org/ + contact: https://ego-n.org/partners/ + title: '' + fundingAgencyLogo: https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3 + publisher: '' +licenses: +- name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und Metadaten + d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) + vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, bearbeitet + sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten und Daten Anderer + zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen verbunden + werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte und Anwendungen + in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen Netzwerken eingebunden + werden.\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk + enthalten sind:\n(1) Bezeichnung des Bereitstellers nach dessen Ma\u00dfgabe,\n(2) + der Vermerk Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 oder + dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014; + \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\u00e4ndert) + copyrightStatement: null +languages: +- undefined +topics: +- 'model_draft' +keywords: +- '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_climate_zones.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_climate_zones.resource.yaml new file mode 100644 index 000000000..ac37ce762 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_climate_zones.resource.yaml @@ -0,0 +1,194 @@ +'@id': null +name: boundaries.egon_map_zensus_climate_zones +title: Map census cells to era5 climate zones +path: WILL_BE_SET_AT_PUBLICATION +description: Mapping table of census cells to era5 climate zones +subject: +- name: '' + '@id': null +publicationDate: '2022-08-06' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '1' + resolutionUnit: ha + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: ERA5 global reanalysis + authors: [] + description: ERA5 is the fifth generation ECMWF reanalysis for the global climate + and weather for the past 4 to 7 decades. Currently data is available from 1950, + split into Climate Data Store entries for 1950-1978 (preliminary back extension) + and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces + the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) + for more information. + publicationYear: null + path: https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency + sourceLicenses: + - name: Licence to use Copernicus Products + title: Licence to use Copernicus Products + path: https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf + instruction: 'This Licence is free of charge, worldwide, non-exclusive, royalty + free and perpetual. Access to Copernicus Products is given for any purpose in + so far as it is lawful, whereas use may include, but is not limited to: reproduction; + distribution; communication to the public; adaptation, modification and combination + with other data and information; or any combination of the foregoing' + attribution: © Copernicus Climate Change Service (C3S) Climate Data Store + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: zensus_population_id + description: 'Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)' + type: integer + nullable: false + unit: null + isAbout: [] + valueReference: [] + - name: w_id + description: 'Weather cell id (reference to: supply.egon_era5_weather_cells)' + type: Integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - z + - e + - n + - s + - u + - s + - _ + - p + - o + - p + - u + - l + - a + - t + - i + - o + - n + - _ + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_mvgd_buildings.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_mvgd_buildings.resource.yaml new file mode 100644 index 000000000..083c4b33a --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/boundaries_egon_map_zensus_mvgd_buildings.resource.yaml @@ -0,0 +1,206 @@ +'@id': null +name: boundaries.egon_map_zensus_mvgd_buildings +title: eGon buildings mapping table +path: WILL_BE_SET_AT_PUBLICATION +description: Mapping table for all used buildings via project internal building_id. + Additional info about if it is a synthetic or a openstreetmap building, where it + is located (census cell_id, mvgd_id), in which sector (residential, cts) the building + is used, if there are heat or electricity profiles. +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: sector + description: Sector (cts, residential) + type: characters varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: zensus_population_id + description: 'Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: bus_id + description: 'MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: electricity + description: True if respectives electricity demand profile exists + type: bool + nullable: true + unit: None + isAbout: [] + valueReference: [] + - name: heat + description: True if respectives heat demand profile exists + type: bool + nullable: true + unit: None + isAbout: [] + valueReference: [] + - name: osm + description: is OSM building if true else syntheticaly created + type: bool + nullable: true + unit: None + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + - ',' + - ' ' + - s + - e + - c + - t + - o + - r + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_electricity_peak_loads.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_electricity_peak_loads.resource.yaml new file mode 100644 index 000000000..e241a7936 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_electricity_peak_loads.resource.yaml @@ -0,0 +1,277 @@ +'@id': null +name: demand.egon_building_electricity_peak_loads +title: Electricity peak loads for egon buildings +path: WILL_BE_SET_AT_PUBLICATION +description: 'Electricity peak load table of residential and cts buildings with voltage + level for two scenarios ' +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: DemandRegio + authors: [] + description: Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen + Auflösung von Energienachfragen + publicationYear: null + path: https://doi.org/10.34805/ffe-119-20 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scenario + description: Scenario (eGon100RE, eGon2035) + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: sector + description: Sector (cts, residential) + type: characters varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: peak_load_in_w + description: Peak load value + type: real + nullable: true + unit: W + isAbout: [] + valueReference: [] + - name: voltage_level + description: Voltage level of building connection after MitnetzStrom Classification + type: int + nullable: true + unit: None + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + - ',' + - ' ' + - s + - c + - e + - n + - a + - r + - i + - o + - ',' + - ' ' + - s + - e + - c + - t + - o + - r + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_heat_peak_loads.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_heat_peak_loads.resource.yaml new file mode 100644 index 000000000..591ca0092 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_building_heat_peak_loads.resource.yaml @@ -0,0 +1,272 @@ +'@id': null +name: demand.egon_building_heat_peak_loads +title: Heat peak loads for egon buildings +path: WILL_BE_SET_AT_PUBLICATION +description: 'Heat peak load table of residential and cts buildings for two scenarios ' +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: Verwaltungsgebiete 1:250 000 (Ebenen) + authors: [] + description: Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen + Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen + Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung + der Verwaltungsebene des jeweiligen Landes. + publicationYear: null + path: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland – Namensnennung – Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland + - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten + dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) + vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte + übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt + und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und + externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht + öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist + sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) + Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz + Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf + den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf + den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) + bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen + oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, + dass die Daten geändert wurden. + attribution: © Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert) + copyrightStatement: null +- title: ERA5 global reanalysis + authors: [] + description: ERA5 is the fifth generation ECMWF reanalysis for the global climate + and weather for the past 4 to 7 decades. Currently data is available from 1950, + split into Climate Data Store entries for 1950-1978 (preliminary back extension) + and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces + the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) + for more information. + publicationYear: null + path: https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency + sourceLicenses: + - name: Licence to use Copernicus Products + title: Licence to use Copernicus Products + path: https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf + instruction: 'This Licence is free of charge, worldwide, non-exclusive, royalty + free and perpetual. Access to Copernicus Products is given for any purpose in + so far as it is lawful, whereas use may include, but is not limited to: reproduction; + distribution; communication to the public; adaptation, modification and combination + with other data and information; or any combination of the foregoing' + attribution: © Copernicus Climate Change Service (C3S) Climate Data Store + copyrightStatement: null +- title: Peta5 0 1 HD res + authors: [] + description: modelled Heat Demand distribution (in GJ per hectare grid cell) for + residential heat demands for space heating and hot water for the year 2015 using + HRE4 data and the combined top-down bottom-up approach of HRE4 + publicationYear: null + path: https://s-eenergies-open-data-euf.hub.arcgis.com/search + sourceLicenses: + - name: Creative Commons Attribution 4.0 International + title: CC BY 4.0 + path: https://creativecommons.org/licenses/by/4.0/ + instruction: 'You are free: To Share, To Adapt; As long as you: Attribute!' + attribution: \u00a9 Flensburg, Halmstad and Aalborg universities + copyrightStatement: null +- title: Peta5 0 1 HD ser + authors: [] + description: modelled Heat Demand distribution (in GJ per hectare grid cell) for + service-sector heat demands for space heating, hot water and process heat for + the year 2015 using HRE4 data and the combined top-down bottom-up approach of + HRE4 + publicationYear: null + path: https://s-eenergies-open-data-euf.hub.arcgis.com/search + sourceLicenses: + - name: Creative Commons Attribution 4.0 International + title: CC BY 4.0 + path: https://creativecommons.org/licenses/by/4.0/ + instruction: 'You are free: To Share, To Adapt; As long as you: Attribute!' + attribution: \u00a9 Flensburg, Halmstad and Aalborg universities + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scenario + description: Scenario (eGon100RE, eGon2035) + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: sector + description: Sector (cts, residential) + type: characters varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: peak_load_in_w + description: Peak load value + type: real + nullable: true + unit: W + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + - ',' + - ' ' + - s + - c + - e + - n + - a + - r + - i + - o + - ',' + - ' ' + - s + - e + - c + - t + - o + - r + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_electricity_demand_building_share.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_electricity_demand_building_share.resource.yaml new file mode 100644 index 000000000..ccd40e5e9 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_electricity_demand_building_share.resource.yaml @@ -0,0 +1,263 @@ +'@id': null +name: demand.egon_cts_electricity_demand_building_share +title: Electricity demand profile share of egon cts buildings +path: WILL_BE_SET_AT_PUBLICATION +description: Electricity demand profile share of egon cts buildings for two scenarios + with bus_id +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: DemandRegio + authors: [] + description: Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen + Auflösung von Energienachfragen + publicationYear: null + path: https://doi.org/10.34805/ffe-119-20 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scenario + description: 'Scenario (eGon100RE, eGon2035) (reference: demand.egon_etrago_electricity_cts.scn_name)' + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: bus_id + description: 'MVGD bus_id (reference to: demand.egon_etrago_electricity_cts.bus_id, + grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: profile_share + description: Profile share of MVGD elctricity profile + type: real + nullable: true + unit: None + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + - ',' + - ' ' + - s + - c + - e + - n + - a + - r + - i + - o + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_heat_demand_building_share.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_heat_demand_building_share.resource.yaml new file mode 100644 index 000000000..d16808c1f --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_cts_heat_demand_building_share.resource.yaml @@ -0,0 +1,215 @@ +'@id': null +name: demand.egon_cts_heat_demand_building_share +title: Heat demand profile share of egon cts buildings +path: WILL_BE_SET_AT_PUBLICATION +description: Heat demand profile share of egon cts buildings for two scenarios with + bus_id +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: Peta5 0 1 HD ser + authors: [] + description: modelled Heat Demand distribution (in GJ per hectare grid cell) for + service-sector heat demands for space heating, hot water and process heat for + the year 2015 using HRE4 data and the combined top-down bottom-up approach of + HRE4 + publicationYear: null + path: https://s-eenergies-open-data-euf.hub.arcgis.com/search + sourceLicenses: + - name: Creative Commons Attribution 4.0 International + title: CC BY 4.0 + path: https://creativecommons.org/licenses/by/4.0/ + instruction: 'You are free: To Share, To Adapt; As long as you: Attribute!' + attribution: \u00a9 Flensburg, Halmstad and Aalborg universities + copyrightStatement: null +- title: Peta5 0 1 HD res + authors: [] + description: modelled Heat Demand distribution (in GJ per hectare grid cell) for + residential heat demands for space heating and hot water for the year 2015 using + HRE4 data and the combined top-down bottom-up approach of HRE4 + publicationYear: null + path: https://s-eenergies-open-data-euf.hub.arcgis.com/search + sourceLicenses: + - name: Creative Commons Attribution 4.0 International + title: CC BY 4.0 + path: https://creativecommons.org/licenses/by/4.0/ + instruction: 'You are free: To Share, To Adapt; As long as you: Attribute!' + attribution: \u00a9 Flensburg, Halmstad and Aalborg universities + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scenario + description: Scenario (eGon100RE, eGon2035) + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: bus_id + description: 'MVGD bus_id (reference to: demand.egon_etrago_heat_cts.bus_id, grid.egon_hvmv_substation.bus_id, + grid.egon_mv_grid_district.bus_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: profile_share + description: 'Profile share of MVGD heat profile ' + type: real + nullable: true + unit: None + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + - ',' + - ' ' + - s + - c + - e + - n + - a + - r + - i + - o + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_daily_heat_demand_per_climate_zone.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_daily_heat_demand_per_climate_zone.resource.yaml new file mode 100644 index 000000000..abc49b448 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_daily_heat_demand_per_climate_zone.resource.yaml @@ -0,0 +1,190 @@ +'@id': null +name: demand.egon_daily_heat_demand_per_climate_zone +title: eGon daily heat demand share per climate zone +path: WILL_BE_SET_AT_PUBLICATION +description: Share of daily heat demands per climate zone +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: climate + resolutionUnit: zones + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: ERA5 global reanalysis + authors: [] + description: ERA5 is the fifth generation ECMWF reanalysis for the global climate + and weather for the past 4 to 7 decades. Currently data is available from 1950, + split into Climate Data Store entries for 1950-1978 (preliminary back extension) + and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces + the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) + for more information. + publicationYear: null + path: https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency + sourceLicenses: + - name: Licence to use Copernicus Products + title: Licence to use Copernicus Products + path: https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf + instruction: 'This Licence is free of charge, worldwide, non-exclusive, royalty + free and perpetual. Access to Copernicus Products is given for any purpose in + so far as it is lawful, whereas use may include, but is not limited to: reproduction; + distribution; communication to the public; adaptation, modification and combination + with other data and information; or any combination of the foregoing' + attribution: © Copernicus Climate Change Service (C3S) Climate Data Store + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: climate_zone + description: Name of the corresponding climate zone + type: str + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: daily_demand_share + description: Share of annual heat demand for each day + type: double precision + nullable: true + unit: per unit + isAbout: [] + valueReference: [] + - name: day_of_year + description: Number of day in the year + type: integer + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: temperature_class + description: Temperature class (considering the mean outside temeperature during + the day) + type: integer + nullable: true + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - c + - l + - i + - m + - a + - t + - e + - _ + - z + - o + - n + - e + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: '' +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_cts.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_cts.resource.yaml new file mode 100644 index 000000000..5f7498b43 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_cts.resource.yaml @@ -0,0 +1,247 @@ +'@id': null +name: demand.egon_etrago_electricity_cts +title: MV cts electricity demand profiles +path: WILL_BE_SET_AT_PUBLICATION +description: Aggregated electricity demand profile of cts sector at mvgd bus level + for two scenarios. Values are given in MWh. +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: MV + resolutionUnit: Grid + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '2016-01-01' + end: '2016-12-31' + resolutionValue: '1' + resolutionUnit: h + alignment: left + aggregationType: sum +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: DemandRegio + authors: [] + description: Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen + Auflösung von Energienachfragen + publicationYear: null + path: https://doi.org/10.34805/ffe-119-20 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: bus_id + description: 'MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, + grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scn_name + description: 'Scenario name (reference: demand.egon_cts_electricity_demand_building_share.scenario)' + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: p_set + description: Cts electricity demand load timeseries for one year (8760 hours) + in MWh + type: array of real + nullable: true + unit: MWh + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + - ',' + - ' ' + - s + - c + - n + - _ + - n + - a + - m + - e + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_households.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_households.resource.yaml new file mode 100644 index 000000000..a8b2ad89b --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_electricity_households.resource.yaml @@ -0,0 +1,247 @@ +'@id': null +name: demand.egon_etrago_electricity_households +title: MV residential electricity demand profiles +path: WILL_BE_SET_AT_PUBLICATION +description: Aggregated electricity demand profile of residential sector at mvgd bus + level for two scenarios. Values are given in MWh. +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: MV + resolutionUnit: Grid + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '2016-01-01' + end: '2016-12-31' + resolutionValue: '1' + resolutionUnit: h + alignment: left + aggregationType: sum +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: DemandRegio + authors: [] + description: Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen + Auflösung von Energienachfragen + publicationYear: null + path: https://doi.org/10.34805/ffe-119-20 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: bus_id + description: 'MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, + grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scn_name + description: Scenario name + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: p_set + description: Residential electricity demand load timeseries for one year (8760 + hours) in MWh + type: array of double precission + nullable: true + unit: MWh + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + - ',' + - ' ' + - s + - c + - n + - _ + - n + - a + - m + - e + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_heat_cts.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_heat_cts.resource.yaml new file mode 100644 index 000000000..2ac89c575 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_etrago_heat_cts.resource.yaml @@ -0,0 +1,282 @@ +'@id': null +name: demand.egon_etrago_heat_cts +title: MV cts heat demand profiles +path: WILL_BE_SET_AT_PUBLICATION +description: Aggregated heat demand profile of cts sector at mvgd bus level for two + scenarios. Values are given in MWh. +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: MV + resolutionUnit: Grid + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2023-03-13' + timeseries: + - start: '2016-01-01' + end: '2016-12-31' + resolutionValue: '1' + resolutionUnit: h + alignment: left + aggregationType: sum +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: Verwaltungsgebiete 1:250 000 (Ebenen) + authors: [] + description: Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen + Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen + Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung + der Verwaltungsebene des jeweiligen Landes. + publicationYear: null + path: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland – Namensnennung – Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland + - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten + dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) + vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte + übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt + und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und + externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht + öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist + sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) + Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz + Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf + den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf + den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) + bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen + oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, + dass die Daten geändert wurden. + attribution: © Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert) + copyrightStatement: null +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +- title: ERA5 global reanalysis + authors: [] + description: ERA5 is the fifth generation ECMWF reanalysis for the global climate + and weather for the past 4 to 7 decades. Currently data is available from 1950, + split into Climate Data Store entries for 1950-1978 (preliminary back extension) + and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces + the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) + for more information. + publicationYear: null + path: https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency + sourceLicenses: + - name: Licence to use Copernicus Products + title: Licence to use Copernicus Products + path: https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf + instruction: 'This Licence is free of charge, worldwide, non-exclusive, royalty + free and perpetual. Access to Copernicus Products is given for any purpose in + so far as it is lawful, whereas use may include, but is not limited to: reproduction; + distribution; communication to the public; adaptation, modification and combination + with other data and information; or any combination of the foregoing' + attribution: © Copernicus Climate Change Service (C3S) Climate Data Store + copyrightStatement: null +- title: Peta5 0 1 HD ser + authors: [] + description: modelled Heat Demand distribution (in GJ per hectare grid cell) for + service-sector heat demands for space heating, hot water and process heat for + the year 2015 using HRE4 data and the combined top-down bottom-up approach of + HRE4 + publicationYear: null + path: https://s-eenergies-open-data-euf.hub.arcgis.com/search + sourceLicenses: + - name: Creative Commons Attribution 4.0 International + title: CC BY 4.0 + path: https://creativecommons.org/licenses/by/4.0/ + instruction: 'You are free: To Share, To Adapt; As long as you: Attribute!' + attribution: \u00a9 Flensburg, Halmstad and Aalborg universities + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: bus_id + description: 'MV bus_id (reference: demand.egon_cts_heat_demand_building_share.bus_id, + grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: scn_name + description: 'Scenario name (reference: demand.egon_cts_heat_demand_building_share.scenario)' + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: p_set + description: Cts heat demand load timeseries for one year (8760 hours) in MWh + type: array of real + nullable: true + unit: MWh + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + - ',' + - ' ' + - s + - c + - n + - _ + - n + - a + - m + - e + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_idp_pool.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_idp_pool.resource.yaml new file mode 100644 index 000000000..6ad2cca5a --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_idp_pool.resource.yaml @@ -0,0 +1,165 @@ +'@id': null +name: demand.egon_heat_idp_pool +title: eGon pool of residential intra-day heat demand profiles +path: WILL_BE_SET_AT_PUBLICATION +description: 'Pool of individual, normalized intra-day heat demand profiles of residential + buildings. Timeseries only represent one day but are used to be merged for a whole + year ' +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: none + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '' + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: 2011-01-01 00:00 + end: 2011-12-31 23:00 + resolutionValue: '1' + resolutionUnit: h + alignment: left + aggregationType: sum +sources: +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: house + description: Type of residential house, either SFH (single family house) or MFH + (multi family house) + type: str + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: idp + description: Normalized intra-day-profile + type: array of double precision + nullable: false + unit: per unit + isAbout: [] + valueReference: [] + - name: index + description: Unique identifier + type: integer + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: temperature_class + description: Temperature class (considering the mean outside temeperature during + the day) + type: integer + nullable: true + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - i + - n + - d + - e + - x + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: '' +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_timeseries_selected_profiles.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_timeseries_selected_profiles.resource.yaml new file mode 100644 index 000000000..c25acdc14 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_heat_timeseries_selected_profiles.resource.yaml @@ -0,0 +1,175 @@ +'@id': null +name: demand.egon_heat_timeseries_selected_profiles +title: eGon selected intra-day profiles per building +path: WILL_BE_SET_AT_PUBLICATION +description: List of selected intra-day-profiles for each residential building in + Germany +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: none + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at ''referenceDate'' + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: Index of the corresponding building + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: selected_idp_profiles + description: List of selected profiles for each day in the year + type: array of integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: zensus_population_id + description: Index of corresponding census cell + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - i + - l + - d + - i + - n + - g + - _ + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: '' +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_in_census_cell.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_in_census_cell.resource.yaml new file mode 100644 index 000000000..01f759fef --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_in_census_cell.resource.yaml @@ -0,0 +1,255 @@ +'@id': null +name: demand.egon_household_electricity_profile_in_census_cell +title: eGon household electricity profiles in census cells +path: WILL_BE_SET_AT_PUBLICATION +description: Mapping table for residential electricity profiles to census cell including + scaling factors for two scenarios (eGon2035, eGon100RE). +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '100' + resolutionUnit: m x 100 m + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: Verwaltungsgebiete 1:250 000 (Ebenen) + authors: [] + description: Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen + Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen + Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung + der Verwaltungsebene des jeweiligen Landes. + publicationYear: null + path: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland – Namensnennung – Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland + - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten + dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) + vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte + übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt + und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und + externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht + öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist + sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) + Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz + Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf + den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf + den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) + bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen + oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, + dass die Daten geändert wurden. + attribution: © Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert) + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +- title: DemandRegio + authors: [] + description: Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen + Auflösung von Energienachfragen + publicationYear: null + path: https://doi.org/10.34805/ffe-119-20 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nailend + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Updated metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: cell_id + description: 'Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: grid_id + description: Descriptive ID of census cell given by Statistisches Bundesamt (Destatis) + type: character varying + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: cell_profile_ids + description: Household profile ids + type: array of integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: nuts3 + description: NUTS-3 ID of the census cell + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: nuts1 + description: NUTS-1 ID of the census cell + type: character varying + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: factor_2035 + description: Scaling factor for all profiles in the respective census cell for + scenario eGon2035 + type: double precision + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: factor_2050 + description: Scaling factor for all profiles in the respective census cell for + scenario eGon100RE + type: double precision + nullable: true + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - c + - e + - l + - l + - _ + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_of_buildings.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_of_buildings.resource.yaml new file mode 100644 index 000000000..0c06800d0 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_egon_household_electricity_profile_of_buildings.resource.yaml @@ -0,0 +1,182 @@ +'@id': null +name: demand.egon_household_electricity_profile_of_buildings +title: eGon household electricity profiles of buildings +path: WILL_BE_SET_AT_PUBLICATION +description: 'Mapping table for residential electricity profiles to buildings via + project internal building_id and corresponding census cell_id. ' +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nailend + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Updated metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: id + description: Unique identifier + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_residential.id, + openstreetmap.osm_buildings_synthetic.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: cell_id + description: 'Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: profile_id + description: 'ID of iee household profiles (reference to: iee_household_load_profiles.type)' + type: character varying + nullable: false + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_iee_household_load_profiles.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_iee_household_load_profiles.resource.yaml new file mode 100644 index 000000000..1757507fd --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/demand_iee_household_load_profiles.resource.yaml @@ -0,0 +1,166 @@ +'@id': null +name: demand.iee_household_load_profiles +title: eGon pool of electricity household load profiles +path: WILL_BE_SET_AT_PUBLICATION +description: '100.000 annual profiles in hourly resolution of electricity demand of + private households for different household types (singles, couples, other) with + varying number of elderly and children. The profiles were created using a bottom-up + load profile generator by Fraunhofer IEE developed in the Bachelors thesis "Auswirkungen + verschiedener Haushaltslastprofile auf PV-Batterie-Systeme" by Jonas Haack, Fachhochschule + Flensburg, December 2012. The columns are named as follows: "a", + e.g. P2a0000 is the first profile of a couples household with 2 children. See publication + below for the list of prefixes. Values are given in Wh.' +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '' + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-06-10' + timeseries: + - start: '2016-01-01' + end: '2016-12-31' + resolutionValue: '1' + resolutionUnit: h + alignment: left + aggregationType: sum +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nailend + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Updated metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: id + description: Unique identifier + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: type + description: Profile id + type: character(8) + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: load_in_wh + description: Residential demand load timeseries for one year (8760 hours) in Wh + type: array of real + nullable: true + unit: Wh + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/grid_egon_mv_grid_district.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/grid_egon_mv_grid_district.resource.yaml new file mode 100644 index 000000000..0813fca50 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/grid_egon_mv_grid_district.resource.yaml @@ -0,0 +1,185 @@ +'@id': null +name: grid.egon_mv_grid_district +title: eGon MV grid districts +path: WILL_BE_SET_AT_PUBLICATION +description: Mapping table of bus_id to polygons shapes for MV grid districts. +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: MV + resolutionUnit: Grids + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: Verwaltungsgebiete 1:250 000 (Ebenen) + authors: [] + description: Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen + Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen + Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung + der Verwaltungsebene des jeweiligen Landes. + publicationYear: null + path: https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland – Namensnennung – Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland + - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten + dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) + vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte + übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt + und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und + externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht + öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist + sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) + Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz + Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf + den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf + den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) + bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen + oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, + dass die Daten geändert wurden. + attribution: © Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert) + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: building_id + description: 'Building id (reference to: openstreetmap.osm_buildings_filtered.id, + openstreetmap.osm_buildings_synthetic.id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: bus_id + description: 'MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: geom + description: 'Polygon of mv grid district (CRS: ERTS89/EPSG:3035)' + type: geometry + nullable: true + unit: None + isAbout: [] + valueReference: [] + - name: area + description: Area covered by mvgd + type: double precision + nullable: true + unit: m^2 + isAbout: [] + valueReference: [] + primaryKey: + - b + - u + - s + - _ + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_filtered.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_filtered.resource.yaml new file mode 100644 index 000000000..6aab50119 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_filtered.resource.yaml @@ -0,0 +1,204 @@ +'@id': null +name: openstreetmap.osm_buildings_filtered +title: eGon filtered buildings from openstreetmap +path: WILL_BE_SET_AT_PUBLICATION +description: Tag filtered list of buildings from OpenStreetMap which might have electricity + or heat demand - (c) OpenStreetMap contributors +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-01-01' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Created metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: osm_id + description: Openstreetmap identifier + type: bigint + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: amenitiy + description: Type of amenitiy if given + type: text + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: building + description: Type of building + type: text + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: name + description: Name of the building if given + type: text + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: geom_building + description: 'Polygon of the building (CRS: ERTS89/EPSG:3035)' + type: geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: area + description: Surface area of building + type: double precision + nullable: true + unit: m^2 + isAbout: [] + valueReference: [] + - name: geom_point + description: 'Centroid of the building (CRS: ERTS89/EPSG:3035)' + type: geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: tags + description: Opensteetmap tags assigned to the building + type: hstore + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: id + description: 'Unique identifyer and project internal building id (reference to: + demand.egon_hp_capacity_buildings.building_id, demand.egon_cts_heat_demand_building_share.building_id, + demand.egon_cts_electricity_demand_building_share.building_id, demand.egon_household_electricity_profile_of_buildings.building_id, + demand.egon_building_heat_peak_loads.building_id, demand.egon_building_electricity_peak_loads.building_id, + boundaries.egon_map_zensus_mvgd_buildings.building_id)' + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_synthetic.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_synthetic.resource.yaml new file mode 100644 index 000000000..00b8bd74c --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/openstreetmap_osm_buildings_synthetic.resource.yaml @@ -0,0 +1,202 @@ +'@id': null +name: openstreetmap.osm_buildings_synthetic +title: eGon synthetic residential buildings +path: WILL_BE_SET_AT_PUBLICATION +description: ' List of generated synthetic buildings ' +subject: +- name: '' + '@id': null +publicationDate: '2022-09-14' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: Buildings + resolutionUnit: '' + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '2022-09-14' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: 'Data bundle for egon-data: A transparent and reproducible data processing + pipeline for energy system modeling' + authors: [] + description: egon-data provides a transparent and reproducible open data based data + processing pipeline for generating data models suitable for energy system modeling. + The data is customized for the requirements of the research project eGon. The + research project aims to develop tools for an open and cross-sectoral planning + of transmission and distribution grids. For further information please visit the + eGon project website or its Github repository. egon-data retrieves and processes + data from several different external input sources. As not all data dependencies + can be downloaded automatically from external sources we provide a data bundle + to be downloaded by egon-data. + publicationYear: null + path: https://doi.org/10.5281/zenodo.6630616 + sourceLicenses: + - name: CC-BY-4.0 + title: Creative Commons Attribution 4.0 International + path: https://creativecommons.org/licenses/by/4.0/legalcode + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute.' + attribution: © eGon-data developers, © Leibniz Institute for Applied Geophysics, + © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, + © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden + 2008, © Statistische Ämter des Bundes und der Länder 2021 + copyrightStatement: null +- title: eGon-data + authors: [] + description: Workflow to download, process and generate data sets suitable for the + further research conducted in the project eGon (https://ego-n.org/) + publicationYear: null + path: https://github.com/openego/eGon-data + sourceLicenses: + - name: AGPL-3.0 License + title: GNU Affero General Public License v3.0 + path: https://www.gnu.org/licenses/agpl-3.0.de.html + instruction: Permissions of this strongest copyleft license are conditioned on + making available complete source code of licensed works and modifications, which + include larger works using a licensed work, under the same license. Copyright + and license notices must be preserved. Contributors provide an express grant + of patent rights. When a modified version is used to provide a service over + a network, the complete source code of the modified version must be made available. + attribution: © eGon-data developers + copyrightStatement: null +- title: OpenStreetMap Data Extracts (Geofabrik) + authors: [] + description: Full data extract of OpenStreetMap data for defined spatial extent + at 2022-01-01 + publicationYear: null + path: https://download.geofabrik.de/europe/germany-210101.osm.pbf + sourceLicenses: + - name: ODbL-1.0 + title: Open Data Commons Open Database License 1.0 + path: https://opendatacommons.org/licenses/odbl/1.0/ + instruction: 'You are free: To Share, To Create, To Adapt; As long as you: Attribute, + Share-Alike, Keep open!' + attribution: © OpenStreetMap contributors + copyrightStatement: null +contributors: +- title: ClaraBuettner + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Create metadata +- title: nailend + path: null + organization: null + roles: + - '' + date: '2022-09-12' + object: metadata + comment: Updated metadata +- title: nesnoj + path: null + organization: null + roles: + - '' + date: '2022-09-14' + object: metadata + comment: Update metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: id + description: 'Unique identifyer and project internal building id (reference to: + openstreetmap.osm_buildings_residential.id, demand.egon_household_electricity_profile_of_buildings.building_id)' + type: character varying + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: area + description: Surface area of building + type: real + nullable: true + unit: m^2 + isAbout: [] + valueReference: [] + - name: building + description: Type of building (residential or cts) + type: character varying(11) + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: cell_id + description: 'Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)' + type: character varying + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: geom_building + description: 'Polygon of building (CRS: ERTS89/EPSG:3035)' + type: geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: geom_point + description: 'Centroid of building (CRS: ERTS89/EPSG:3035)' + type: geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: n_amenities_inside + description: Number of amenities inside the building (always zero if not cts building) + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_destatis_zensus_population_per_ha.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_destatis_zensus_population_per_ha.resource.yaml new file mode 100644 index 000000000..495a31ea0 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_destatis_zensus_population_per_ha.resource.yaml @@ -0,0 +1,176 @@ +'@id': null +name: society.destatis_zensus_population_per_ha +title: DESTATIS - Zensus 2011 - Population per hectar +path: WILL_BE_SET_AT_PUBLICATION +description: National census in Germany in 2011. +subject: +- name: '' + '@id': null +publicationDate: '2022-08-06' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '1' + resolutionUnit: ha + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: id + description: Unique identifier + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: grid_id + description: Grid number of source + type: string + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: population + description: Number of registred residents + type: integer + nullable: true + unit: resident + isAbout: [] + valueReference: [] + - name: geom_point + description: 'Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)' + type: Geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: geom + description: 'Geometry of zensus cell (CRS: ERTS89/EPSG:3035)' + type: Geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - i + - d + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' diff --git a/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_egon_destatis_zensus_apartment_building_population_per_ha.resource.yaml b/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_egon_destatis_zensus_apartment_building_population_per_ha.resource.yaml new file mode 100644 index 000000000..72f8ed1c3 --- /dev/null +++ b/src/egon/data/metadata/dataset_metadata/resources/egon-data/society_egon_destatis_zensus_apartment_building_population_per_ha.resource.yaml @@ -0,0 +1,193 @@ +'@id': null +name: society.egon_destatis_zensus_apartment_building_population_per_ha +title: eGon - Zensus 2011 - Apartments, Buildings, Population per hectar +path: WILL_BE_SET_AT_PUBLICATION +description: Apartments, Buildings, Population from national census in Germany in + 2011. +subject: +- name: '' + '@id': null +publicationDate: '2023-03-13' +embargoPeriod: + start: '' + end: '' + isActive: false +spatial: + location: + address: '' + '@id': null + latitude: '' + longitude: '' + extent: + name: Germany + '@id': null + resolutionValue: '1' + resolutionUnit: ha + boundingBox: + - 0 + - 0 + - 0 + - 0 + crs: '' +temporal: + referenceDate: '' + timeseries: + - start: '' + end: '' + resolutionValue: '' + resolutionUnit: '' + alignment: '' + aggregationType: '' +sources: +- title: Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download + authors: [] + description: Als Download bieten wir Ihnen auf dieser Seite zus\u00e4tzlich zur + Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- + und Familien- sowie Geb\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen + auf Bundes-, L\u00e4nder-, Kreis- und Gemeindeebene vor. Au\u00dferdem sind einzelne + Ergebnisse f\u00fcr Gitterzellen verf\u00fcgbar. + publicationYear: null + path: https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistische \u00c4mter des Bundes und der L\u00e4nder 2014 + copyrightStatement: null +- title: Dokumentation - Zensus 2011 - Methoden und Verfahren + authors: [] + description: Diese Publikation beschreibt ausf\u00fchrlich die Methoden und Verfahren + des registergest\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung + bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von + den Statistischen \u00c4mtern des Bundes und der L\u00e4nder im Juni 2015 ver\u00f6ffentlicht. + publicationYear: null + path: https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile + sourceLicenses: + - name: dl-by-de/2.0 + title: Datenlizenz Deutschland \u2013 Namensnennung \u2013 Version 2.0 + path: www.govdata.de/dl-de/by-2-0 + instruction: Jede Nutzung ist unter den Bedingungen dieser \u201eDatenlizenz Deutschland + - Namensnennung - Version 2.0 zul\u00e4ssig.\nDie bereitgestellten Daten und + Metadaten d\u00fcrfen f\u00fcr die kommerzielle und nicht kommerzielle Nutzung + insbesondere:(1) vervielf\u00e4ltigt, ausgedruckt, pr\u00e4sentiert, ver\u00e4ndert, + bearbeitet sowie an Dritte \u00fcbermittelt werden;\n (2) mit eigenen Daten + und Daten Anderer zusammengef\u00fchrt und zu selbst\u00e4ndigen neuen Datens\u00e4tzen + verbunden werden;\n (3) in interne und externe Gesch\u00e4ftsprozesse, Produkte + und Anwendungen in \u00f6ffentlichen und nicht \u00f6ffentlichen elektronischen + Netzwerken eingebunden werden.\nBei der Nutzung ist sicherzustellen, dass folgende + Angaben als Quellenvermerk enthalten sind:\n(1) Bezeichnung des Bereitstellers + nach dessen Ma\u00dfgabe,\n(2) der Vermerk Datenlizenz Deutschland \u2013 Namensnennung + \u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 + sowie\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende + Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\nVer\u00e4nderungen, + Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk + mit dem Hinweis zu versehen, dass die Daten ge\u00e4ndert wurden. + attribution: \u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft) + copyrightStatement: null +contributors: +- title: nailend + path: null + organization: null + roles: + - '' + date: '2023-03-13' + object: metadata + comment: Create metadata +- title: Open Energy Platform oemetadata conversion to v2 + path: https://github.com/OpenEnergyPlatform + role: + - platform-maintainer + organization: OpenEnergyFamily + date: '2021-09-01' + object: conversion of all metadata to oemetadata version 2.0.4 + comment: The conversion was done by the OpenEnergyFamily team using the OMI software.We + did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries + will be lost. +type: Table +format: PostgreSQL +encoding: UTF-8 +schema: + fields: + - name: grid_id + description: Grid number of source + type: string + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: zensus_population_id + description: Unique identifier + type: integer + nullable: false + unit: none + isAbout: [] + valueReference: [] + - name: building_count + description: 'Number of buildings within zensus cell ' + type: smallint + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: apartment_count + description: Number of appartments within zensus cells + type: smallint + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: population + description: Number of registred residents + type: integer + nullable: true + unit: resident + isAbout: [] + valueReference: [] + - name: geom + description: 'Geometry of zensus cell (CRS: ERTS89/EPSG:3035)' + type: Geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + - name: geom_point + description: 'Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)' + type: Geometry + nullable: true + unit: none + isAbout: [] + valueReference: [] + primaryKey: + - n + - o + - n + - e + foreignKeys: + - fields: + - '' + reference: + resource: '' + fields: + - '' +dialect: + delimiter: '' + decimalSeparator: . +review: + path: '' + badge: '' From 7d2cfe2c14b0e520c621fd24cf37513c02e4c47f Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:22:28 +0100 Subject: [PATCH 29/32] #1177: manually fix metadata to match oemetadata spec --- datapackage.egon-data.json | 5356 +++++++++++++++++ ...undaries.egon_map_zensus_weather_cell.json | 2 +- ...nd.egon_etrago_electricity_households.json | 4 +- 3 files changed, 5360 insertions(+), 2 deletions(-) create mode 100644 datapackage.egon-data.json diff --git a/datapackage.egon-data.json b/datapackage.egon-data.json new file mode 100644 index 000000000..deb116aff --- /dev/null +++ b/datapackage.egon-data.json @@ -0,0 +1,5356 @@ +{ + "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/production/oemetadata/latest/context.json", + "name": "egon-data", + "title": "", + "description": "", + "@id": null, + "resources": [ + { + "@id": null, + "name": "boundaries.egon_map_zensus_climate_zones", + "title": "Map census cells to era5 climate zones", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table of census cells to era5 climate zones", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-08-06", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "1", + "resolutionUnit": "ha", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "ERA5 global reanalysis", + "authors": [], + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "publicationYear": null, + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "sourceLicenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "© Copernicus Climate Change Service (C3S) Climate Data Store", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "zensus_population_id", + "description": "Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)", + "type": "integer", + "nullable": false, + "unit": null, + "isAbout": [], + "valueReference": [] + }, + { + "name": "w_id", + "description": "Weather cell id (reference to: supply.egon_era5_weather_cells)", + "type": "Integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "z", + "e", + "n", + "s", + "u", + "s", + "_", + "p", + "o", + "p", + "u", + "l", + "a", + "t", + "i", + "o", + "n", + "_", + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "boundaries.egon_map_zensus_mvgd_buildings", + "title": "eGon buildings mapping table", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for all used buildings via project internal building_id. Additional info about if it is a synthetic or a openstreetmap building, where it is located (census cell_id, mvgd_id), in which sector (residential, cts) the building is used, if there are heat or electricity profiles.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "zensus_population_id", + "description": "Zensus population id (reference to: society.destatis_zensus_population_per_ha.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "bus_id", + "description": "MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "electricity", + "description": "True if respectives electricity demand profile exists", + "type": "bool", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + }, + { + "name": "heat", + "description": "True if respectives heat demand profile exists", + "type": "bool", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + }, + { + "name": "osm", + "description": "is OSM building if true else syntheticaly created", + "type": "bool", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d", + ",", + " ", + "s", + "e", + "c", + "t", + "o", + "r" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_building_electricity_peak_loads", + "title": "Electricity peak loads for egon buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Electricity peak load table of residential and cts buildings with voltage level for two scenarios ", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "DemandRegio", + "authors": [], + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Auflösung von Energienachfragen", + "publicationYear": null, + "path": "https://doi.org/10.34805/ffe-119-20", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "peak_load_in_w", + "description": "Peak load value", + "type": "real", + "nullable": true, + "unit": "W", + "isAbout": [], + "valueReference": [] + }, + { + "name": "voltage_level", + "description": "Voltage level of building connection after MitnetzStrom Classification", + "type": "int", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d", + ",", + " ", + "s", + "c", + "e", + "n", + "a", + "r", + "i", + "o", + ",", + " ", + "s", + "e", + "c", + "t", + "o", + "r" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_building_heat_peak_loads", + "title": "Heat peak loads for egon buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat peak load table of residential and cts buildings for two scenarios ", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "authors": [], + "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "publicationYear": null, + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten geändert wurden.", + "attribution": "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert)", + "copyrightStatement": null + } + ] + }, + { + "title": "ERA5 global reanalysis", + "authors": [], + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "publicationYear": null, + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "sourceLicenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "© Copernicus Climate Change Service (C3S) Climate Data Store", + "copyrightStatement": null + } + ] + }, + { + "title": "Peta5 0 1 HD res", + "authors": [], + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for residential heat demands for space heating and hot water for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "publicationYear": null, + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "sourceLicenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities", + "copyrightStatement": null + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "authors": [], + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "publicationYear": null, + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "sourceLicenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "sector", + "description": "Sector (cts, residential)", + "type": "characters varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "peak_load_in_w", + "description": "Peak load value", + "type": "real", + "nullable": true, + "unit": "W", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d", + ",", + " ", + "s", + "c", + "e", + "n", + "a", + "r", + "i", + "o", + ",", + " ", + "s", + "e", + "c", + "t", + "o", + "r" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_cts_electricity_demand_building_share", + "title": "Electricity demand profile share of egon cts buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Electricity demand profile share of egon cts buildings for two scenarios with bus_id", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "DemandRegio", + "authors": [], + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Auflösung von Energienachfragen", + "publicationYear": null, + "path": "https://doi.org/10.34805/ffe-119-20", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035) (reference: demand.egon_etrago_electricity_cts.scn_name)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "bus_id", + "description": "MVGD bus_id (reference to: demand.egon_etrago_electricity_cts.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "profile_share", + "description": "Profile share of MVGD elctricity profile", + "type": "real", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d", + ",", + " ", + "s", + "c", + "e", + "n", + "a", + "r", + "i", + "o" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_cts_heat_demand_building_share", + "title": "Heat demand profile share of egon cts buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Heat demand profile share of egon cts buildings for two scenarios with bus_id", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "authors": [], + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "publicationYear": null, + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "sourceLicenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities", + "copyrightStatement": null + } + ] + }, + { + "title": "Peta5 0 1 HD res", + "authors": [], + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for residential heat demands for space heating and hot water for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "publicationYear": null, + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "sourceLicenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id, boundaries.egon_map_zensus_mvgd_building)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scenario", + "description": "Scenario (eGon100RE, eGon2035)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "bus_id", + "description": "MVGD bus_id (reference to: demand.egon_etrago_heat_cts.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "profile_share", + "description": "Profile share of MVGD heat profile ", + "type": "real", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d", + ",", + " ", + "s", + "c", + "e", + "n", + "a", + "r", + "i", + "o" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_daily_heat_demand_per_climate_zone", + "title": "eGon daily heat demand share per climate zone", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Share of daily heat demands per climate zone", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "climate", + "resolutionUnit": "zones", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "ERA5 global reanalysis", + "authors": [], + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "publicationYear": null, + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "sourceLicenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "© Copernicus Climate Change Service (C3S) Climate Data Store", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "climate_zone", + "description": "Name of the corresponding climate zone", + "type": "str", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "daily_demand_share", + "description": "Share of annual heat demand for each day", + "type": "double precision", + "nullable": true, + "unit": "per unit", + "isAbout": [], + "valueReference": [] + }, + { + "name": "day_of_year", + "description": "Number of day in the year", + "type": "integer", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "temperature_class", + "description": "Temperature class (considering the mean outside temeperature during the day)", + "type": "integer", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "c", + "l", + "i", + "m", + "a", + "t", + "e", + "_", + "z", + "o", + "n", + "e" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_etrago_electricity_cts", + "title": "MV cts electricity demand profiles", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated electricity demand profile of cts sector at mvgd bus level for two scenarios. Values are given in MWh.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "MV", + "resolutionUnit": "Grid", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "2016-01-01", + "end": "2016-12-31", + "resolutionValue": "1", + "resolutionUnit": "h", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "DemandRegio", + "authors": [], + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Auflösung von Energienachfragen", + "publicationYear": null, + "path": "https://doi.org/10.34805/ffe-119-20", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scn_name", + "description": "Scenario name (reference: demand.egon_cts_electricity_demand_building_share.scenario)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "p_set", + "description": "Cts electricity demand load timeseries for one year (8760 hours) in MWh", + "type": "array of real", + "nullable": true, + "unit": "MWh", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d", + ",", + " ", + "s", + "c", + "n", + "_", + "n", + "a", + "m", + "e" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_etrago_electricity_households", + "title": "MV residential electricity demand profiles", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated electricity demand profile of residential sector at mvgd bus level for two scenarios. Values are given in MWh.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "MV", + "resolutionUnit": "Grid", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "2016-01-01", + "end": "2016-12-31", + "resolutionValue": "1", + "resolutionUnit": "h", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "DemandRegio", + "authors": [], + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Auflösung von Energienachfragen", + "publicationYear": null, + "path": "https://doi.org/10.34805/ffe-119-20", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_electricity_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id))", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scn_name", + "description": "Scenario name", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "p_set", + "description": "Residential electricity demand load timeseries for one year (8760 hours) in MWh", + "type": "array of double precission", + "nullable": true, + "unit": "MWh", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d", + ",", + " ", + "s", + "c", + "n", + "_", + "n", + "a", + "m", + "e" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_etrago_heat_cts", + "title": "MV cts heat demand profiles", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Aggregated heat demand profile of cts sector at mvgd bus level for two scenarios. Values are given in MWh.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "MV", + "resolutionUnit": "Grid", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2023-03-13", + "timeseries": [ + { + "start": "2016-01-01", + "end": "2016-12-31", + "resolutionValue": "1", + "resolutionUnit": "h", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "authors": [], + "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "publicationYear": null, + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten geändert wurden.", + "attribution": "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert)", + "copyrightStatement": null + } + ] + }, + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + }, + { + "title": "ERA5 global reanalysis", + "authors": [], + "description": "ERA5 is the fifth generation ECMWF reanalysis for the global climate and weather for the past 4 to 7 decades. Currently data is available from 1950, split into Climate Data Store entries for 1950-1978 (preliminary back extension) and from 1979 onwards (final release plus timely updates, this page). ERA5 replaces the ERA-Interim reanalysis. See the online ERA5 documentation (https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency) for more information.", + "publicationYear": null, + "path": "https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Dataupdatefrequency", + "sourceLicenses": [ + { + "name": "Licence to use Copernicus Products", + "title": "Licence to use Copernicus Products", + "path": "https://cds.climate.copernicus.eu/api/v2/terms/static/licence-to-use-copernicus-products.pdf", + "instruction": "This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing", + "attribution": "© Copernicus Climate Change Service (C3S) Climate Data Store", + "copyrightStatement": null + } + ] + }, + { + "title": "Peta5 0 1 HD ser", + "authors": [], + "description": "modelled Heat Demand distribution (in GJ per hectare grid cell) for service-sector heat demands for space heating, hot water and process heat for the year 2015 using HRE4 data and the combined top-down bottom-up approach of HRE4", + "publicationYear": null, + "path": "https://s-eenergies-open-data-euf.hub.arcgis.com/search", + "sourceLicenses": [ + { + "name": "Creative Commons Attribution 4.0 International", + "title": "CC BY 4.0", + "path": "https://creativecommons.org/licenses/by/4.0/", + "instruction": "You are free: To Share, To Adapt; As long as you: Attribute!", + "attribution": "\\u00a9 Flensburg, Halmstad and Aalborg universities", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "bus_id", + "description": "MV bus_id (reference: demand.egon_cts_heat_demand_building_share.bus_id, grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "scn_name", + "description": "Scenario name (reference: demand.egon_cts_heat_demand_building_share.scenario)", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "p_set", + "description": "Cts heat demand load timeseries for one year (8760 hours) in MWh", + "type": "array of real", + "nullable": true, + "unit": "MWh", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d", + ",", + " ", + "s", + "c", + "n", + "_", + "n", + "a", + "m", + "e" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_heat_idp_pool", + "title": "eGon pool of residential intra-day heat demand profiles", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Pool of individual, normalized intra-day heat demand profiles of residential buildings. Timeseries only represent one day but are used to be merged for a whole year ", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "none", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "2011-01-01 00:00", + "end": "2011-12-31 23:00", + "resolutionValue": "1", + "resolutionUnit": "h", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "house", + "description": "Type of residential house, either SFH (single family house) or MFH (multi family house)", + "type": "str", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "idp", + "description": "Normalized intra-day-profile", + "type": "array of double precision", + "nullable": false, + "unit": "per unit", + "isAbout": [], + "valueReference": [] + }, + { + "name": "index", + "description": "Unique identifier", + "type": "integer", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "temperature_class", + "description": "Temperature class (considering the mean outside temeperature during the day)", + "type": "integer", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "n", + "d", + "e", + "x" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_heat_timeseries_selected_profiles", + "title": "eGon selected intra-day profiles per building", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "List of selected intra-day-profiles for each residential building in Germany", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "none", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at ''referenceDate''", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Index of the corresponding building", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "selected_idp_profiles", + "description": "List of selected profiles for each day in the year", + "type": "array of integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "zensus_population_id", + "description": "Index of corresponding census cell", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "i", + "l", + "d", + "i", + "n", + "g", + "_", + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "" + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_household_electricity_profile_in_census_cell", + "title": "eGon household electricity profiles in census cells", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for residential electricity profiles to census cell including scaling factors for two scenarios (eGon2035, eGon100RE).", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "100", + "resolutionUnit": "m x 100 m", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "authors": [], + "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "publicationYear": null, + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten geändert wurden.", + "attribution": "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert)", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + }, + { + "title": "DemandRegio", + "authors": [], + "description": "Harmonisierung und Entwicklung von Verfahren zur regionalen und zeitlichen Auflösung von Energienachfragen", + "publicationYear": null, + "path": "https://doi.org/10.34805/ffe-119-20", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "grid_id", + "description": "Descriptive ID of census cell given by Statistisches Bundesamt (Destatis)", + "type": "character varying", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "cell_profile_ids", + "description": "Household profile ids", + "type": "array of integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "nuts3", + "description": "NUTS-3 ID of the census cell", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "nuts1", + "description": "NUTS-1 ID of the census cell", + "type": "character varying", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "factor_2035", + "description": "Scaling factor for all profiles in the respective census cell for scenario eGon2035", + "type": "double precision", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "factor_2050", + "description": "Scaling factor for all profiles in the respective census cell for scenario eGon100RE", + "type": "double precision", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "c", + "e", + "l", + "l", + "_", + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.egon_household_electricity_profile_of_buildings", + "title": "eGon household electricity profiles of buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table for residential electricity profiles to buildings via project internal building_id and corresponding census cell_id. ", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_residential.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "profile_id", + "description": "ID of iee household profiles (reference to: iee_household_load_profiles.type)", + "type": "character varying", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "demand.iee_household_load_profiles", + "title": "eGon pool of electricity household load profiles", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "100.000 annual profiles in hourly resolution of electricity demand of private households for different household types (singles, couples, other) with varying number of elderly and children. The profiles were created using a bottom-up load profile generator by Fraunhofer IEE developed in the Bachelors thesis \"Auswirkungen verschiedener Haushaltslastprofile auf PV-Batterie-Systeme\" by Jonas Haack, Fachhochschule Flensburg, December 2012. The columns are named as follows: \"a\", e.g. P2a0000 is the first profile of a couples household with 2 children. See publication below for the list of prefixes. Values are given in Wh.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-06-10", + "timeseries": [ + { + "start": "2016-01-01", + "end": "2016-12-31", + "resolutionValue": "1", + "resolutionUnit": "h", + "alignment": "left", + "aggregationType": "sum" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "type", + "description": "Profile id", + "type": "character(8)", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "load_in_wh", + "description": "Residential demand load timeseries for one year (8760 hours) in Wh", + "type": "array of real", + "nullable": true, + "unit": "Wh", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "grid.egon_mv_grid_district", + "title": "eGon MV grid districts", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Mapping table of bus_id to polygons shapes for MV grid districts.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "MV", + "resolutionUnit": "Grids", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "Verwaltungsgebiete 1:250 000 (Ebenen)", + "authors": [], + "description": "Der Datenbestand umfasst sämtliche Verwaltungseinheiten der hierarchischen Verwaltungsebenen vom Staat bis zu den Gemeinden mit ihren Grenzen, statistischen Schlüsselzahlen, Namen der Verwaltungseinheit sowie die spezifische Bezeichnung der Verwaltungsebene des jeweiligen Landes.", + "publicationYear": null, + "path": "https://daten.gdz.bkg.bund.de/produkte/vg/vg250_ebenen_0101/2020/vg250_01-01.geo84.shape.ebenen.zip", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland – Namensnennung – Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland - Namensnennung - Version 2.0 zulässig.Die bereitgestellten Daten und Metadaten dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte übermittelt werden; (2) mit eigenen Daten und Daten Anderer zusammengeführt und zu selbständigen neuen Datensätzen verbunden werden; (3) in interne und externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht öffentlichen elektronischen Netzwerken eingebunden werden.Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:(1) Bezeichnung des Bereitstellers nach dessen Maßgabe,(2) der Vermerk Datenlizenz Deutschland – Namensnennung – Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.Veränderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten geändert wurden.", + "attribution": "© Bundesamt für Kartographie und Geodäsie 2020 (Daten verändert)", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "building_id", + "description": "Building id (reference to: openstreetmap.osm_buildings_filtered.id, openstreetmap.osm_buildings_synthetic.id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "bus_id", + "description": "MVGD bus id (reference to: grid.egon_hvmv_substation.bus_id, grid.egon_mv_grid_district.bus_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom", + "description": "Polygon of mv grid district (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "nullable": true, + "unit": "None", + "isAbout": [], + "valueReference": [] + }, + { + "name": "area", + "description": "Area covered by mvgd", + "type": "double precision", + "nullable": true, + "unit": "m^2", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "b", + "u", + "s", + "_", + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "openstreetmap.osm_buildings_filtered", + "title": "eGon filtered buildings from openstreetmap", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Tag filtered list of buildings from OpenStreetMap which might have electricity or heat demand - (c) OpenStreetMap contributors", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-01-01", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Created metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "osm_id", + "description": "Openstreetmap identifier", + "type": "bigint", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "amenitiy", + "description": "Type of amenitiy if given", + "type": "text", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "building", + "description": "Type of building", + "type": "text", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "name", + "description": "Name of the building if given", + "type": "text", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_building", + "description": "Polygon of the building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "area", + "description": "Surface area of building", + "type": "double precision", + "nullable": true, + "unit": "m^2", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_point", + "description": "Centroid of the building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "tags", + "description": "Opensteetmap tags assigned to the building", + "type": "hstore", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "id", + "description": "Unique identifyer and project internal building id (reference to: demand.egon_hp_capacity_buildings.building_id, demand.egon_cts_heat_demand_building_share.building_id, demand.egon_cts_electricity_demand_building_share.building_id, demand.egon_household_electricity_profile_of_buildings.building_id, demand.egon_building_heat_peak_loads.building_id, demand.egon_building_electricity_peak_loads.building_id, boundaries.egon_map_zensus_mvgd_buildings.building_id)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "openstreetmap.osm_buildings_synthetic", + "title": "eGon synthetic residential buildings", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": " List of generated synthetic buildings ", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-09-14", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "Buildings", + "resolutionUnit": "", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "2022-09-14", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Data bundle for egon-data: A transparent and reproducible data processing pipeline for energy system modeling", + "authors": [], + "description": "egon-data provides a transparent and reproducible open data based data processing pipeline for generating data models suitable for energy system modeling. The data is customized for the requirements of the research project eGon. The research project aims to develop tools for an open and cross-sectoral planning of transmission and distribution grids. For further information please visit the eGon project website or its Github repository. egon-data retrieves and processes data from several different external input sources. As not all data dependencies can be downloaded automatically from external sources we provide a data bundle to be downloaded by egon-data.", + "publicationYear": null, + "path": "https://doi.org/10.5281/zenodo.6630616", + "sourceLicenses": [ + { + "name": "CC-BY-4.0", + "title": "Creative Commons Attribution 4.0 International", + "path": "https://creativecommons.org/licenses/by/4.0/legalcode", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute.", + "attribution": "© eGon-data developers, © Leibniz Institute for Applied Geophysics, © Fraunhofer IEE, © BGR Hannover, © Danielle Schmidt, © Übertragungsnetzbetreiber, © DIW Berlin, © Reiner Lemoine Institut, © Statistisches Bundesamt, Wiesbaden 2008, © Statistische Ämter des Bundes und der Länder 2021", + "copyrightStatement": null + } + ] + }, + { + "title": "eGon-data", + "authors": [], + "description": "Workflow to download, process and generate data sets suitable for the further research conducted in the project eGon (https://ego-n.org/)", + "publicationYear": null, + "path": "https://github.com/openego/eGon-data", + "sourceLicenses": [ + { + "name": "AGPL-3.0 License", + "title": "GNU Affero General Public License v3.0", + "path": "https://www.gnu.org/licenses/agpl-3.0.de.html", + "instruction": "Permissions of this strongest copyleft license are conditioned on making available complete source code of licensed works and modifications, which include larger works using a licensed work, under the same license. Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. When a modified version is used to provide a service over a network, the complete source code of the modified version must be made available.", + "attribution": "© eGon-data developers", + "copyrightStatement": null + } + ] + }, + { + "title": "OpenStreetMap Data Extracts (Geofabrik)", + "authors": [], + "description": "Full data extract of OpenStreetMap data for defined spatial extent at 2022-01-01", + "publicationYear": null, + "path": "https://download.geofabrik.de/europe/germany-210101.osm.pbf", + "sourceLicenses": [ + { + "name": "ODbL-1.0", + "title": "Open Data Commons Open Database License 1.0", + "path": "https://opendatacommons.org/licenses/odbl/1.0/", + "instruction": "You are free: To Share, To Create, To Adapt; As long as you: Attribute, Share-Alike, Keep open!", + "attribution": "© OpenStreetMap contributors", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "ClaraBuettner", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-12", + "object": "metadata", + "comment": "Updated metadata" + }, + { + "title": "nesnoj", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2022-09-14", + "object": "metadata", + "comment": "Update metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifyer and project internal building id (reference to: openstreetmap.osm_buildings_residential.id, demand.egon_household_electricity_profile_of_buildings.building_id)", + "type": "character varying", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "area", + "description": "Surface area of building", + "type": "real", + "nullable": true, + "unit": "m^2", + "isAbout": [], + "valueReference": [] + }, + { + "name": "building", + "description": "Type of building (residential or cts)", + "type": "character varying(11)", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "cell_id", + "description": "Census cell id (reference to: society.destatis_zensus_population_per_ha_inside_germany.id)", + "type": "character varying", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_building", + "description": "Polygon of building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_point", + "description": "Centroid of building (CRS: ERTS89/EPSG:3035)", + "type": "geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "n_amenities_inside", + "description": "Number of amenities inside the building (always zero if not cts building)", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "society.destatis_zensus_population_per_ha", + "title": "DESTATIS - Zensus 2011 - Population per hectar", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "National census in Germany in 2011.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2022-08-06", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "1", + "resolutionUnit": "ha", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "id", + "description": "Unique identifier", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "grid_id", + "description": "Grid number of source", + "type": "string", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "population", + "description": "Number of registred residents", + "type": "integer", + "nullable": true, + "unit": "resident", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_point", + "description": "Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom", + "description": "Geometry of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "i", + "d" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + }, + { + "@id": null, + "name": "society.egon_destatis_zensus_apartment_building_population_per_ha", + "title": "eGon - Zensus 2011 - Apartments, Buildings, Population per hectar", + "path": "WILL_BE_SET_AT_PUBLICATION", + "description": "Apartments, Buildings, Population from national census in Germany in 2011.", + "subject": [ + { + "name": "", + "@id": null + } + ], + "publicationDate": "2023-03-13", + "embargoPeriod": { + "start": "", + "end": "", + "isActive": false + }, + "spatial": { + "location": { + "address": "", + "@id": null, + "latitude": "", + "longitude": "" + }, + "extent": { + "name": "Germany", + "@id": null, + "resolutionValue": "1", + "resolutionUnit": "ha", + "boundingBox": [ + 0, + 0, + 0, + 0 + ], + "crs": "" + } + }, + "temporal": { + "referenceDate": "", + "timeseries": [ + { + "start": "", + "end": "", + "resolutionValue": "", + "resolutionUnit": "", + "alignment": "", + "aggregationType": "" + } + ] + }, + "sources": [ + { + "title": "Statistisches Bundesamt (Destatis) - Ergebnisse des Zensus 2011 zum Download", + "authors": [], + "description": "Als Download bieten wir Ihnen auf dieser Seite zus\\u00e4tzlich zur Zensusdatenbank CSV- und teilweise Excel-Tabellen mit umfassenden Personen-, Haushalts- und Familien- sowie Geb\\u00e4ude- und Wohnungsmerkmalen. Die Ergebnisse liegen auf Bundes-, L\\u00e4nder-, Kreis- und Gemeindeebene vor. Au\\u00dferdem sind einzelne Ergebnisse f\\u00fcr Gitterzellen verf\\u00fcgbar.", + "publicationYear": null, + "path": "https://www.zensus2011.de/DE/Home/Aktuelles/DemografischeGrunddaten.html", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014", + "copyrightStatement": null + } + ] + }, + { + "title": "Dokumentation - Zensus 2011 - Methoden und Verfahren", + "authors": [], + "description": "Diese Publikation beschreibt ausf\\u00fchrlich die Methoden und Verfahren des registergest\\u00fctzten Zensus 2011; von der Datengewinnung und -aufbereitung bis hin zur Ergebniserstellung und Geheimhaltung. Der vorliegende Band wurde von den Statistischen \\u00c4mtern des Bundes und der L\\u00e4nder im Juni 2015 ver\\u00f6ffentlicht.", + "publicationYear": null, + "path": "https://www.destatis.de/DE/Publikationen/Thematisch/Bevoelkerung/Zensus/ZensusBuLaMethodenVerfahren5121105119004.pdf?__blob=publicationFile", + "sourceLicenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (im Auftrag der Herausgebergemeinschaft)", + "copyrightStatement": null + } + ] + } + ], + "contributors": [ + { + "title": "nailend", + "path": null, + "organization": null, + "roles": [ + "" + ], + "date": "2023-03-13", + "object": "metadata", + "comment": "Create metadata" + }, + { + "title": "Open Energy Platform oemetadata conversion to v2", + "path": "https://github.com/OpenEnergyPlatform", + "role": [ + "platform-maintainer" + ], + "organization": "OpenEnergyFamily", + "date": "2021-09-01", + "object": "conversion of all metadata to oemetadata version 2.0.4", + "comment": "The conversion was done by the OpenEnergyFamily team using the OMI software.We did our best to mitigate data loss. Most unexpected or incorrect metadata propertyentries will be lost." + } + ], + "type": "Table", + "format": "PostgreSQL", + "encoding": "UTF-8", + "schema": { + "fields": [ + { + "name": "grid_id", + "description": "Grid number of source", + "type": "string", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "zensus_population_id", + "description": "Unique identifier", + "type": "integer", + "nullable": false, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "building_count", + "description": "Number of buildings within zensus cell ", + "type": "smallint", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "apartment_count", + "description": "Number of appartments within zensus cells", + "type": "smallint", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "population", + "description": "Number of registred residents", + "type": "integer", + "nullable": true, + "unit": "resident", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom", + "description": "Geometry of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + }, + { + "name": "geom_point", + "description": "Geometry centroid of zensus cell (CRS: ERTS89/EPSG:3035)", + "type": "Geometry", + "nullable": true, + "unit": "none", + "isAbout": [], + "valueReference": [] + } + ], + "primaryKey": [ + "n", + "o", + "n", + "e" + ], + "foreignKeys": [ + { + "fields": [ + "" + ], + "reference": { + "resource": "", + "fields": [ + "" + ] + } + } + ] + }, + "dialect": { + "delimiter": "", + "decimalSeparator": "." + }, + "review": { + "path": "", + "badge": "" + }, + "context": { + "grantNo": "03EI1002", + "fundingAgency": "Bundesministerium f\\u00fcr Wirtschaft und Energie", + "sourceCode": "https://github.com/openego/eGon-data", + "documentation": "https://egon-data.readthedocs.io/en/latest/", + "publisherLogo": "https://ego-n.org/images/eGon_logo_noborder_transbg.svg", + "homepage": "https://ego-n.org/", + "contact": "https://ego-n.org/partners/", + "title": "", + "fundingAgencyLogo": "https://www.innovation-beratung-foerderung.de/INNO/Redaktion/DE/Bilder/Titelbilder/titel_foerderlogo_bmwi.jpg?__blob=normal&v=3", + "publisher": "" + }, + "licenses": [ + { + "name": "dl-by-de/2.0", + "title": "Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0", + "path": "www.govdata.de/dl-de/by-2-0", + "instruction": "Jede Nutzung ist unter den Bedingungen dieser \\u201eDatenlizenz Deutschland - Namensnennung - Version 2.0 zul\\u00e4ssig.\\nDie bereitgestellten Daten und Metadaten d\\u00fcrfen f\\u00fcr die kommerzielle und nicht kommerzielle Nutzung insbesondere:(1) vervielf\\u00e4ltigt, ausgedruckt, pr\\u00e4sentiert, ver\\u00e4ndert, bearbeitet sowie an Dritte \\u00fcbermittelt werden;\\n (2) mit eigenen Daten und Daten Anderer zusammengef\\u00fchrt und zu selbst\\u00e4ndigen neuen Datens\\u00e4tzen verbunden werden;\\n (3) in interne und externe Gesch\\u00e4ftsprozesse, Produkte und Anwendungen in \\u00f6ffentlichen und nicht \\u00f6ffentlichen elektronischen Netzwerken eingebunden werden.\\nBei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind:\\n(1) Bezeichnung des Bereitstellers nach dessen Ma\\u00dfgabe,\\n(2) der Vermerk Datenlizenz Deutschland \\u2013 Namensnennung \\u2013 Version 2.0 oder dl-de/by-2-0 mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie\\n(3) einen Verweis auf den Datensatz (URI).Dies gilt nur soweit die datenhaltende Stelle die Angaben(1) bis (3) zum Quellenvermerk bereitstellt.\\nVer\\u00e4nderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten ge\\u00e4ndert wurden.", + "attribution": "\\u00a9 Statistische \\u00c4mter des Bundes und der L\\u00e4nder 2014; \\u00a9 Statistisches Bundesamt, Wiesbaden 2015 (Daten ver\\u00e4ndert)", + "copyrightStatement": null + } + ], + "languages": [ + "undefined" + ], + "topics": [ + "model_draft" + ], + "keywords": [ + "" + ] + } + ], + "metaMetadata": { + "metadataVersion": "OEMetadata-2.0.4", + "metadataLicense": { + "name": "CC0-1.0", + "title": "Creative Commons Zero v1.0 Universal", + "path": "https://creativecommons.org/publicdomain/zero/1.0" + } + } +} diff --git a/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json b/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json index 5431be2ad..234e45c8e 100644 --- a/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json +++ b/src/egon/data/metadata/results/boundaries.egon_map_zensus_weather_cell.json @@ -3,7 +3,7 @@ "title": "Map census cells to era5 climate zones", "id": "WILL_BE_SET_AT_PUBLICATION", "description": "Mapping table of census cells to era5 climate zones", - "language": "en-US", + "language": ["en-US"], "keywords": [ "" ], diff --git a/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json b/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json index eb74fbc29..a114fe85e 100644 --- a/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json +++ b/src/egon/data/metadata/results/demand.egon_etrago_electricity_households.json @@ -3,7 +3,9 @@ "title": "MV residential electricity demand profiles", "id": "WILL_BE_SET_AT_PUBLICATION", "description": "Aggregated electricity demand profile of residential sector at mvgd bus level for two scenarios. Values are given in MWh.", - "language": "en-US", + "language": [ + "en-US" + ], "keywords": [ "" ], From 0f06482c6955553ae356a531c82ad38acb598912 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:26:55 +0100 Subject: [PATCH 30/32] #1177: add script to convert json to yaml metadata in a structure which works for the new metadata module and new omi functionality --- .../script/migrate_json_to_split_yaml.py | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 src/egon/data/metadata/script/migrate_json_to_split_yaml.py diff --git a/src/egon/data/metadata/script/migrate_json_to_split_yaml.py b/src/egon/data/metadata/script/migrate_json_to_split_yaml.py new file mode 100644 index 000000000..156014a91 --- /dev/null +++ b/src/egon/data/metadata/script/migrate_json_to_split_yaml.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +from typing import Any, Dict, List, Tuple +import argparse +import json + +import yaml + +# ---------- YAML helpers ---------- + + +def _dump_yaml(path: Path, obj: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + yaml.safe_dump(obj, f, sort_keys=False, allow_unicode=True) + + +# ---------- list/dict helpers for template inference ---------- + +_LIST_TEMPLATE_KEYS = {"keywords", "topics", "languages"} +_DICT_TEMPLATE_KEYS = {"context"} +_LIST_EQUALITY_KEYS = { + "licenses" +} # consider as common only if identical across all + + +def _deep_intersection_dict(dicts: List[Dict[str, Any]]) -> Dict[str, Any]: + if not dicts: + return {} + keys_in_all = set(dicts[0].keys()) + for d in dicts[1:]: + keys_in_all &= set(d.keys()) + + out: Dict[str, Any] = {} + for k in keys_in_all: + vals = [d[k] for d in dicts] + if not all(type(v) is type(vals[0]) for v in vals): # noqa: E721 + continue + v0 = vals[0] + if isinstance(v0, dict): + # type: ignore[arg-type] + sub = _deep_intersection_dict([v for v in vals]) + if sub: + out[k] = sub + elif isinstance(v0, list): + if all(v == v0 for v in vals[1:]): + out[k] = deepcopy(v0) + else: + if all(v == v0 for v in vals[1:]): + out[k] = v0 + return out + + +def _all_identical(values: List[Any]) -> Tuple[bool, Any]: + if not values: + return False, None + v0 = values[0] + return (all(v == v0 for v in values[1:])), v0 + + +# ---------- schema fix ---------- + + +def _ensure_nullable_in_fields(resource: Dict[str, Any]) -> None: + schema = resource.get("schema") + if not isinstance(schema, dict): + return + fields = schema.get("fields") + if not isinstance(fields, list): + return + for fld in fields: + if isinstance(fld, dict) and "nullable" not in fld: + fld["nullable"] = False + + +# ---------- names ---------- + + +def _slug(s: str) -> str: + return "".join(c if c.isalnum() or c in ("-", "_") else "_" for c in s) + + +def _resource_filename(name: str) -> str: + return f"{_slug(name)}.resource.yaml" + + +# ---------- core migration ---------- + + +def _strip_resource_by_template( + res: Dict[str, Any], tmpl: Dict[str, Any] +) -> Dict[str, Any]: + out = deepcopy(res) + + # strip list-keys if identical + for k in list(_LIST_TEMPLATE_KEYS | _LIST_EQUALITY_KEYS): + if k in tmpl and k in out and out[k] == tmpl[k]: + out.pop(k, None) + + # strip dict-keys by matching deep subset of template keys + for k in _DICT_TEMPLATE_KEYS: + if ( + k not in tmpl + or k not in out + or not isinstance(out[k], dict) + or not isinstance(tmpl[k], dict) + ): + continue + pruned = {} + for kk, vv in out[k].items(): + if kk in tmpl[k] and tmpl[k][kk] == vv: + continue + pruned[kk] = vv + if pruned: + out[k] = pruned + else: + out.pop(k, None) + + return out + + +def _compute_template(resources: List[Dict[str, Any]]) -> Dict[str, Any]: + template: Dict[str, Any] = {} + if not resources: + return template + + # dict keys (deep intersection) + for k in _DICT_TEMPLATE_KEYS: + dicts = [r.get(k, {}) for r in resources if isinstance(r.get(k), dict)] + if len(dicts) == len(resources) and dicts: + inter = _deep_intersection_dict(dicts) + if inter: + template[k] = inter + + # list equality keys (identical lists across all) + for k in _LIST_EQUALITY_KEYS: + lists = [r.get(k) for r in resources] + if all(isinstance(v, list) for v in lists): + same, val = _all_identical(lists) + if same and val: + template[k] = deepcopy(val) + + # simple list keys that must be identical across resources + for k in _LIST_TEMPLATE_KEYS: + vals = [r.get(k) for r in resources] + if all(isinstance(v, list) for v in vals): + same, val = _all_identical(vals) + if same and val: + template[k] = deepcopy(val) + + return template + + +def migrate_monolithic( + data: Dict[str, Any], out_dir: Path, dataset_id: str +) -> None: + resources = data.get("resources", []) + if not isinstance(resources, list): + resources = [] + + # Build dataset block (skip @context/metaMetadata/resources) + dataset: Dict[str, Any] = { + k: v + for k, v in data.items() + if k not in {"@context", "metaMetadata", "resources"} + } + version = dataset.get("version", "OEMetadata-2.0.4") + + # Fix fields + for r in resources: + if isinstance(r, dict): + _ensure_nullable_in_fields(r) + + # Compute template & strip + template = _compute_template(resources) + stripped_resources = [ + _strip_resource_by_template(r, template) for r in resources + ] + + # Write YAMLs + ds_path = out_dir / "datasets" / f"{dataset_id}.dataset.yaml" + _dump_yaml(ds_path, {"version": version, "dataset": dataset}) + + if template: + tp_path = out_dir / "datasets" / f"{dataset_id}.template.yaml" + _dump_yaml(tp_path, template) + + res_dir = out_dir / "resources" / dataset_id + for r in stripped_resources: + name = str(r.get("name") or "resource") + _dump_yaml(res_dir / _resource_filename(name), r) + + print( + f"[OK] {dataset_id}: wrote dataset + {len(stripped_resources)}" + "resources" + ) + + +def migrate_resource_only( + data: Dict[str, Any], out_dir: Path, dataset_id: str +) -> None: + """ + For JSONs that contain a single resource (no top-level 'resources'). + We just write one resource YAML. Dataset/template can be added later. + """ + name = str(data.get("name") or "resource") + _ensure_nullable_in_fields(data) + res_dir = out_dir / "resources" / dataset_id + _dump_yaml(res_dir / _resource_filename(name), data) + print(f"[OK] {dataset_id}: wrote single resource '{name}'") + + +# ---------- dataset_id inference ---------- + + +def pick_dataset_id( + mode: str, + data: Dict[str, Any], + file_path: Path, + fixed_id: str | None, +) -> str: + if mode == "fixed": + if not fixed_id: + raise ValueError( + "--dataset-id is required when --dataset-id-mode=fixed" + ) + return fixed_id + if mode == "name": + n = data.get("name") + if isinstance(n, str) and n.strip(): + return _slug(n) + return _slug(file_path.stem) + if mode == "filename": + return _slug(file_path.stem) + if mode == "parent": + return _slug(file_path.parent.name) + raise ValueError(f"Unknown dataset-id-mode: {mode}") + + +# ---------- CLI ---------- + + +def main() -> None: + ap = argparse.ArgumentParser( + description="Migrate OEMetadata JSON → split YAML" + "(handles many files)." + ) + ap.add_argument( + "--input", + required=True, + type=Path, + help="Path to a JSON file or a directory to scan recursively.", + ) + ap.add_argument( + "--out-dir", + required=True, + type=Path, + help="Root of the split-YAML metadata tree to write.", + ) + ap.add_argument( + "--dataset-id-mode", + choices=["name", "filename", "parent", "fixed"], + default="name", + help="How to determine dataset_id per input file (default: name).", + ) + ap.add_argument( + "--dataset-id", + default=None, + help="Required if --dataset-id-mode=fixed; ignored otherwise.", + ) + args = ap.parse_args() + + inputs: List[Path] + if args.input.is_dir(): + inputs = sorted(args.input.rglob("*.json")) + if not inputs: + print(f"[WARN] No JSON files found under {args.input}") + return + else: + inputs = [args.input] + + out_dir = args.out_dir + + for j in inputs: + try: + data = json.loads(j.read_text(encoding="utf-8")) + except Exception as e: + print(f"[SKIP] {j}: cannot read/parse JSON ({e})") + continue + + try: + dsid = pick_dataset_id( + args.dataset_id_mode, data, j, args.dataset_id + ) + except Exception as e: + print(f"[SKIP] {j}: {e}") + continue + + if isinstance(data.get("resources"), list): + migrate_monolithic(data, out_dir, dsid) + else: + # treat as a single resource JSON (best-effort) + migrate_resource_only(data, out_dir, dsid) + + +if __name__ == "__main__": + main() From 186a0a00289ea0896b3b8f48b3a3bb594472eaf4 Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:27:46 +0100 Subject: [PATCH 31/32] 1177: add script to use omis metadata version conversion and apply it to the exsisting egon-data metadata JSON files --- .../data/metadata/script/convert_oem_to_v2.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/egon/data/metadata/script/convert_oem_to_v2.py diff --git a/src/egon/data/metadata/script/convert_oem_to_v2.py b/src/egon/data/metadata/script/convert_oem_to_v2.py new file mode 100644 index 000000000..35499ee7b --- /dev/null +++ b/src/egon/data/metadata/script/convert_oem_to_v2.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +""" +Script to convert metadata files from OEMetadata-1.x to OEMetadata-2.0 format. + +Requires at least oemetadata in version OEP-1.5.2. +""" + +from pathlib import Path +import json +import logging + +from omi.conversion import convert_metadata + +logger = logging.getLogger(__name__) + +# Metadata input/output directories +SOURCE_DIR = Path(__file__).parent.parent / "results" +TARGET_DIR = Path(__file__).parent.parent / "results" / "converted" + +# OMI target version for metadata version conversion +TARGET_VERSION = "OEMetadata-2.0" + + +def run_conversion(): + source_dir = SOURCE_DIR + target_dir = TARGET_DIR + target_version = TARGET_VERSION + + if not target_version: + logger.error("TARGET_VERSION is not defined in settings.py") + return + + if not source_dir.exists(): + logger.error(f"Source directory does not exist: {source_dir}") + return + + target_dir.mkdir(parents=True, exist_ok=True) + logger.info("Starting metadata conversion") + logger.debug(f"Source: {source_dir}") + logger.debug(f"Target: {target_dir}") + logger.debug(f"Target version: {target_version}") + + for json_file in sorted(source_dir.glob("*.json")): + if not json_file.is_file(): + continue + + logger.info(f"Converting {json_file.name}") + try: + with open(json_file, "r", encoding="utf-8") as f: + metadata = json.load(f) + + converted = convert_metadata(metadata, target_version) + + output_path = target_dir / json_file.name + with open(output_path, "w", encoding="utf-8") as f: + json.dump(converted, f, ensure_ascii=False, indent=2) + + logger.info(f"Saved converted metadata: {output_path}") + + except Exception: + logger.exception(f"Error converting file: {json_file.name}") + + logger.info("Metadata conversion completed.") + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + run_conversion() From b1bf41a3b6e867bf55f330cef210153a266a9a3c Mon Sep 17 00:00:00 2001 From: jh-RLI Date: Wed, 26 Nov 2025 18:30:14 +0100 Subject: [PATCH 32/32] #1177: Change map_zensus_vg250 import to "append" - remove string wrappers required for older versions of omi (used to parse metadata) --- src/egon/data/datasets/zensus_vg250.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/egon/data/datasets/zensus_vg250.py b/src/egon/data/datasets/zensus_vg250.py index b7e8df888..d9598b170 100755 --- a/src/egon/data/datasets/zensus_vg250.py +++ b/src/egon/data/datasets/zensus_vg250.py @@ -249,7 +249,7 @@ def map_zensus_vg250(): cfg["targets"]["map"]["table"], schema=cfg["targets"]["map"]["schema"], con=local_engine, - if_exists="replace", + if_exists="append", ) @@ -531,7 +531,7 @@ def add_metadata_zensus_inside_ger(): ], } - meta_json = "'" + json.dumps(metadata) + "'" + meta_json = json.dumps(metadata) db.submit_comment( meta_json, @@ -674,7 +674,7 @@ def add_metadata_vg250_gem_pop(): ], } - meta_json = "'" + json.dumps(metadata) + "'" + meta_json = json.dumps(metadata) db.submit_comment( meta_json,