From e2fced5679ef5d0924cea28ddf40049d240ae7cc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 16:56:29 +0200 Subject: [PATCH 001/162] feature: copy trigger_schedule endpoint from SensorAPI to AssetAPI Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 183 ++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 14c81edaba..a789bc2cb3 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -318,3 +318,186 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): """ sensors = flatten_unique(asset.sensors_to_show) return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) + + @route("//schedules/trigger", methods=["POST"]) + @use_kwargs( + {"sensor": SensorIdField(data_key="id")}, + location="path", + ) + @use_kwargs( + { + "start_of_schedule": AwareDateTimeField( + data_key="start", format="iso", required=True + ), + "belief_time": AwareDateTimeField(format="iso", data_key="prior"), + "duration": PlanningDurationField( + load_default=PlanningDurationField.load_default + ), + "flex_model": fields.Dict(data_key="flex-model"), + "flex_context": fields.Dict(required=False, data_key="flex-context"), + }, + location="json", + ) + @permission_required_for_context("create-children", ctx_arg_name="sensor") + def trigger_schedule( + self, + sensor: Sensor, + start_of_schedule: datetime, + duration: timedelta, + belief_time: datetime | None = None, + flex_model: dict | None = None, + flex_context: dict | None = None, + **kwargs, + ): + """ + Trigger FlexMeasures to create a schedule. + + .. :quickref: Schedule; Trigger scheduling job + + Trigger FlexMeasures to create a schedule for this sensor. + The assumption is that this sensor is the power sensor on a flexible asset. + + In this request, you can describe: + + - the schedule's main features (when does it start, what unit should it report, prior to what time can we assume knowledge) + - the flexibility model for the sensor (state and constraint variables, e.g. current state of charge of a battery, or connection capacity) + - the flexibility context which the sensor operates in (other sensors under the same EMS which are relevant, e.g. prices) + + For details on flexibility model and context, see :ref:`describing_flexibility`. + Below, we'll also list some examples. + + .. note:: This endpoint does not support to schedule an EMS with multiple flexible sensors at once. This will happen in another endpoint. + See https://github.com/FlexMeasures/flexmeasures/issues/485. Until then, it is possible to call this endpoint for one flexible endpoint at a time + (considering already scheduled sensors as inflexible). + + The length of the schedule can be set explicitly through the 'duration' field. + Otherwise, it is set by the config setting :ref:`planning_horizon_config`, which defaults to 48 hours. + If the flex-model contains targets that lie beyond the planning horizon, the length of the schedule is extended to accommodate them. + Finally, the schedule length is limited by :ref:`max_planning_horizon_config`, which defaults to 2520 steps of the sensor's resolution. + Targets that exceed the max planning horizon are not accepted. + + The appropriate algorithm is chosen by FlexMeasures (based on asset type). + It's also possible to use custom schedulers and custom flexibility models, see :ref:`plugin_customization`. + + If you have ideas for algorithms that should be part of FlexMeasures, let us know: https://flexmeasures.io/get-in-touch/ + + **Example request A** + + This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh. + + .. code-block:: json + + { + "start": "2015-06-02T10:00:00+00:00", + "flex-model": { + "soc-at-start": 12.1, + "soc-unit": "kWh" + } + } + + **Example request B** + + This message triggers a 24-hour schedule for a storage asset, starting at 10.00am, + at which the state of charge (soc) is 12.1 kWh, with a target state of charge of 25 kWh at 4.00pm. + + The charging efficiency is constant (120%) and the discharging efficiency is determined by the contents of sensor + with id 98. If just the ``roundtrip-efficiency`` is known, it can be described with its own field. + The global minimum and maximum soc are set to 10 and 25 kWh, respectively. + To guarantee a minimum SOC in the period prior, the sensor with ID 300 contains beliefs at 2.00pm and 3.00pm, for 15kWh and 20kWh, respectively. + Storage efficiency is set to 99.99%, denoting the state of charge left after each time step equal to the sensor's resolution. + Aggregate consumption (of all devices within this EMS) should be priced by sensor 9, + and aggregate production should be priced by sensor 10, + where the aggregate power flow in the EMS is described by the sum over sensors 13, 14 and 15 + (plus the flexible sensor being optimized, of course). + + + The battery consumption power capacity is limited by sensor 42 and the production capacity is constant (30 kW). + Finally, the site consumption capacity is limited by sensor 32. + + Note that, if forecasts for sensors 13, 14 and 15 are not available, a schedule cannot be computed. + + .. code-block:: json + + { + "start": "2015-06-02T10:00:00+00:00", + "duration": "PT24H", + "flex-model": { + "soc-at-start": 12.1, + "soc-unit": "kWh", + "soc-targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + }, + ], + "soc-minima": {"sensor" : 300}, + "soc-min": 10, + "soc-max": 25, + "charging-efficiency": "120%", + "discharging-efficiency": {"sensor": 98}, + "storage-efficiency": 0.9999, + "power-capacity": "25kW", + "consumption-capacity" : {"sensor": 42}, + "production-capacity" : "30 kW" + }, + "flex-context": { + "consumption-price-sensor": 9, + "production-price-sensor": 10, + "inflexible-device-sensors": [13, 14, 15], + "site-power-capacity": "100kW", + "site-production-capacity": "80kW", + "site-consumption-capacity": {"sensor": 32} + } + } + + **Example response** + + This message indicates that the scheduling request has been processed without any error. + A scheduling job has been created with some Universally Unique Identifier (UUID), + which will be picked up by a worker. + The given UUID may be used to obtain the resulting schedule: see /sensors//schedules/. + + .. sourcecode:: json + + { + "status": "PROCESSED", + "schedule": "364bfd06-c1fa-430b-8d25-8f5a547651fb", + "message": "Request has been processed." + } + + :reqheader Authorization: The authentication token + :reqheader Content-Type: application/json + :resheader Content-Type: application/json + :status 200: PROCESSED + :status 400: INVALID_DATA + :status 401: UNAUTHORIZED + :status 403: INVALID_SENDER + :status 405: INVALID_METHOD + :status 422: UNPROCESSABLE_ENTITY + """ + end_of_schedule = start_of_schedule + duration + scheduler_kwargs = dict( + sensor=sensor, + start=start_of_schedule, + end=end_of_schedule, + resolution=sensor.event_resolution, + belief_time=belief_time, # server time if no prior time was sent + flex_model=flex_model, + flex_context=flex_context, + ) + + try: + job = create_scheduling_job( + **scheduler_kwargs, + enqueue=True, + ) + except ValidationError as err: + return invalid_flex_config(err.messages) + except ValueError as err: + return invalid_flex_config(str(err)) + + db.session.commit() + + response = dict(schedule=job.id) + d, s = request_processed() + return dict(**response, **d), s From bfdd94f9a5fe9e410a4ac1452f6c23dbd47eb743 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 16:57:46 +0200 Subject: [PATCH 002/162] fix: update function signature Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index a789bc2cb3..7cc7fdeafb 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from datetime import datetime, timedelta import json from flask import current_app @@ -12,7 +15,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.schemas import AwareDateTimeField +from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField @@ -321,7 +324,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): @route("//schedules/trigger", methods=["POST"]) @use_kwargs( - {"sensor": SensorIdField(data_key="id")}, + {"asset": AssetIdField(data_key="id")}, location="path", ) @use_kwargs( @@ -338,16 +341,16 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): }, location="json", ) - @permission_required_for_context("create-children", ctx_arg_name="sensor") + @permission_required_for_context("create-children", ctx_arg_name="asset") def trigger_schedule( - self, - sensor: Sensor, - start_of_schedule: datetime, - duration: timedelta, - belief_time: datetime | None = None, - flex_model: dict | None = None, - flex_context: dict | None = None, - **kwargs, + self, + asset: GenericAsset, + start_of_schedule: datetime, + duration: timedelta, + belief_time: datetime | None = None, + flex_model: dict | None = None, + flex_context: dict | None = None, + **kwargs, ): """ Trigger FlexMeasures to create a schedule. From 5795e1bf96f067557006b1f1d21bed03bbda433e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:10:41 +0200 Subject: [PATCH 003/162] fix: get rid of deprecation warning Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index edb31eace8..917894e963 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -363,7 +363,7 @@ def trigger_schedule( """ end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( - sensor=sensor, + asset_or_sensor=sensor, start=start_of_schedule, end=end_of_schedule, resolution=sensor.event_resolution, @@ -465,7 +465,6 @@ def get_schedule( # noqa: C901 ) return unrecognized_event(job.meta["fallback_job_id"], "fallback-job") - scheduler_info_msg = "" scheduler_info = job.meta.get("scheduler_info", dict(scheduler="")) scheduler_info_msg = f"{scheduler_info['scheduler']} was used." From 2e4cfe0f74a6f9cc28b692f727c37ec20bfe0ba4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:13:24 +0200 Subject: [PATCH 004/162] fix: update docstring Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 68 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 7cc7fdeafb..96cc48ef03 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -357,26 +357,26 @@ def trigger_schedule( .. :quickref: Schedule; Trigger scheduling job - Trigger FlexMeasures to create a schedule for this sensor. - The assumption is that this sensor is the power sensor on a flexible asset. + Trigger FlexMeasures to create a schedule for this asset. + The assumption is that this is a flexible asset containing multiple power sensors. In this request, you can describe: - the schedule's main features (when does it start, what unit should it report, prior to what time can we assume knowledge) - - the flexibility model for the sensor (state and constraint variables, e.g. current state of charge of a battery, or connection capacity) - - the flexibility context which the sensor operates in (other sensors under the same EMS which are relevant, e.g. prices) + - the flexibility models for the asset's relevant sensors (state and constraint variables, e.g. current state of charge of a battery, or connection capacity) + - the flexibility context which the asset operates in (other sensors under the same EMS which are relevant, e.g. prices) For details on flexibility model and context, see :ref:`describing_flexibility`. Below, we'll also list some examples. - .. note:: This endpoint does not support to schedule an EMS with multiple flexible sensors at once. This will happen in another endpoint. - See https://github.com/FlexMeasures/flexmeasures/issues/485. Until then, it is possible to call this endpoint for one flexible endpoint at a time + .. note:: This endpoint support scheduling an EMS with multiple flexible sensors at once, + but internally, it does so sequentially (considering already scheduled sensors as inflexible). The length of the schedule can be set explicitly through the 'duration' field. Otherwise, it is set by the config setting :ref:`planning_horizon_config`, which defaults to 48 hours. If the flex-model contains targets that lie beyond the planning horizon, the length of the schedule is extended to accommodate them. - Finally, the schedule length is limited by :ref:`max_planning_horizon_config`, which defaults to 2520 steps of the sensor's resolution. + Finally, the schedule length is limited by :ref:`max_planning_horizon_config`, which defaults to 2520 steps of each sensor's resolution. Targets that exceed the max planning horizon are not accepted. The appropriate algorithm is chosen by FlexMeasures (based on asset type). @@ -392,10 +392,13 @@ def trigger_schedule( { "start": "2015-06-02T10:00:00+00:00", - "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh" - } + "flex-model": [ + { + "sensor": 931, + "soc-at-start": 12.1, + "soc-unit": "kWh" + } + ] } **Example request B** @@ -424,25 +427,28 @@ def trigger_schedule( { "start": "2015-06-02T10:00:00+00:00", "duration": "PT24H", - "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh", - "soc-targets": [ - { - "value": 25, - "datetime": "2015-06-02T16:00:00+00:00" - }, - ], - "soc-minima": {"sensor" : 300}, - "soc-min": 10, - "soc-max": 25, - "charging-efficiency": "120%", - "discharging-efficiency": {"sensor": 98}, - "storage-efficiency": 0.9999, - "power-capacity": "25kW", - "consumption-capacity" : {"sensor": 42}, - "production-capacity" : "30 kW" - }, + "flex-model": [ + { + "sensor": 931, + "soc-at-start": 12.1, + "soc-unit": "kWh", + "soc-targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + }, + ], + "soc-minima": {"sensor" : 300}, + "soc-min": 10, + "soc-max": 25, + "charging-efficiency": "120%", + "discharging-efficiency": {"sensor": 98}, + "storage-efficiency": 0.9999, + "power-capacity": "25kW", + "consumption-capacity" : {"sensor": 42}, + "production-capacity" : "30 kW" + }, + ], "flex-context": { "consumption-price-sensor": 9, "production-price-sensor": 10, @@ -458,7 +464,7 @@ def trigger_schedule( This message indicates that the scheduling request has been processed without any error. A scheduling job has been created with some Universally Unique Identifier (UUID), which will be picked up by a worker. - The given UUID may be used to obtain the resulting schedule: see /sensors//schedules/. + The given UUID may be used to obtain the resulting schedule: see /assets//schedules/. .. sourcecode:: json From 2e28ed96024771dacaa03ff463e6a3d667d9d639 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:13:52 +0200 Subject: [PATCH 005/162] fix: imports Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 96cc48ef03..f587bbe7b2 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -7,7 +7,7 @@ from flask_classful import FlaskView, route from flask_security import auth_required from flask_json import as_json -from marshmallow import fields +from marshmallow import fields, ValidationError from webargs.flaskparser import use_kwargs, use_args from sqlalchemy import select, delete @@ -17,8 +17,13 @@ from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema +from flexmeasures.data.services.scheduling import create_scheduling_job from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField +from flexmeasures.api.common.responses import ( + invalid_flex_config, + request_processed, +) from flexmeasures.utils.coding_utils import flatten_unique from flexmeasures.ui.utils.view_utils import set_session_variables @@ -486,7 +491,7 @@ def trigger_schedule( """ end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( - sensor=sensor, + asset_or_sensor=asset, start=start_of_schedule, end=end_of_schedule, resolution=sensor.event_resolution, From f3711dec8f9183dbc50bc3d3b780b654922b8049 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:23:59 +0200 Subject: [PATCH 006/162] fix: obtain sensors from flex-model Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index f587bbe7b2..66f9ad1aaf 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -15,6 +15,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema from flexmeasures.data.services.scheduling import create_scheduling_job @@ -490,8 +491,14 @@ def trigger_schedule( :status 422: UNPROCESSABLE_ENTITY """ end_of_schedule = start_of_schedule + duration + for sensor_flex_model in flex_model: + sensor_id = sensor_flex_model.get("sensor") + if sensor_id is None: + return invalid_flex_config(f"Missing 'sensor' in flex-model list item: {sensor_flex_model}.") + sensor = SensorIdField().deserialize(sensor_id) + scheduler_kwargs = dict( - asset_or_sensor=asset, + asset_or_sensor=sensor, start=start_of_schedule, end=end_of_schedule, resolution=sensor.event_resolution, From 305d76d502b6afcf3604a6982b4dbf7637faa5b2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:46:23 +0200 Subject: [PATCH 007/162] fix: create one scheduling job for each sensor listed in the flex-model Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 66f9ad1aaf..eccd80f5e0 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -497,25 +497,25 @@ def trigger_schedule( return invalid_flex_config(f"Missing 'sensor' in flex-model list item: {sensor_flex_model}.") sensor = SensorIdField().deserialize(sensor_id) - scheduler_kwargs = dict( - asset_or_sensor=sensor, - start=start_of_schedule, - end=end_of_schedule, - resolution=sensor.event_resolution, - belief_time=belief_time, # server time if no prior time was sent - flex_model=flex_model, - flex_context=flex_context, - ) - - try: - job = create_scheduling_job( - **scheduler_kwargs, - enqueue=True, + scheduler_kwargs = dict( + asset_or_sensor=sensor, + start=start_of_schedule, + end=end_of_schedule, + resolution=sensor.event_resolution, + belief_time=belief_time, # server time if no prior time was sent + flex_model=flex_model, + flex_context=flex_context, ) - except ValidationError as err: - return invalid_flex_config(err.messages) - except ValueError as err: - return invalid_flex_config(str(err)) + + try: + job = create_scheduling_job( + **scheduler_kwargs, + enqueue=True, + ) + except ValidationError as err: + return invalid_flex_config(err.messages) + except ValueError as err: + return invalid_flex_config(str(err)) db.session.commit() From f6e8075b32324a1722026e48f2d5386ac5ea6021 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 16 May 2024 17:47:47 +0200 Subject: [PATCH 008/162] fix: add todos Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index eccd80f5e0..336511d0e6 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -497,6 +497,8 @@ def trigger_schedule( return invalid_flex_config(f"Missing 'sensor' in flex-model list item: {sensor_flex_model}.") sensor = SensorIdField().deserialize(sensor_id) + # todo make sure each sensor lives under the asset + scheduler_kwargs = dict( asset_or_sensor=sensor, start=start_of_schedule, @@ -519,6 +521,7 @@ def trigger_schedule( db.session.commit() + # todo: make a 'done job' and pass that job's ID here response = dict(schedule=job.id) d, s = request_processed() return dict(**response, **d), s From 4bf91dc749992c5f4098c749299d9eb62915fc7c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 09:12:01 +0200 Subject: [PATCH 009/162] docs: update endpoint main descriptions and quickrefs Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 4 ++-- flexmeasures/api/v3_0/sensors.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 336511d0e6..05b4669162 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -359,9 +359,9 @@ def trigger_schedule( **kwargs, ): """ - Trigger FlexMeasures to create a schedule. + Trigger FlexMeasures to create a schedule for a collection of flexible devices. - .. :quickref: Schedule; Trigger scheduling job + .. :quickref: Schedule; Trigger scheduling job for multiple devices Trigger FlexMeasures to create a schedule for this asset. The assumption is that this is a flexible asset containing multiple power sensors. diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 917894e963..5b4e600aa3 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -236,9 +236,9 @@ def trigger_schedule( **kwargs, ): """ - Trigger FlexMeasures to create a schedule. + Trigger FlexMeasures to create a schedule for a single flexible device. - .. :quickref: Schedule; Trigger scheduling job + .. :quickref: Schedule; Trigger scheduling job for one device Trigger FlexMeasures to create a schedule for this sensor. The assumption is that this sensor is the power sensor on a flexible asset. From 0917896e55b219c70f88011fdd4eae135abf610a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 09:13:08 +0200 Subject: [PATCH 010/162] style: black Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 05b4669162..4156992521 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -494,7 +494,9 @@ def trigger_schedule( for sensor_flex_model in flex_model: sensor_id = sensor_flex_model.get("sensor") if sensor_id is None: - return invalid_flex_config(f"Missing 'sensor' in flex-model list item: {sensor_flex_model}.") + return invalid_flex_config( + f"Missing 'sensor' in flex-model list item: {sensor_flex_model}." + ) sensor = SensorIdField().deserialize(sensor_id) # todo make sure each sensor lives under the asset From 753b41143522615b8d8fd3ea3e57d43c943905eb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 09:38:25 +0200 Subject: [PATCH 011/162] fix: changelog syntax Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 493dd36ae1..9a848ff180 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -37,7 +37,7 @@ v0.20.1 | May 7, 2024 Bugfixes ----------- -* Prevent **p**lay/**p**ause/**s**top of replays when editing a text field in the UI [see `PR #1024 `_] +* Prevent **p**\ lay/**p**\ ause/**s**\ top of replays when editing a text field in the UI [see `PR #1024 `_] * Skip unit conversion of :abbr:`SoC (state of charge)` related fields that are defined as sensors in a ``flex-model`` (specifically, ``soc-maxima``, ``soc-minima`` and ``soc-targets`` [see `PR #1047 `_] From ba90dd1bc104ec7cb2712668e2aa3e06ee1c95d4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 09:54:07 +0200 Subject: [PATCH 012/162] fix: remove redundant session commit when calling the API to trigger a scheduling job Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 -- flexmeasures/api/v3_0/sensors.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 4156992521..31cb95c108 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -521,8 +521,6 @@ def trigger_schedule( except ValueError as err: return invalid_flex_config(str(err)) - db.session.commit() - # todo: make a 'done job' and pass that job's ID here response = dict(schedule=job.id) d, s = request_processed() diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 5b4e600aa3..c029bec6cb 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -382,8 +382,6 @@ def trigger_schedule( except ValueError as err: return invalid_flex_config(str(err)) - db.session.commit() - response = dict(schedule=job.id) d, s = request_processed() return dict(**response, **d), s From af4debfe0875165d1d1f7c67c057ffe91213c8a0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 11:44:26 +0200 Subject: [PATCH 013/162] docs: update FlexContextSchema docstring Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ea3bff4762..0f03bbb712 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -4,9 +4,7 @@ class FlexContextSchema(Schema): - """ - This schema lists fields that can be used to describe sensors in the optimised portfolio - """ + """This schema defines fields that provide context to the portfolio to be optimized.""" ems_power_capacity_in_mw = QuantityOrSensor( "MW", From 49a1a127fb27b5c7d9ab916e52066cafd6d1d410 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 13:28:25 +0200 Subject: [PATCH 014/162] feature: helpful message for test developers Signed-off-by: F.N. Claessen --- flexmeasures/api/tests/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/tests/utils.py b/flexmeasures/api/tests/utils.py index 61b71f8fe7..ed2a719e33 100644 --- a/flexmeasures/api/tests/utils.py +++ b/flexmeasures/api/tests/utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pytest import UsageError + import json from flask import url_for, current_app, Response @@ -71,7 +73,12 @@ class UserContext(object): """ def __init__(self, user_email: str): - self.the_user = find_user_by_email(user_email) + user = find_user_by_email(user_email) + if user is None: + raise UsageError( + f"no user with email {user_email} found - test is possible missing a fixture that sets up this user", + ) + self.the_user = user def __enter__(self): return self.the_user From 8a3e1120b167d1ebf55bcd7ab2b13361baa8109a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 13:34:07 +0200 Subject: [PATCH 015/162] feature: check auth on sensors referenced in flex-context Signed-off-by: F.N. Claessen --- flexmeasures/api/conftest.py | 39 ---------- flexmeasures/conftest.py | 36 +++++++++ .../data/schemas/scheduling/__init__.py | 17 ++++- flexmeasures/data/schemas/scheduling/utils.py | 24 ++++++ flexmeasures/data/schemas/tests/conftest.py | 75 ++++++++++++++++--- .../data/schemas/tests/test_scheduling.py | 5 +- 6 files changed, 146 insertions(+), 50 deletions(-) delete mode 100644 flexmeasures/api/conftest.py create mode 100644 flexmeasures/data/schemas/scheduling/utils.py diff --git a/flexmeasures/api/conftest.py b/flexmeasures/api/conftest.py deleted file mode 100644 index 277245588e..0000000000 --- a/flexmeasures/api/conftest.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from flask_login import login_user, logout_user - -from flexmeasures.api.tests.utils import UserContext - - -@pytest.fixture -def requesting_user(request): - """Use this fixture to log in a user for the scope of a test. - - Sets the user by passing it an email address (see usage examples below), or pass None to get the AnonymousUser. - Passes the user object to the test. - Logs the user out after the test ran. - - Usage: - - >>> @pytest.mark.parametrize("requesting_user", ["test_prosumer_user_2@seita.nl", None], indirect=True) - - Or in combination with other parameters: - - @pytest.mark.parametrize( - "requesting_user, status_code", - [ - (None, 401), - ("test_prosumer_user_2@seita.nl", 200), - ], - indirect=["requesting_user"], - ) - - """ - email = request.param - if email is not None: - with UserContext(email) as user: - login_user(user) - yield user - logout_user() - else: - yield diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index d3bee45882..fb04bcc438 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -9,6 +9,7 @@ import pandas as pd import numpy as np from flask import request, jsonify +from flask_login import login_user, logout_user from flask_sqlalchemy import SQLAlchemy from flask_security import roles_accepted from pytest_mock import MockerFixture @@ -22,6 +23,7 @@ Gone, ) +from flexmeasures.api.tests.utils import UserContext from flexmeasures.app import create as create_app from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE from flexmeasures.data.services.users import create_user @@ -1362,3 +1364,37 @@ def add_beliefs( @pytest.fixture def mock_get_status(mocker: MockerFixture): return mocker.patch("flexmeasures.data.services.sensors.get_status", autospec=True) + + +@pytest.fixture +def requesting_user(request): + """Use this fixture to log in a user for the scope of a test. + + Sets the user by passing it an email address (see usage examples below), or pass None to get the AnonymousUser. + Passes the user object to the test. + Logs the user out after the test ran. + + Usage: + + >>> @pytest.mark.parametrize("requesting_user", ["test_prosumer_user_2@seita.nl", None], indirect=True) + + Or in combination with other parameters: + + @pytest.mark.parametrize( + "requesting_user, status_code", + [ + (None, 401), + ("test_prosumer_user_2@seita.nl", 200), + ], + indirect=["requesting_user"], + ) + + """ + email = request.param + if email is not None: + with UserContext(email) as user: + login_user(user) + yield user + logout_user() + else: + yield diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 0f03bbb712..ae1d99a0f5 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,5 +1,8 @@ -from marshmallow import Schema, fields, validate +from marshmallow import Schema, fields, validate, validates_schema, ValidationError +from werkzeug.exceptions import Forbidden +from flexmeasures.auth.policy import check_access +from flexmeasures.data.schemas.scheduling.utils import find_sensors from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField @@ -29,3 +32,15 @@ class FlexContextSchema(Schema): inflexible_device_sensors = fields.List( SensorIdField(), data_key="inflexible-device-sensors" ) + + @validates_schema + def check_read_access_on_sensors(self, data: dict, **kwargs): + sensors = find_sensors(data) + for sensor, field_name in sensors: + try: + check_access(context=sensor, permission="read") + except Forbidden: + raise ValidationError( + message=f"User has no read access to sensor {sensor.id}.", + field_name=self.fields[field_name].data_key, + ) diff --git a/flexmeasures/data/schemas/scheduling/utils.py b/flexmeasures/data/schemas/scheduling/utils.py new file mode 100644 index 0000000000..4655a4a04a --- /dev/null +++ b/flexmeasures/data/schemas/scheduling/utils.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from flexmeasures import Sensor + + +def find_sensors(data, parent_key='', path='') -> list[tuple[Sensor, str]]: + """Recursively find all sensors in a nested dictionary or list along with the fields referring to them.""" + sensors = [] + + if isinstance(data, dict): + for key, value in data.items(): + new_parent_key = f"{parent_key}.{key}" if parent_key else key + new_path = f"{path}.{key}" if path else key + if isinstance(value, Sensor): + sensors.append((value, f"{new_parent_key}{path}")) + else: + sensors.extend(find_sensors(value, new_parent_key, new_path)) + elif isinstance(data, list): + for index, item in enumerate(data): + new_parent_key = f"{parent_key}[{index}]" + new_path = f"{path}[{index}]" + sensors.extend(find_sensors(item, new_parent_key, new_path)) + + return sensors diff --git a/flexmeasures/data/schemas/tests/conftest.py b/flexmeasures/data/schemas/tests/conftest.py index 0754f0f6e6..d393dd6dae 100644 --- a/flexmeasures/data/schemas/tests/conftest.py +++ b/flexmeasures/data/schemas/tests/conftest.py @@ -1,25 +1,80 @@ import pytest from datetime import timedelta -from flexmeasures.data.models.time_series import Sensor, TimedBelief +from flask_security import SQLAlchemySessionUserDatastore, hash_password + +from flexmeasures import Sensor, User, UserRole +from flexmeasures.data.models.time_series import TimedBelief +from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @pytest.fixture(scope="module") -def dummy_asset(db, app): +def dummy_accounts(db, app): + dummy_account_1 = Account(name="dummy account 1") + db.session.add(dummy_account_1) + + dummy_account_2 = Account(name="dummy account 2") + db.session.add(dummy_account_2) + + # Assign account IDs + db.session.flush() + + return { + dummy_account_1.name: dummy_account_1, + dummy_account_2.name: dummy_account_2, + } + + +@pytest.fixture(scope="module") +def dummy_user(db, app, dummy_accounts): + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, UserRole) + user = user_datastore.create_user( + username="dummy user", + email="dummy_user@seita.nl", + password=hash_password("testtest"), + account_id=dummy_accounts["dummy account 1"].id, + active=True, + ) + return user + + +@pytest.fixture(scope="module") +def dummy_assets(db, app, dummy_accounts): dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") db.session.add(dummy_asset_type) - _dummy_asset = GenericAsset( - name="DummyGenericAsset", generic_asset_type=dummy_asset_type + dummy_asset_1 = GenericAsset( + name="dummy asset 1", + generic_asset_type=dummy_asset_type, + owner=dummy_accounts["dummy account 1"], + ) + db.session.add(dummy_asset_1) + + dummy_asset_2 = GenericAsset( + name="dummy asset 2", + generic_asset_type=dummy_asset_type, + owner=dummy_accounts["dummy account 1"], + ) + db.session.add(dummy_asset_2) + + dummy_asset_3 = GenericAsset( + name="dummy asset 3", + generic_asset_type=dummy_asset_type, + owner=dummy_accounts["dummy account 2"], ) - db.session.add(_dummy_asset) + db.session.add(dummy_asset_3) - return _dummy_asset + return { + dummy_asset_1.name: dummy_asset_1, + dummy_asset_2.name: dummy_asset_2, + dummy_asset_3.name: dummy_asset_3, + } @pytest.fixture(scope="module") -def setup_dummy_sensors(db, app, dummy_asset): +def setup_dummy_sensors(db, app, dummy_assets): + dummy_asset = dummy_assets["dummy asset 1"] sensor1 = Sensor( "sensor 1", generic_asset=dummy_asset, @@ -58,7 +113,8 @@ def setup_dummy_sensors(db, app, dummy_asset): @pytest.fixture(scope="module") -def setup_efficiency_sensors(db, app, dummy_asset): +def setup_efficiency_sensors(db, app, dummy_assets): + dummy_asset = dummy_assets["dummy asset 1"] sensor = Sensor( "efficiency", generic_asset=dummy_asset, @@ -72,7 +128,8 @@ def setup_efficiency_sensors(db, app, dummy_asset): @pytest.fixture(scope="module") -def setup_site_capacity_sensor(db, app, dummy_asset, setup_sources): +def setup_site_capacity_sensor(db, app, dummy_assets, setup_sources): + dummy_asset = dummy_assets["dummy asset 1"] sensor = Sensor( "site-power-capacity", generic_asset=dummy_asset, diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 2d18e85ef5..0eb7aab20a 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -239,7 +239,10 @@ def load_schema(): ), ], ) -def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context, fails): +@pytest.mark.parametrize( + "requesting_user", ["dummy_user@seita.nl"], indirect=True +) +def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context, fails, dummy_user, requesting_user): schema = FlexContextSchema() # Replace sensor name with sensor ID From 25d11804b915b00fc148c31dbe5dcfd05619c0b3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:46:24 +0200 Subject: [PATCH 016/162] feature: allow checking permissions on optional fields Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 2655bb42a2..8c09260713 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -178,7 +178,10 @@ def decorated_view(*args, **kwargs): elif ctx_arg_pos is not None: context_from_args = args[ctx_arg_pos] elif ctx_arg_name is not None: - context_from_args = kwargs[ctx_arg_name] + context_from_args = kwargs.get(ctx_arg_name) + # skip check in case (optional) argument was not passed + if context_from_args is None: + return fn(*args, **kwargs) elif len(args) > 0: context_from_args = args[0] From 21ac09a0fe6b4d8c6005de5fdd3aff6ad7ee91f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:52:03 +0200 Subject: [PATCH 017/162] feature: decorator supports custom error handler Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 8c09260713..9dcf94ba3e 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -111,6 +111,7 @@ def permission_required_for_context( ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, pass_ctx_to_loader: bool = False, + error_handler: Callable | None = None, ): """ This decorator can be used to make sure that the current user has the necessary permission to access the context. @@ -119,6 +120,7 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. + A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. We will now explain how to load a context, and give an example: @@ -199,7 +201,12 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - check_access(context, permission) + try: + check_access(context, permission) + except Exception as e: + if error_handler: + return error_handler(context, permission, ctx_arg_name) + raise e return fn(*args, **kwargs) From b4ad01a0c721bde1cd45e465c8b552728e298b00 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:53:22 +0200 Subject: [PATCH 018/162] docs: add inline note explaining status code Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index be7b67760f..e7fd1f40db 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -84,7 +84,7 @@ def test_trigger_schedule_with_invalid_flexmodel( ) print("Server responded with:\n%s" % trigger_schedule_response.json) check_deprecation(trigger_schedule_response, deprecation=None, sunset=None) - assert trigger_schedule_response.status_code == 422 + assert trigger_schedule_response.status_code == 422 # Unprocessable entity assert field in trigger_schedule_response.json["message"]["json"] if isinstance(trigger_schedule_response.json["message"]["json"], str): # ValueError From 5b4c30fb18274b6b91d8b70fcfc586853b8e158e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 17:56:23 +0200 Subject: [PATCH 019/162] feature: flex_context_loader lists all sensors contained in a flex-context Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/utils.py | 69 ++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 908cf65e6f..b1ba739fa0 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -441,3 +441,72 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser # [right]: datetime64[ns, UTC] value = value.tz_convert("UTC") return s.fillna(value).clip(upper=value) + + +def flex_context_loader(context) -> list[tuple[Sensor, str]]: + sensor_ids = find_sensor_ids(context, "flex-context") + sensors = [ + (db.session.get(Sensor, sensor_id), field_name) + for sensor_id, field_name in sensor_ids + ] + return sensors + + +def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: + """ + Recursively find all sensor IDs in a nested dictionary or list along with the fields referring to them. + + Args: + data (dict or list): The input data which can be a dictionary or a list containing nested dictionaries and lists. + parent_key (str): The key of the parent element in the recursion, used to track the referring fields. + + Returns: + list: A list of tuples, each containing a sensor ID and the referring field. + + Example: + nested_dict = { + "flex-model": [ + { + "sensor": 931, + "soc-at-start": 12.1, + "soc-unit": "kWh", + "soc-targets": [ + { + "value": 25, + "datetime": "2015-06-02T16:00:00+00:00" + }, + ], + "soc-minima": {"sensor": 300}, + "soc-min": 10, + "soc-max": 25, + "charging-efficiency": "120%", + "discharging-efficiency": {"sensor": 98}, + "storage-efficiency": 0.9999, + "power-capacity": "25kW", + "consumption-capacity": {"sensor": 42}, + "production-capacity": "30 kW" + }, + ], + } + + sensor_ids = find_sensor_ids(nested_dict) + print(sensor_ids) # Output: [(931, 'sensor'), (300, 'soc-minima.sensor'), (98, 'discharging-efficiency.sensor'), (42, 'consumption-capacity.sensor')] + """ + sensor_ids = [] + + if isinstance(data, dict): + for key, value in data.items(): + new_parent_key = f"{parent_key}.{key}" if parent_key else key + if key[-6:] == "sensor": + sensor_ids.append((value, new_parent_key)) + elif key[-7:] == "sensors": + for v in value: + sensor_ids.append((v, new_parent_key)) + else: + sensor_ids.extend(find_sensor_ids(value, new_parent_key)) + elif isinstance(data, list): + for index, item in enumerate(data): + new_parent_key = f"{parent_key}[{index}]" + sensor_ids.extend(find_sensor_ids(item, new_parent_key)) + + return sensor_ids From 484a022ada850082ef4793b4cbce81dcaaeece91 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:01:45 +0200 Subject: [PATCH 020/162] feature: support context loader that returns multiple contexts Signed-off-by: F.N. Claessen --- flexmeasures/auth/decorators.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 9dcf94ba3e..70d710c07d 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -120,7 +120,7 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. - A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. + A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and a context origin. We will now explain how to load a context, and give an example: @@ -148,6 +148,7 @@ def view(resource_id: int, the_resource: Resource): The ctx_loader: The ctx_loader can be a function without arguments or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). + It should return the context or a list of contexts. A special case is useful when the arguments contain the context ID (not the instance). Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. @@ -201,12 +202,22 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - try: - check_access(context, permission) - except Exception as e: - if error_handler: - return error_handler(context, permission, ctx_arg_name) - raise e + # Check access for possibly multiple contexts + if not isinstance(context, list): + context = [context] + for ctx in context: + if isinstance(ctx, tuple): + c = ctx[0] # the context + origin = ctx[1] # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) + else: + c = ctx + origin = ctx_arg_name + try: + check_access(c, permission) + except Exception as e: + if error_handler: + return error_handler(c, permission, origin) + raise e return fn(*args, **kwargs) From 8374a9cf3ac2b0219b690a582fe67f4033165103 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:04:14 +0200 Subject: [PATCH 021/162] feature: check permissions on sensors referenced in flex-context Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index c029bec6cb..f377d11324 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -33,6 +33,7 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.models.planning.utils import flex_context_loader from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField @@ -225,6 +226,16 @@ def get_data(self, sensor_data_description: dict): location="json", ) @permission_required_for_context("create-children", ctx_arg_name="sensor") + @permission_required_for_context( + "read", + ctx_arg_name="flex_context", + ctx_loader=flex_context_loader, + pass_ctx_to_loader=True, + error_handler=lambda context, permission, origin: invalid_flex_config( + f"User has no {permission} authorization on sensor {context.id}", + origin, + ), + ) def trigger_schedule( self, sensor: Sensor, From 4621ef6486cdb1f76a79d8ba5c484a86a6af9971 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:04:38 +0200 Subject: [PATCH 022/162] feature: add test checking permissions Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/conftest.py | 13 +++++ .../api/v3_0/tests/test_sensor_schedules.py | 48 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 74a70383f0..8d2277892f 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -201,3 +201,16 @@ def add_temperature_measurements(db, source: Source, sensor: Sensor): for event_start, event_value in zip(event_starts, event_values) ] db.session.add_all(beliefs) + + +@pytest.fixture(scope="module") +def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets): + asset = setup_generic_assets["test_wind_turbine"] + sensor = Sensor( + name="capacity", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MVA", + ) + db.session.add(sensor) + return sensor diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index e7fd1f40db..4428be296c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -422,3 +422,51 @@ def test_get_schedule_fallback_not_redirect( assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False + + +@pytest.mark.parametrize( + "message, flex_config, field, err_msg", + [ + (message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", "no read authorization"), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_with_unauthorized_sensor( + app, + add_battery_assets, + setup_capacity_sensor_on_asset_in_supplier_account, + keep_scheduling_queue_empty, + message, + flex_config, + field, + err_msg, + requesting_user, +): + """Test triggering a schedule using a flex config that refers to a capacity sensor from a different account. + + The user is not authorized to read sensors from the other account, so we expect a 403 (Forbidden) response. + """ + sensor = add_battery_assets["Test battery"].sensors[0] + with app.test_client() as client: + if flex_config not in message: + message[flex_config] = {} + sensor_id = setup_capacity_sensor_on_asset_in_supplier_account.id + message[flex_config][field] = {"sensor": sensor_id} + + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + assert trigger_schedule_response.status_code == 422 # Unprocessable entity + assert f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] + if isinstance(trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], str): + # ValueError + assert err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"] + else: + # ValidationError (marshmallow) + assert ( + err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"][field][0] + ) From ff22fea526f47a7433fa589eb10f97b80c7e6847 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 May 2024 18:08:44 +0200 Subject: [PATCH 023/162] fix: response with field names Signed-off-by: F.N. Claessen --- flexmeasures/api/common/responses.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index c0e959a62f..1f0d3bc763 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -274,10 +274,12 @@ def fallback_schedule_redirect(message: str, location: str) -> ResponseTuple: ) -def invalid_flex_config(message: str) -> ResponseTuple: +def invalid_flex_config(message: str, field_name: str | None = None) -> ResponseTuple: return ( dict( - result="Rejected", status="UNPROCESSABLE_ENTITY", message=dict(json=message) + result="Rejected", + status="UNPROCESSABLE_ENTITY", + message={field_name if field_name else "json": message}, ), 422, ) From be51f1b9e8513119739d121ad7756394ecc7ff65 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 17 May 2024 18:33:56 +0200 Subject: [PATCH 024/162] add create_sequential_scheduling_job function Signed-off-by: Victor Garcia Reolid --- flexmeasures/api/v3_0/assets.py | 48 +++++------- flexmeasures/data/services/scheduling.py | 98 ++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 30 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 31cb95c108..e1b52fa641 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -15,10 +15,11 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema -from flexmeasures.data.services.scheduling import create_scheduling_job +from flexmeasures.data.services.scheduling import ( + create_sequential_scheduling_job, +) from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.responses import ( @@ -491,35 +492,22 @@ def trigger_schedule( :status 422: UNPROCESSABLE_ENTITY """ end_of_schedule = start_of_schedule + duration - for sensor_flex_model in flex_model: - sensor_id = sensor_flex_model.get("sensor") - if sensor_id is None: - return invalid_flex_config( - f"Missing 'sensor' in flex-model list item: {sensor_flex_model}." - ) - sensor = SensorIdField().deserialize(sensor_id) - - # todo make sure each sensor lives under the asset - - scheduler_kwargs = dict( - asset_or_sensor=sensor, - start=start_of_schedule, - end=end_of_schedule, - resolution=sensor.event_resolution, - belief_time=belief_time, # server time if no prior time was sent - flex_model=flex_model, - flex_context=flex_context, - ) - try: - job = create_scheduling_job( - **scheduler_kwargs, - enqueue=True, - ) - except ValidationError as err: - return invalid_flex_config(err.messages) - except ValueError as err: - return invalid_flex_config(str(err)) + scheduler_kwargs = dict( + start=start_of_schedule, + end=end_of_schedule, + belief_time=belief_time, # server time if no prior time was sent + flex_model=flex_model, + flex_context=flex_context, + ) + try: + job = create_sequential_scheduling_job( + asset_or_sensor=asset, enqueue=True, **scheduler_kwargs + ) + except ValidationError as err: + return invalid_flex_config(err.messages) + except ValueError as err: + return invalid_flex_config(str(err)) # todo: make a 'done job' and pass that job's ID here response = dict(schedule=job.id) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index b3043a7341..5ee73923f3 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -11,10 +11,12 @@ from importlib.abc import Loader from typing import Type import inspect +from copy import deepcopy from flask import current_app import click +from jsonschema import ValidationError from rq import get_current_job, Callback from rq.job import Job import timely_beliefs as tb @@ -30,6 +32,7 @@ from flexmeasures.data.models.generic_assets import GenericAsset as Asset from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.utils import get_data_source, save_to_db +from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.utils.time_utils import server_now from flexmeasures.data.services.utils import ( job_cache, @@ -150,6 +153,7 @@ def create_scheduling_job( requeue: bool = False, force_new_job_creation: bool = False, scheduler_specs: dict | None = None, + depends_on: Job | list[Job] | None = None, **scheduler_kwargs, ) -> Job: """ @@ -219,6 +223,7 @@ def create_scheduling_job( ).total_seconds() ), # NB job.cleanup docs says a negative number of seconds means persisting forever on_failure=Callback(trigger_optional_fallback), + depends_on=depends_on, ) job.meta["asset_or_sensor"] = asset_or_sensor @@ -241,6 +246,99 @@ def create_scheduling_job( return job +def cb_done_sequential_scheduling_job(jobs_ids: list[str]): + """ + TODO: add logic + """ + + # jobs = [Job.fetch(job_id) for job_id in jobs_ids] + pass + + +def create_sequential_scheduling_job( + asset_or_sensor: Asset | Sensor | None = None, + sensor: Sensor | None = None, + job_id: str | None = None, + enqueue: bool = True, + requeue: bool = False, + force_new_job_creation: bool = False, + scheduler_specs: dict | None = None, + depends_on: list[Job] | None = None, + **scheduler_kwargs, +): + flex_model = scheduler_kwargs["flex_model"] + jobs = [] + previous_sensors = [] + previous_job = depends_on + for child_flex_model in flex_model: + sensor_id = child_flex_model.pop("sensor") + if sensor_id is None: + raise ValidationError( + f"Missing 'sensor' in flex-model list item: {child_flex_model}." + ) + + sensor = SensorIdField().deserialize(sensor_id) + + # todo make sure each sensor lives under the asset + + current_scheduler_kwargs = deepcopy(scheduler_kwargs) + + current_scheduler_kwargs["flex_model"] = child_flex_model + current_scheduler_kwargs["flex_context"]["inflexible-device-sensors"].extend( + previous_sensors + ) + current_scheduler_kwargs["resolution"] = sensor.event_resolution + current_scheduler_kwargs["sensor"] = sensor + + job = create_scheduling_job( + **current_scheduler_kwargs, + scheduler_specs=scheduler_specs, + requeue=requeue, + job_id=job_id, + enqueue=enqueue, + depends_on=previous_job, + force_new_job_creation=force_new_job_creation, + ) + jobs.append(job) + previous_sensors.append(sensor.id) + previous_job = job + + # create that triggers when the last job is done + job = Job.create( + func=cb_done_sequential_scheduling_job, + args=([j.id for j in jobs],), + depends_on=previous_job, + ttl=int( + current_app.config.get( + "FLEXMEASURES_JOB_TTL", timedelta(-1) + ).total_seconds() + ), + result_ttl=int( + current_app.config.get( + "FLEXMEASURES_PLANNING_TTL", timedelta(-1) + ).total_seconds() + ), # NB job.cleanup docs says a negative number of seconds means persisting forever + connection=current_app.queues["scheduling"].connection, + ) + + job_status = job.get_status(refresh=True) + + jobs.append(job) + + # with job_status=None, we ensure that only fresh new jobs are enqueued (in the contrary they should be requeued) + if enqueue and not job_status: + for job in jobs: + current_app.queues["scheduling"].enqueue_job(job) + current_app.job_cache.add( + asset_or_sensor["id"], + job.id, + queue="scheduling", + asset_or_sensor_type=asset_or_sensor["class"].lower(), + ) + + return jobs + + def make_schedule( sensor_id: int | None = None, start: datetime | None = None, From 061e1f38bd3c49ab6b3d76521c761a3f4764a373 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 17 May 2024 18:35:47 +0200 Subject: [PATCH 025/162] add fixtures Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/tests/conftest.py | 127 +++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index b85995b809..5f78d54faa 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -8,6 +8,7 @@ import numpy as np from flask_sqlalchemy import SQLAlchemy from statsmodels.api import OLS +from flexmeasures import AssetType, Asset, Sensor import timely_beliefs as tb from sqlalchemy import select from flexmeasures.data.models.reporting import Reporter @@ -15,7 +16,7 @@ from flexmeasures.data.schemas.reporting import ReporterParametersSchema from flexmeasures.data.models.annotations import Annotation from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.models.time_series import TimedBelief, Sensor +from flexmeasures.data.models.time_series import TimedBelief from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.forecasting import model_map from flexmeasures.data.models.forecasting.model_spec_factory import ( @@ -226,3 +227,127 @@ def _compute_report(self, **kwargs) -> list: db.session.commit() return ds + + +@pytest.fixture +def smart_building_types(app, db): + site = AssetType(name="site") + solar = AssetType(name="solar") + building = AssetType(name="building") + battery = AssetType(name="battery") + ev = AssetType(name="ev") + + db.session.add_all([site, solar, building, battery, ev]) + db.session.flush() + + return (site, solar, building, battery, ev) + + +@pytest.fixture +def smart_building(app, db, smart_building_types): + """ + Topology of the sytstem: + + +----------+ + | | + +----------------+ Site +---------------+ + | | | | + | +--+----+--+ | + | | | | + | | | | + | +----- ----+ | + | | | | + +----+----+ +------+-----+ +---+---+ +------+------+ + | | | | | | | | + | Solar | | Building | | EV | | Battery | + | | | | | | | | + +---------+ +------------+ +-------+ +-------------+ + + Diagram created with: https://textik.com/#924f8a2112551f92 + + """ + site, solar, building, battery, ev = smart_building_types + coordinates = {"latitude": 0, "longitude": 0} + + test_site = Asset(name="Test Site", generic_asset_type=site, **coordinates) + db.session.add(test_site) + db.session.flush() + + test_building = Asset( + name="Test Building", + generic_asset_type=building, + parent_asset_id=site.id, + **coordinates, + ) + test_solar = Asset( + name="Test Solar", + generic_asset_type=solar, + parent_asset_id=site.id, + **coordinates, + ) + test_battery = Asset( + name="Test Battery", + generic_asset_type=battery, + parent_asset_id=site.id, + **coordinates, + ) + test_ev = Asset( + name="Test EV", generic_asset_type=ev, parent_asset_id=site.id, **coordinates + ) + + assets = (test_site, test_building, test_solar, test_battery, test_ev) + + db.session.add_all(assets) + db.session.flush() + + sensors = [] + + # Add power sensor + for asset in assets: + sensor = Sensor( + name="power", + unit="MW", + event_resolution="PT15M", + generic_asset=asset, + # TODO: add knowledge horizon function? + ) + sensors.append(sensor) + + db.session.add_all(sensors) + db.session.flush() + asset_names = [asset.name for asset in assets] + return dict(zip(asset_names, assets)), dict(zip(asset_names, sensors)) + + +@pytest.fixture +def flex_description_sequential(smart_building, setup_markets): + assets, sensors = smart_building + + return { + "flex_model": [ + { + "sensor": sensors["Test EV"].id, + "power-capacity": "10kW", + "soc-at-start": 0.01, # 10 kWh + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.05, # 50 kWh + }, + { + "sensor": sensors["Test Battery"].id, + "power-capacity": "20kW", + "soc-at-start": 0.01, # 10 kWh + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.1, # 100 kWh + }, + ], + "flex_context": { + "consumption-price-sensor": setup_markets["epex_da"].id, + "production-price-sensor": setup_markets["epex_da_production"].id, + "inflexible-device-sensors": [ + sensors["Test Solar"].id, + sensors["Test Building"].id, + ], + }, + } From 6eb122b2827f72472e46bb53eabb6d044158c1e7 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Fri, 17 May 2024 18:36:54 +0200 Subject: [PATCH 026/162] add test_create_sequential_jobs Signed-off-by: Victor Garcia Reolid --- .../data/tests/test_scheduling_sequential.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 flexmeasures/data/tests/test_scheduling_sequential.py diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py new file mode 100644 index 0000000000..ad183cad81 --- /dev/null +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -0,0 +1,57 @@ +import pandas as pd +from flexmeasures.data.services.scheduling import create_sequential_scheduling_job + +# from flexmeasures.data.tests.utils import work_on_rq, exception_reporter + + +def test_create_sequential_jobs(db, app, flex_description_sequential, smart_building): + assets, sensors = smart_building + # queue = app.queues["scheduling"] + start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") + end = pd.Timestamp("2015-01-04").tz_localize("Europe/Amsterdam") + + scheduler_specs = { + "module": "flexmeasures.data.models.planning.storage", + "class": "StorageScheduler", + } + + flex_description_sequential["start"] = start.isoformat() + flex_description_sequential["end"] = end.isoformat() + + jobs = create_sequential_scheduling_job( + asset_or_sensor=assets["Test Site"], + scheduler_specs=scheduler_specs, + enqueue=False, + **flex_description_sequential + ) + + assert len(jobs) == 3 + + # The EV is scheduled firstly. + assert jobs[0].kwargs["asset_or_sensor"] == { + "id": sensors["Test EV"].id, + "class": "Sensor", + } + # It uses the inflxible-device-sensors that are defined in the flex-conctext, exclusively. + assert jobs[0].kwargs["flex_context"]["inflexible-device-sensors"] == [ + sensors["Test Solar"].id, + sensors["Test Building"].id, + ] + + # The Battery is scheduled secondly. + assert jobs[1].kwargs["asset_or_sensor"] == { + "id": sensors["Test Battery"].id, + "class": "Sensor", + } + # In addition to the inflexible devices already present in the flex-context (PV and Building), the power sensor of the EV is included. + assert jobs[1].kwargs["flex_context"]["inflexible-device-sensors"] == [ + sensors["Test Solar"].id, + sensors["Test Building"].id, + sensors["Test EV"].id, + ] + + # TODO: enqueue jobs, let them run and check results + # for job in jobs: + # queue.enqueue_job(job) + + # work_on_rq(queue, exc_handler=exception_reporter) From 22834ca0084ca4ab42ac01bd6bf6506209a7e8b4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:01 +0200 Subject: [PATCH 027/162] Revert "fix: response with field names" This reverts commit ff22fea526f47a7433fa589eb10f97b80c7e6847. --- flexmeasures/api/common/responses.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index 1f0d3bc763..c0e959a62f 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -274,12 +274,10 @@ def fallback_schedule_redirect(message: str, location: str) -> ResponseTuple: ) -def invalid_flex_config(message: str, field_name: str | None = None) -> ResponseTuple: +def invalid_flex_config(message: str) -> ResponseTuple: return ( dict( - result="Rejected", - status="UNPROCESSABLE_ENTITY", - message={field_name if field_name else "json": message}, + result="Rejected", status="UNPROCESSABLE_ENTITY", message=dict(json=message) ), 422, ) From dc07b11d96d24e5af6e4cd283ae9f6b41e19cd3e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:01 +0200 Subject: [PATCH 028/162] Revert "feature: add test checking permissions" This reverts commit 4621ef6486cdb1f76a79d8ba5c484a86a6af9971. --- flexmeasures/api/v3_0/tests/conftest.py | 13 ----- .../api/v3_0/tests/test_sensor_schedules.py | 48 ------------------- 2 files changed, 61 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py index 8d2277892f..74a70383f0 100644 --- a/flexmeasures/api/v3_0/tests/conftest.py +++ b/flexmeasures/api/v3_0/tests/conftest.py @@ -201,16 +201,3 @@ def add_temperature_measurements(db, source: Source, sensor: Sensor): for event_start, event_value in zip(event_starts, event_values) ] db.session.add_all(beliefs) - - -@pytest.fixture(scope="module") -def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets): - asset = setup_generic_assets["test_wind_turbine"] - sensor = Sensor( - name="capacity", - generic_asset=asset, - event_resolution=timedelta(minutes=15), - unit="MVA", - ) - db.session.add(sensor) - return sensor diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 4428be296c..e7fd1f40db 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -422,51 +422,3 @@ def test_get_schedule_fallback_not_redirect( assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - - -@pytest.mark.parametrize( - "message, flex_config, field, err_msg", - [ - (message_for_trigger_schedule(), "flex-context", "site-consumption-capacity", "no read authorization"), - ], -) -@pytest.mark.parametrize( - "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True -) -def test_trigger_schedule_with_unauthorized_sensor( - app, - add_battery_assets, - setup_capacity_sensor_on_asset_in_supplier_account, - keep_scheduling_queue_empty, - message, - flex_config, - field, - err_msg, - requesting_user, -): - """Test triggering a schedule using a flex config that refers to a capacity sensor from a different account. - - The user is not authorized to read sensors from the other account, so we expect a 403 (Forbidden) response. - """ - sensor = add_battery_assets["Test battery"].sensors[0] - with app.test_client() as client: - if flex_config not in message: - message[flex_config] = {} - sensor_id = setup_capacity_sensor_on_asset_in_supplier_account.id - message[flex_config][field] = {"sensor": sensor_id} - - trigger_schedule_response = client.post( - url_for("SensorAPI:trigger_schedule", id=sensor.id), - json=message, - ) - print("Server responded with:\n%s" % trigger_schedule_response.json) - assert trigger_schedule_response.status_code == 422 # Unprocessable entity - assert f"{flex_config}.{field}.sensor" in trigger_schedule_response.json["message"] - if isinstance(trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"], str): - # ValueError - assert err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"] - else: - # ValidationError (marshmallow) - assert ( - err_msg in trigger_schedule_response.json["message"][f"{flex_config}.{field}.sensor"][field][0] - ) From b583c893f1215c6846efb072b47d4509db5223c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:01 +0200 Subject: [PATCH 029/162] Revert "feature: check permissions on sensors referenced in flex-context" This reverts commit 8374a9cf3ac2b0219b690a582fe67f4033165103. --- flexmeasures/api/v3_0/sensors.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index f377d11324..c029bec6cb 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -33,7 +33,6 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.planning.utils import flex_context_loader from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.queries.utils import simplify_index from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField @@ -226,16 +225,6 @@ def get_data(self, sensor_data_description: dict): location="json", ) @permission_required_for_context("create-children", ctx_arg_name="sensor") - @permission_required_for_context( - "read", - ctx_arg_name="flex_context", - ctx_loader=flex_context_loader, - pass_ctx_to_loader=True, - error_handler=lambda context, permission, origin: invalid_flex_config( - f"User has no {permission} authorization on sensor {context.id}", - origin, - ), - ) def trigger_schedule( self, sensor: Sensor, From d04bed0a327fb4e5d700372824a3cf0e5a91c7fd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:03 +0200 Subject: [PATCH 030/162] Revert "feature: support context loader that returns multiple contexts" This reverts commit 484a022ada850082ef4793b4cbce81dcaaeece91. --- flexmeasures/auth/decorators.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 70d710c07d..9dcf94ba3e 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -120,7 +120,7 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. - A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and a context origin. + A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. We will now explain how to load a context, and give an example: @@ -148,7 +148,6 @@ def view(resource_id: int, the_resource: Resource): The ctx_loader: The ctx_loader can be a function without arguments or it takes the context loaded from the arguments as input (using pass_ctx_to_loader=True). - It should return the context or a list of contexts. A special case is useful when the arguments contain the context ID (not the instance). Then, the loader can be a subclass of AuthModelMixin, and this decorator will look up the instance. @@ -202,22 +201,12 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - # Check access for possibly multiple contexts - if not isinstance(context, list): - context = [context] - for ctx in context: - if isinstance(ctx, tuple): - c = ctx[0] # the context - origin = ctx[1] # the context loader may narrow down the origin of the context (e.g. a nested field rather than a function argument) - else: - c = ctx - origin = ctx_arg_name - try: - check_access(c, permission) - except Exception as e: - if error_handler: - return error_handler(c, permission, origin) - raise e + try: + check_access(context, permission) + except Exception as e: + if error_handler: + return error_handler(context, permission, ctx_arg_name) + raise e return fn(*args, **kwargs) From 5fd2f631b416136244801d7a7aa60cc0e77701c0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:05 +0200 Subject: [PATCH 031/162] Revert "feature: flex_context_loader lists all sensors contained in a flex-context" This reverts commit 5b4c30fb18274b6b91d8b70fcfc586853b8e158e. --- flexmeasures/data/models/planning/utils.py | 69 ---------------------- 1 file changed, 69 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index b1ba739fa0..908cf65e6f 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -441,72 +441,3 @@ def nanmin_of_series_and_value(s: pd.Series, value: float | pd.Series) -> pd.Ser # [right]: datetime64[ns, UTC] value = value.tz_convert("UTC") return s.fillna(value).clip(upper=value) - - -def flex_context_loader(context) -> list[tuple[Sensor, str]]: - sensor_ids = find_sensor_ids(context, "flex-context") - sensors = [ - (db.session.get(Sensor, sensor_id), field_name) - for sensor_id, field_name in sensor_ids - ] - return sensors - - -def find_sensor_ids(data, parent_key="") -> list[tuple[int, str]]: - """ - Recursively find all sensor IDs in a nested dictionary or list along with the fields referring to them. - - Args: - data (dict or list): The input data which can be a dictionary or a list containing nested dictionaries and lists. - parent_key (str): The key of the parent element in the recursion, used to track the referring fields. - - Returns: - list: A list of tuples, each containing a sensor ID and the referring field. - - Example: - nested_dict = { - "flex-model": [ - { - "sensor": 931, - "soc-at-start": 12.1, - "soc-unit": "kWh", - "soc-targets": [ - { - "value": 25, - "datetime": "2015-06-02T16:00:00+00:00" - }, - ], - "soc-minima": {"sensor": 300}, - "soc-min": 10, - "soc-max": 25, - "charging-efficiency": "120%", - "discharging-efficiency": {"sensor": 98}, - "storage-efficiency": 0.9999, - "power-capacity": "25kW", - "consumption-capacity": {"sensor": 42}, - "production-capacity": "30 kW" - }, - ], - } - - sensor_ids = find_sensor_ids(nested_dict) - print(sensor_ids) # Output: [(931, 'sensor'), (300, 'soc-minima.sensor'), (98, 'discharging-efficiency.sensor'), (42, 'consumption-capacity.sensor')] - """ - sensor_ids = [] - - if isinstance(data, dict): - for key, value in data.items(): - new_parent_key = f"{parent_key}.{key}" if parent_key else key - if key[-6:] == "sensor": - sensor_ids.append((value, new_parent_key)) - elif key[-7:] == "sensors": - for v in value: - sensor_ids.append((v, new_parent_key)) - else: - sensor_ids.extend(find_sensor_ids(value, new_parent_key)) - elif isinstance(data, list): - for index, item in enumerate(data): - new_parent_key = f"{parent_key}[{index}]" - sensor_ids.extend(find_sensor_ids(item, new_parent_key)) - - return sensor_ids From ee5e19ebac6b376046f850add82847299d6cbdbe Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:06 +0200 Subject: [PATCH 032/162] Revert "docs: add inline note explaining status code" This reverts commit b4ad01a0c721bde1cd45e465c8b552728e298b00. --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index e7fd1f40db..be7b67760f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -84,7 +84,7 @@ def test_trigger_schedule_with_invalid_flexmodel( ) print("Server responded with:\n%s" % trigger_schedule_response.json) check_deprecation(trigger_schedule_response, deprecation=None, sunset=None) - assert trigger_schedule_response.status_code == 422 # Unprocessable entity + assert trigger_schedule_response.status_code == 422 assert field in trigger_schedule_response.json["message"]["json"] if isinstance(trigger_schedule_response.json["message"]["json"], str): # ValueError From c2bf1afe97a95c6532b8b390d1338b18e28219d3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:08 +0200 Subject: [PATCH 033/162] Revert "feature: decorator supports custom error handler" This reverts commit 21ac09a0fe6b4d8c6005de5fdd3aff6ad7ee91f2. --- flexmeasures/auth/decorators.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 9dcf94ba3e..8c09260713 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -111,7 +111,6 @@ def permission_required_for_context( ctx_arg_name: str | None = None, ctx_loader: Callable | None = None, pass_ctx_to_loader: bool = False, - error_handler: Callable | None = None, ): """ This decorator can be used to make sure that the current user has the necessary permission to access the context. @@ -120,7 +119,6 @@ def permission_required_for_context( A 403 response is raised if there is no principal for the required permission. A 401 response is raised if the user is not authenticated at all. - A custom response can be generated by passing an error_handler, which should be a function that accepts the context, permission and ctx_arg_name. We will now explain how to load a context, and give an example: @@ -201,12 +199,7 @@ def decorated_view(*args, **kwargs): else: context = context_from_args - try: - check_access(context, permission) - except Exception as e: - if error_handler: - return error_handler(context, permission, ctx_arg_name) - raise e + check_access(context, permission) return fn(*args, **kwargs) From 529baa2a8da5b97195e594670b2b1b72947ec005 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:09 +0200 Subject: [PATCH 034/162] Revert "feature: allow checking permissions on optional fields" This reverts commit 25d11804b915b00fc148c31dbe5dcfd05619c0b3. --- flexmeasures/auth/decorators.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flexmeasures/auth/decorators.py b/flexmeasures/auth/decorators.py index 8c09260713..2655bb42a2 100644 --- a/flexmeasures/auth/decorators.py +++ b/flexmeasures/auth/decorators.py @@ -178,10 +178,7 @@ def decorated_view(*args, **kwargs): elif ctx_arg_pos is not None: context_from_args = args[ctx_arg_pos] elif ctx_arg_name is not None: - context_from_args = kwargs.get(ctx_arg_name) - # skip check in case (optional) argument was not passed - if context_from_args is None: - return fn(*args, **kwargs) + context_from_args = kwargs[ctx_arg_name] elif len(args) > 0: context_from_args = args[0] From 1665974f88d45278854a2a231c22a0dbbce45804 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:21:11 +0200 Subject: [PATCH 035/162] Revert "feature: check auth on sensors referenced in flex-context" This reverts commit 8a3e1120b167d1ebf55bcd7ab2b13361baa8109a. --- flexmeasures/api/conftest.py | 39 ++++++++++ flexmeasures/conftest.py | 36 --------- .../data/schemas/scheduling/__init__.py | 17 +---- flexmeasures/data/schemas/scheduling/utils.py | 24 ------ flexmeasures/data/schemas/tests/conftest.py | 75 +++---------------- .../data/schemas/tests/test_scheduling.py | 5 +- 6 files changed, 50 insertions(+), 146 deletions(-) create mode 100644 flexmeasures/api/conftest.py delete mode 100644 flexmeasures/data/schemas/scheduling/utils.py diff --git a/flexmeasures/api/conftest.py b/flexmeasures/api/conftest.py new file mode 100644 index 0000000000..277245588e --- /dev/null +++ b/flexmeasures/api/conftest.py @@ -0,0 +1,39 @@ +import pytest + +from flask_login import login_user, logout_user + +from flexmeasures.api.tests.utils import UserContext + + +@pytest.fixture +def requesting_user(request): + """Use this fixture to log in a user for the scope of a test. + + Sets the user by passing it an email address (see usage examples below), or pass None to get the AnonymousUser. + Passes the user object to the test. + Logs the user out after the test ran. + + Usage: + + >>> @pytest.mark.parametrize("requesting_user", ["test_prosumer_user_2@seita.nl", None], indirect=True) + + Or in combination with other parameters: + + @pytest.mark.parametrize( + "requesting_user, status_code", + [ + (None, 401), + ("test_prosumer_user_2@seita.nl", 200), + ], + indirect=["requesting_user"], + ) + + """ + email = request.param + if email is not None: + with UserContext(email) as user: + login_user(user) + yield user + logout_user() + else: + yield diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index fb04bcc438..d3bee45882 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -9,7 +9,6 @@ import pandas as pd import numpy as np from flask import request, jsonify -from flask_login import login_user, logout_user from flask_sqlalchemy import SQLAlchemy from flask_security import roles_accepted from pytest_mock import MockerFixture @@ -23,7 +22,6 @@ Gone, ) -from flexmeasures.api.tests.utils import UserContext from flexmeasures.app import create as create_app from flexmeasures.auth.policy import ADMIN_ROLE, ADMIN_READER_ROLE from flexmeasures.data.services.users import create_user @@ -1364,37 +1362,3 @@ def add_beliefs( @pytest.fixture def mock_get_status(mocker: MockerFixture): return mocker.patch("flexmeasures.data.services.sensors.get_status", autospec=True) - - -@pytest.fixture -def requesting_user(request): - """Use this fixture to log in a user for the scope of a test. - - Sets the user by passing it an email address (see usage examples below), or pass None to get the AnonymousUser. - Passes the user object to the test. - Logs the user out after the test ran. - - Usage: - - >>> @pytest.mark.parametrize("requesting_user", ["test_prosumer_user_2@seita.nl", None], indirect=True) - - Or in combination with other parameters: - - @pytest.mark.parametrize( - "requesting_user, status_code", - [ - (None, 401), - ("test_prosumer_user_2@seita.nl", 200), - ], - indirect=["requesting_user"], - ) - - """ - email = request.param - if email is not None: - with UserContext(email) as user: - login_user(user) - yield user - logout_user() - else: - yield diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ae1d99a0f5..0f03bbb712 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,8 +1,5 @@ -from marshmallow import Schema, fields, validate, validates_schema, ValidationError -from werkzeug.exceptions import Forbidden +from marshmallow import Schema, fields, validate -from flexmeasures.auth.policy import check_access -from flexmeasures.data.schemas.scheduling.utils import find_sensors from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField @@ -32,15 +29,3 @@ class FlexContextSchema(Schema): inflexible_device_sensors = fields.List( SensorIdField(), data_key="inflexible-device-sensors" ) - - @validates_schema - def check_read_access_on_sensors(self, data: dict, **kwargs): - sensors = find_sensors(data) - for sensor, field_name in sensors: - try: - check_access(context=sensor, permission="read") - except Forbidden: - raise ValidationError( - message=f"User has no read access to sensor {sensor.id}.", - field_name=self.fields[field_name].data_key, - ) diff --git a/flexmeasures/data/schemas/scheduling/utils.py b/flexmeasures/data/schemas/scheduling/utils.py deleted file mode 100644 index 4655a4a04a..0000000000 --- a/flexmeasures/data/schemas/scheduling/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - -from flexmeasures import Sensor - - -def find_sensors(data, parent_key='', path='') -> list[tuple[Sensor, str]]: - """Recursively find all sensors in a nested dictionary or list along with the fields referring to them.""" - sensors = [] - - if isinstance(data, dict): - for key, value in data.items(): - new_parent_key = f"{parent_key}.{key}" if parent_key else key - new_path = f"{path}.{key}" if path else key - if isinstance(value, Sensor): - sensors.append((value, f"{new_parent_key}{path}")) - else: - sensors.extend(find_sensors(value, new_parent_key, new_path)) - elif isinstance(data, list): - for index, item in enumerate(data): - new_parent_key = f"{parent_key}[{index}]" - new_path = f"{path}[{index}]" - sensors.extend(find_sensors(item, new_parent_key, new_path)) - - return sensors diff --git a/flexmeasures/data/schemas/tests/conftest.py b/flexmeasures/data/schemas/tests/conftest.py index d393dd6dae..0754f0f6e6 100644 --- a/flexmeasures/data/schemas/tests/conftest.py +++ b/flexmeasures/data/schemas/tests/conftest.py @@ -1,80 +1,25 @@ import pytest from datetime import timedelta -from flask_security import SQLAlchemySessionUserDatastore, hash_password - -from flexmeasures import Sensor, User, UserRole -from flexmeasures.data.models.time_series import TimedBelief -from flexmeasures.data.models.user import Account +from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @pytest.fixture(scope="module") -def dummy_accounts(db, app): - dummy_account_1 = Account(name="dummy account 1") - db.session.add(dummy_account_1) - - dummy_account_2 = Account(name="dummy account 2") - db.session.add(dummy_account_2) - - # Assign account IDs - db.session.flush() - - return { - dummy_account_1.name: dummy_account_1, - dummy_account_2.name: dummy_account_2, - } - - -@pytest.fixture(scope="module") -def dummy_user(db, app, dummy_accounts): - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, UserRole) - user = user_datastore.create_user( - username="dummy user", - email="dummy_user@seita.nl", - password=hash_password("testtest"), - account_id=dummy_accounts["dummy account 1"].id, - active=True, - ) - return user - - -@pytest.fixture(scope="module") -def dummy_assets(db, app, dummy_accounts): +def dummy_asset(db, app): dummy_asset_type = GenericAssetType(name="DummyGenericAssetType") db.session.add(dummy_asset_type) - dummy_asset_1 = GenericAsset( - name="dummy asset 1", - generic_asset_type=dummy_asset_type, - owner=dummy_accounts["dummy account 1"], - ) - db.session.add(dummy_asset_1) - - dummy_asset_2 = GenericAsset( - name="dummy asset 2", - generic_asset_type=dummy_asset_type, - owner=dummy_accounts["dummy account 1"], - ) - db.session.add(dummy_asset_2) - - dummy_asset_3 = GenericAsset( - name="dummy asset 3", - generic_asset_type=dummy_asset_type, - owner=dummy_accounts["dummy account 2"], + _dummy_asset = GenericAsset( + name="DummyGenericAsset", generic_asset_type=dummy_asset_type ) - db.session.add(dummy_asset_3) + db.session.add(_dummy_asset) - return { - dummy_asset_1.name: dummy_asset_1, - dummy_asset_2.name: dummy_asset_2, - dummy_asset_3.name: dummy_asset_3, - } + return _dummy_asset @pytest.fixture(scope="module") -def setup_dummy_sensors(db, app, dummy_assets): - dummy_asset = dummy_assets["dummy asset 1"] +def setup_dummy_sensors(db, app, dummy_asset): sensor1 = Sensor( "sensor 1", generic_asset=dummy_asset, @@ -113,8 +58,7 @@ def setup_dummy_sensors(db, app, dummy_assets): @pytest.fixture(scope="module") -def setup_efficiency_sensors(db, app, dummy_assets): - dummy_asset = dummy_assets["dummy asset 1"] +def setup_efficiency_sensors(db, app, dummy_asset): sensor = Sensor( "efficiency", generic_asset=dummy_asset, @@ -128,8 +72,7 @@ def setup_efficiency_sensors(db, app, dummy_assets): @pytest.fixture(scope="module") -def setup_site_capacity_sensor(db, app, dummy_assets, setup_sources): - dummy_asset = dummy_assets["dummy asset 1"] +def setup_site_capacity_sensor(db, app, dummy_asset, setup_sources): sensor = Sensor( "site-power-capacity", generic_asset=dummy_asset, diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 0eb7aab20a..2d18e85ef5 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -239,10 +239,7 @@ def load_schema(): ), ], ) -@pytest.mark.parametrize( - "requesting_user", ["dummy_user@seita.nl"], indirect=True -) -def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context, fails, dummy_user, requesting_user): +def test_flex_context_schema(db, app, setup_site_capacity_sensor, flex_context, fails): schema = FlexContextSchema() # Replace sensor name with sensor ID From 8678a0dcae5254a6d74954b6728e909e082d912b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:36:15 +0200 Subject: [PATCH 036/162] docs: add inline note explaining permission decorator Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index e1b52fa641..2b827e79be 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -348,6 +348,8 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): }, location="json", ) + # Simplification of checking for create-children access on each of the flexible sensors, + # which assumes each of the flexible sensors belongs to the given asset. @permission_required_for_context("create-children", ctx_arg_name="asset") def trigger_schedule( self, From f42ceaf1b518cd97f08b59e84f824efa18618055 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:37:55 +0200 Subject: [PATCH 037/162] fix: remove unused parameter (does not need to be formally deprecated on a newly introduced method) Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 5ee73923f3..508012ae9d 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -257,7 +257,6 @@ def cb_done_sequential_scheduling_job(jobs_ids: list[str]): def create_sequential_scheduling_job( asset_or_sensor: Asset | Sensor | None = None, - sensor: Sensor | None = None, job_id: str | None = None, enqueue: bool = True, requeue: bool = False, From 7b5b3ce18c76761675c958a77eafc6d9567f2cda Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 24 May 2024 13:50:39 +0200 Subject: [PATCH 038/162] feature: check that each flexible device power sensor lives under the asset that is being scheduled Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 20 +++++++++++++++++++- flexmeasures/data/services/scheduling.py | 8 +++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index d344beb3b1..0ddfa4085b 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -166,9 +166,17 @@ class Meta: class SensorIdField(MarshmallowClickMixin, fields.Int): """Field that deserializes to a Sensor and serializes back to an integer.""" - def __init__(self, *args, unit: str | ur.Quantity | None = None, **kwargs): + def __init__( + self, + asset: GenericAsset | None = None, + unit: str | ur.Quantity | None = None, + *args, + **kwargs, + ): super().__init__(*args, **kwargs) + self.asset = asset + if isinstance(unit, str): self.to_unit = ur.Quantity(unit) elif isinstance(unit, ur.Quantity): @@ -187,6 +195,16 @@ def _deserialize(self, value: int, attr, obj, **kwargs) -> Sensor: sensor.generic_asset sensor.generic_asset.generic_asset_type + # if the asset is defined, check if the sensor belongs to it (or to its offspring) + if ( + self.asset is not None + and sensor.generic_asset != self.asset + and sensor.generic_asset not in self.asset.offspring + ): + raise FMValidationError( + f"Sensor {value} must be assigned to asset {self.asset} (or to one of its offspring)" + ) + # if the units are defined, check if the sensor data is convertible to the target units if self.to_unit is not None and not units_are_convertible( sensor.unit, str(self.to_unit.units) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 508012ae9d..139c284bfa 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -276,9 +276,11 @@ def create_sequential_scheduling_job( f"Missing 'sensor' in flex-model list item: {child_flex_model}." ) - sensor = SensorIdField().deserialize(sensor_id) - - # todo make sure each sensor lives under the asset + # get the sensor, while ensuring that it lives under the asset + if isinstance(asset_or_sensor, Asset): + sensor = SensorIdField(asset=asset_or_sensor).deserialize(sensor_id) + else: + sensor = SensorIdField().deserialize(sensor_id) current_scheduler_kwargs = deepcopy(scheduler_kwargs) From 378b0a73f60278aa671874471df81a65fe4568a9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 3 Jun 2024 14:25:00 +0200 Subject: [PATCH 039/162] docs: changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 231a9fd8ad..9f5bb080b9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,7 +8,7 @@ v0.22.0 | June XX, 2024 New features ------------- - +* New API endpoint `[POST] /assets/(id)/schedules/trigger `_ to schedule a site with multiple flexible devices [see `PR #1065 `_] Infrastructure / Support ---------------------- From 754502a51e34a155e762881eea062ea2d6688b26 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 3 Jun 2024 14:26:33 +0200 Subject: [PATCH 040/162] docs: API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 3f32d6313a..3c8a4ac94f 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,11 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-19 | 2024-06-03 +"""""""""""""""""""" +- New API endpoint `[POST] /assets/(id)/schedules/trigger `_ to schedule a site with multiple flexible devices. + + v3.0-18 | 2024-03-07 """""""""""""""""""" - Add support for providing a sensor definition to the ``soc-minima``, ``soc-maxima`` and ``soc-targets`` flex-model fields for `/sensors//schedules/trigger` (POST). From e1e1019526761dea27b57df424bf1e735e99a8a4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 4 Jun 2024 15:23:07 +0200 Subject: [PATCH 041/162] feature: allow loading API data from three places, and avoid clashing keys Signed-off-by: F.N. Claessen --- flexmeasures/api/common/utils/args_parsing.py | 22 ++++++++++++++++--- flexmeasures/api/v3_0/assets.py | 7 ++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/common/utils/args_parsing.py b/flexmeasures/api/common/utils/args_parsing.py index 32383ff7f3..4c598130b8 100644 --- a/flexmeasures/api/common/utils/args_parsing.py +++ b/flexmeasures/api/common/utils/args_parsing.py @@ -38,10 +38,26 @@ def validation_error_handler(error: FMValidationError): @parser.location_loader("args_and_json") def load_data(request, schema): """ - We allow parameters to come from either GET args or POST JSON, - as validators can be attached to either. + We allow parameters to come from URL path, GET args and POST JSON, + as validators can be attached to any of them. """ + + # GET args (i.e. query parameters, such as https://flexmeasures.io/?id=5) newdata = request.args.copy() + + # View args (i.e. path parameters, such as the `/assets/` endpoint) + path_params = request.view_args + # Avoid clashes such as visiting https://flexmeasures.io/assets/4/?id=5 on the /assets/ endpoint + for key in path_params: + if key in newdata: + raise FMValidationError(message=f"{key} already set in the URL path") + newdata.update(path_params) + if request.mimetype == "application/json" and request.method == "POST": - newdata.update(request.get_json()) + json_params = request.get_json() + # Avoid clashes + for key in json_params: + if key in newdata: + raise FMValidationError(message=f"{key} already set in the URL path or query parameters") + newdata.update(json_params) return MultiDictProxy(newdata, schema) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 2b827e79be..8ac8605d44 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -330,12 +330,9 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//schedules/trigger", methods=["POST"]) - @use_kwargs( - {"asset": AssetIdField(data_key="id")}, - location="path", - ) @use_kwargs( { + "asset": AssetIdField(data_key="id"), "start_of_schedule": AwareDateTimeField( data_key="start", format="iso", required=True ), @@ -346,7 +343,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): "flex_model": fields.Dict(data_key="flex-model"), "flex_context": fields.Dict(required=False, data_key="flex-context"), }, - location="json", + location="args_and_json", ) # Simplification of checking for create-children access on each of the flexible sensors, # which assumes each of the flexible sensors belongs to the given asset. From 4f86bb146049e17c9b8fd279508ffd28a409deca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 4 Jun 2024 17:12:20 +0200 Subject: [PATCH 042/162] docs: typos Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index ad183cad81..537a7577a0 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -32,7 +32,7 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil "id": sensors["Test EV"].id, "class": "Sensor", } - # It uses the inflxible-device-sensors that are defined in the flex-conctext, exclusively. + # It uses the inflexible-device-sensors that are defined in the flex-context, exclusively. assert jobs[0].kwargs["flex_context"]["inflexible-device-sensors"] == [ sensors["Test Solar"].id, sensors["Test Building"].id, From de811ec1650056233b0508222d286385ae684f8a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 09:51:08 +0200 Subject: [PATCH 043/162] refactor: parameterize test cases Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_assets_api.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 13192545f2..46051a8f90 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -33,35 +33,37 @@ def test_get_assets_badauth(client, setup_api_test_data, requesting_user, status assert get_assets_response.status_code == status_code +@pytest.mark.parametrize( + ("whose_asset", "exp_status"), + [ + # okay to look at assets in own account + ("test_supplier_user_4@seita.nl", 200), + # not okay to see assets owned by other accounts + ("test_prosumer_user@seita.nl", 403), + # proper 404 for non-existing asset + (None, 404), + ], +) @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) -def test_get_asset_nonaccount_access(client, setup_api_test_data, requesting_user): +def test_get_asset_nonaccount_access( + client, setup_api_test_data, whose_asset, exp_status, requesting_user +): """Without being on the same account, test correct responses when accessing one asset.""" - with UserContext("test_prosumer_user@seita.nl") as prosumer1: - prosumer1_assets = prosumer1.account.generic_assets - with UserContext("test_supplier_user_4@seita.nl") as supplieruser4: - supplieruser4_assets = supplieruser4.account.generic_assets + if isinstance(whose_asset, str): + with UserContext(whose_asset) as owner: + asset_id = owner.account.generic_assets[0].id + else: + asset_id = 8171766575 # non-existent asset ID - # okay to look at assets in own account - asset_response = client.get( - url_for("AssetAPI:fetch_one", id=supplieruser4_assets[0].id), - follow_redirects=True, - ) - assert asset_response.status_code == 200 - # not okay to see assets owned by other accounts - asset_response = client.get( - url_for("AssetAPI:fetch_one", id=prosumer1_assets[0].id), - follow_redirects=True, - ) - assert asset_response.status_code == 403 - # proper 404 for non-existing asset asset_response = client.get( - url_for("AssetAPI:fetch_one", id=8171766575), + url_for("AssetAPI:fetch_one", id=asset_id), follow_redirects=True, ) - assert asset_response.status_code == 404 - assert "not found" in asset_response.json["message"] + assert asset_response.status_code == exp_status + if exp_status == 404: + assert "not found" in asset_response.json["message"] @pytest.mark.parametrize( From 175cd186d9a6f56ea87818202a6a234ee7b2105f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 10:21:20 +0200 Subject: [PATCH 044/162] refactor: merge AssetIdField and GenericAssetIDField Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 1 + flexmeasures/api/v3_0/assets.py | 25 +++++++++++++------ .../api/v3_0/tests/test_assets_api.py | 2 +- flexmeasures/data/schemas/generic_assets.py | 21 ++++++++++++---- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 3c8a4ac94f..511ee6dc36 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,6 +9,7 @@ API change log v3.0-19 | 2024-06-03 """""""""""""""""""" - New API endpoint `[POST] /assets/(id)/schedules/trigger `_ to schedule a site with multiple flexible devices. +- Updated message for 404 Not Found on endpoints for managing assets: `/assets` (GET, POST) and `/assets/` (GET, PATCH, DELETE). v3.0-18 | 2024-03-07 diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 8ac8605d44..88482f8dda 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -16,11 +16,13 @@ from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField -from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema +from flexmeasures.data.schemas.generic_assets import ( + GenericAssetSchema as AssetSchema, + GenericAssetIdField as AssetIdField, +) from flexmeasures.data.services.scheduling import ( create_sequential_scheduling_job, ) -from flexmeasures.api.common.schemas.generic_assets import AssetIdField from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.responses import ( invalid_flex_config, @@ -156,7 +158,9 @@ def post(self, asset_data: dict): return asset_schema.dump(asset), 201 @route("/", methods=["GET"]) - @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") + @use_kwargs( + {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path" + ) @permission_required_for_context("read", ctx_arg_name="asset") @as_json def fetch_one(self, id, asset): @@ -192,7 +196,10 @@ def fetch_one(self, id, asset): @route("/", methods=["PATCH"]) @use_args(partial_asset_schema) - @use_kwargs({"db_asset": AssetIdField(data_key="id")}, location="path") + @use_kwargs( + {"db_asset": AssetIdField(data_key="id", status_if_not_found=404)}, + location="path", + ) @permission_required_for_context("update", ctx_arg_name="db_asset") @as_json def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): @@ -248,7 +255,9 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): return asset_schema.dump(db_asset), 200 @route("/", methods=["DELETE"]) - @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path") + @use_kwargs( + {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path" + ) @permission_required_for_context("delete", ctx_arg_name="asset") @as_json def delete(self, id: int, asset: GenericAsset): @@ -275,7 +284,7 @@ def delete(self, id: int, asset: GenericAsset): @route("//chart", strict_slashes=False) # strict on next version? see #1014 @use_kwargs( - {"asset": AssetIdField(data_key="id")}, + {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path", ) @use_kwargs( @@ -305,7 +314,7 @@ def get_chart(self, id: int, asset: GenericAsset, **kwargs): "//chart_data", strict_slashes=False ) # strict on next version? see #1014 @use_kwargs( - {"asset": AssetIdField(data_key="id")}, + {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path", ) @use_kwargs( @@ -332,7 +341,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): @route("//schedules/trigger", methods=["POST"]) @use_kwargs( { - "asset": AssetIdField(data_key="id"), + "asset": AssetIdField(data_key="id", status_if_not_found=404), "start_of_schedule": AwareDateTimeField( data_key="start", format="iso", required=True ), diff --git a/flexmeasures/api/v3_0/tests/test_assets_api.py b/flexmeasures/api/v3_0/tests/test_assets_api.py index 46051a8f90..0c1dbeceeb 100644 --- a/flexmeasures/api/v3_0/tests/test_assets_api.py +++ b/flexmeasures/api/v3_0/tests/test_assets_api.py @@ -63,7 +63,7 @@ def test_get_asset_nonaccount_access( ) assert asset_response.status_code == exp_status if exp_status == 404: - assert "not found" in asset_response.json["message"] + assert asset_response.json["message"] == "No asset found with ID 8171766575." @pytest.mark.parametrize( diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 745020556d..211b751fff 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -2,6 +2,7 @@ import json +from flask import abort from marshmallow import validates, ValidationError, fields, validates_schema from flask_security import current_user from sqlalchemy import select @@ -146,16 +147,26 @@ class Meta: class GenericAssetIdField(MarshmallowClickMixin, fields.Int): """Field that deserializes to a GenericAsset and serializes back to an integer.""" - @with_appcontext_if_needed() - def _deserialize(self, value, attr, obj, **kwargs) -> GenericAsset: + def __init__(self, status_if_not_found: int | None = None, *args, **kwargs): + self.status_if_not_found = status_if_not_found + super().__init__(*args, **kwargs) + + def _deserialize(self, value: int, attr, obj, **kwargs) -> GenericAsset: """Turn a generic asset id into a GenericAsset.""" - generic_asset = db.session.get(GenericAsset, value) + generic_asset: GenericAsset = db.session.execute( + select(GenericAsset).filter_by(id=int(value)) + ).scalar_one_or_none() if generic_asset is None: - raise FMValidationError(f"No asset found with id {value}.") + message = f"No asset found with ID {value}." + if self.status_if_not_found == 404: + raise abort(404, message) + else: + raise FMValidationError(message) + # lazy loading now (asset is somehow not in session after this) generic_asset.generic_asset_type return generic_asset - def _serialize(self, asset, attr, data, **kwargs): + def _serialize(self, asset: GenericAsset, attr, data, **kwargs) -> int: """Turn a GenericAsset into a generic asset id.""" return asset.id From ffe82b1182b974bb7f660c5a14b442f033784559 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 10:28:20 +0200 Subject: [PATCH 045/162] refactor: move to AssetTriggerSchema Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 19 +++---------------- .../data/schemas/scheduling/__init__.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 88482f8dda..4be917f816 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -15,11 +15,12 @@ from flexmeasures.data import db from flexmeasures.data.models.user import Account from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField from flexmeasures.data.schemas.generic_assets import ( GenericAssetSchema as AssetSchema, GenericAssetIdField as AssetIdField, ) +from flexmeasures.data.schemas.scheduling import AssetTriggerSchema +from flexmeasures.data.schemas.times import AwareDateTimeField from flexmeasures.data.services.scheduling import ( create_sequential_scheduling_job, ) @@ -339,21 +340,7 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs): return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs) @route("//schedules/trigger", methods=["POST"]) - @use_kwargs( - { - "asset": AssetIdField(data_key="id", status_if_not_found=404), - "start_of_schedule": AwareDateTimeField( - data_key="start", format="iso", required=True - ), - "belief_time": AwareDateTimeField(format="iso", data_key="prior"), - "duration": PlanningDurationField( - load_default=PlanningDurationField.load_default - ), - "flex_model": fields.Dict(data_key="flex-model"), - "flex_context": fields.Dict(required=False, data_key="flex-context"), - }, - location="args_and_json", - ) + @use_args(AssetTriggerSchema(), location="args_and_json", as_kwargs=True) # Simplification of checking for create-children access on each of the flexible sensors, # which assumes each of the flexible sensors belongs to the given asset. @permission_required_for_context("create-children", ctx_arg_name="asset") diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 0f03bbb712..5bbe8793be 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,6 +1,8 @@ from marshmallow import Schema, fields, validate +from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField +from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField class FlexContextSchema(Schema): @@ -29,3 +31,16 @@ class FlexContextSchema(Schema): inflexible_device_sensors = fields.List( SensorIdField(), data_key="inflexible-device-sensors" ) + + +class AssetTriggerSchema(Schema): + asset = GenericAssetIdField(data_key="id", status_if_not_found=404) + start_of_schedule = AwareDateTimeField( + data_key="start", format="iso", required=True + ) + belief_time = AwareDateTimeField(format="iso", data_key="prior") + duration = PlanningDurationField( + load_default=PlanningDurationField.load_default + ) + flex_model = fields.Dict(data_key="flex-model") + flex_context = fields.Dict(required=False, data_key="flex-context") From 45592c71c8b8ac960edec25299ccd0430ab0f5b5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 10:43:18 +0200 Subject: [PATCH 046/162] remove Signed-off-by: F.N. Claessen --- .../api/common/schemas/generic_assets.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 flexmeasures/api/common/schemas/generic_assets.py diff --git a/flexmeasures/api/common/schemas/generic_assets.py b/flexmeasures/api/common/schemas/generic_assets.py deleted file mode 100644 index dfda954c56..0000000000 --- a/flexmeasures/api/common/schemas/generic_assets.py +++ /dev/null @@ -1,23 +0,0 @@ -from flask import abort -from marshmallow import fields -from sqlalchemy import select - -from flexmeasures.data import db -from flexmeasures.data.models.generic_assets import GenericAsset - - -class AssetIdField(fields.Integer): - """ - Field that represents a generic asset ID. It de-serializes from the asset id to an asset instance. - """ - - def _deserialize(self, asset_id: int, attr, obj, **kwargs) -> GenericAsset: - asset: GenericAsset = db.session.execute( - select(GenericAsset).filter_by(id=int(asset_id)) - ).scalar_one_or_none() - if asset is None: - raise abort(404, f"GenericAsset {asset_id} not found") - return asset - - def _serialize(self, asset: GenericAsset, attr, data, **kwargs) -> int: - return asset.id From 99bd8daa22cb744ac11cd26ba1ba2d53d5de18b9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 10:46:36 +0200 Subject: [PATCH 047/162] remove: seemingly obsolete workaround Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 211b751fff..62f497e802 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -163,8 +163,6 @@ def _deserialize(self, value: int, attr, obj, **kwargs) -> GenericAsset: else: raise FMValidationError(message) - # lazy loading now (asset is somehow not in session after this) - generic_asset.generic_asset_type return generic_asset def _serialize(self, asset: GenericAsset, attr, data, **kwargs) -> int: From 44cfd0cc6ff312872af14cea116f3d20cf6d801e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:31:41 +0200 Subject: [PATCH 048/162] feature: validate flex-model sensors belong to asset Signed-off-by: F.N. Claessen --- .../data/schemas/scheduling/__init__.py | 32 ++++++++++++++++--- flexmeasures/data/services/scheduling.py | 12 +------ flexmeasures/data/tests/conftest.py | 4 +-- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 5bbe8793be..79fe10fc90 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,8 +1,9 @@ -from marshmallow import Schema, fields, validate +from marshmallow import fields, validate, validates_schema, INCLUDE, Schema from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField from flexmeasures.data.schemas.times import AwareDateTimeField, PlanningDurationField +from flexmeasures.data.schemas.utils import FMValidationError class FlexContextSchema(Schema): @@ -33,14 +34,37 @@ class FlexContextSchema(Schema): ) +class SequentialFlexModelSchema(Schema): + sensor = SensorIdField(required=True) + + class AssetTriggerSchema(Schema): asset = GenericAssetIdField(data_key="id", status_if_not_found=404) start_of_schedule = AwareDateTimeField( data_key="start", format="iso", required=True ) belief_time = AwareDateTimeField(format="iso", data_key="prior") - duration = PlanningDurationField( - load_default=PlanningDurationField.load_default + duration = PlanningDurationField(load_default=PlanningDurationField.load_default) + flex_model = fields.List( + fields.Nested(SequentialFlexModelSchema(unknown=INCLUDE)), + data_key="flex-model", ) - flex_model = fields.Dict(data_key="flex-model") flex_context = fields.Dict(required=False, data_key="flex-context") + + @validates_schema + def check_flex_model_sensors(self, data, **kwargs): + """Verify that the flex-model's sensors live under the asset for which a schedule is triggered.""" + asset = data["asset"] + sensors = [] + for sensor_flex_model in data["flex_model"]: + sensor = sensor_flex_model["sensor"] + if sensor in sensors: + raise FMValidationError( + f"Sensor {sensor_flex_model['sensor'].id} should not occur more than once in the flex-model" + ) + if sensor.generic_asset not in [asset] + asset.offspring: + raise FMValidationError( + f"Sensor {sensor_flex_model['sensor'].id} does not belong to asset {asset.id} (or to one of its offspring)" + ) + sensors.append(sensor) + return data diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 139c284bfa..9b50822f2a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -270,17 +270,7 @@ def create_sequential_scheduling_job( previous_sensors = [] previous_job = depends_on for child_flex_model in flex_model: - sensor_id = child_flex_model.pop("sensor") - if sensor_id is None: - raise ValidationError( - f"Missing 'sensor' in flex-model list item: {child_flex_model}." - ) - - # get the sensor, while ensuring that it lives under the asset - if isinstance(asset_or_sensor, Asset): - sensor = SensorIdField(asset=asset_or_sensor).deserialize(sensor_id) - else: - sensor = SensorIdField().deserialize(sensor_id) + sensor = child_flex_model.pop("sensor") current_scheduler_kwargs = deepcopy(scheduler_kwargs) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 5f78d54faa..106bbddc17 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -326,7 +326,7 @@ def flex_description_sequential(smart_building, setup_markets): return { "flex_model": [ { - "sensor": sensors["Test EV"].id, + "sensor": sensors["Test EV"], "power-capacity": "10kW", "soc-at-start": 0.01, # 10 kWh "soc-unit": "MWh", @@ -334,7 +334,7 @@ def flex_description_sequential(smart_building, setup_markets): "soc-max": 0.05, # 50 kWh }, { - "sensor": sensors["Test Battery"].id, + "sensor": sensors["Test Battery"], "power-capacity": "20kW", "soc-at-start": 0.01, # 10 kWh "soc-unit": "MWh", From ee08883a10ebb51d4c70f9a10102f67fbfa44882 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:42:13 +0200 Subject: [PATCH 049/162] feature: more clearly separate the serialized and deserialized parts of the flex model Signed-off-by: F.N. Claessen --- .../data/schemas/scheduling/__init__.py | 17 +++++++++++-- flexmeasures/data/services/scheduling.py | 2 +- flexmeasures/data/tests/conftest.py | 24 +++++++++++-------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 79fe10fc90..d7bc39ae85 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,4 +1,4 @@ -from marshmallow import fields, validate, validates_schema, INCLUDE, Schema +from marshmallow import fields, pre_load, validate, validates_schema, Schema from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField @@ -36,6 +36,19 @@ class FlexContextSchema(Schema): class SequentialFlexModelSchema(Schema): sensor = SensorIdField(required=True) + sensor_flex_model = fields.Dict(data_key="sensor-flex-model") + + @pre_load + def unwrap_envelope(self, data, **kwargs): + """Any field other than 'sensor' becomes part of the sensor's flex-model.""" + extra = {} + rest = {} + for k, v in data.items(): + if k not in self.fields: + extra[k] = v + else: + rest[k] = v + return {"sensor-flex-model": extra, **rest} class AssetTriggerSchema(Schema): @@ -46,7 +59,7 @@ class AssetTriggerSchema(Schema): belief_time = AwareDateTimeField(format="iso", data_key="prior") duration = PlanningDurationField(load_default=PlanningDurationField.load_default) flex_model = fields.List( - fields.Nested(SequentialFlexModelSchema(unknown=INCLUDE)), + fields.Nested(SequentialFlexModelSchema()), data_key="flex-model", ) flex_context = fields.Dict(required=False, data_key="flex-context") diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 9b50822f2a..e552a993db 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -274,7 +274,7 @@ def create_sequential_scheduling_job( current_scheduler_kwargs = deepcopy(scheduler_kwargs) - current_scheduler_kwargs["flex_model"] = child_flex_model + current_scheduler_kwargs["flex_model"] = child_flex_model["sensor_flex_model"] current_scheduler_kwargs["flex_context"]["inflexible-device-sensors"].extend( previous_sensors ) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 106bbddc17..da94b13a5a 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -327,19 +327,23 @@ def flex_description_sequential(smart_building, setup_markets): "flex_model": [ { "sensor": sensors["Test EV"], - "power-capacity": "10kW", - "soc-at-start": 0.01, # 10 kWh - "soc-unit": "MWh", - "soc-min": 0.0, - "soc-max": 0.05, # 50 kWh + "sensor_flex_model": { + "power-capacity": "10kW", + "soc-at-start": 0.01, # 10 kWh + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.05, # 50 kWh + } }, { "sensor": sensors["Test Battery"], - "power-capacity": "20kW", - "soc-at-start": 0.01, # 10 kWh - "soc-unit": "MWh", - "soc-min": 0.0, - "soc-max": 0.1, # 100 kWh + "sensor_flex_model": { + "power-capacity": "20kW", + "soc-at-start": 0.01, # 10 kWh + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.1, # 100 kWh + } }, ], "flex_context": { From a9ddce830e3b02d9fb53ae5414b8f21d1fc56cb7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:44:03 +0200 Subject: [PATCH 050/162] feature: add end-to-end test for triggering and getting a schedule Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py new file mode 100644 index 0000000000..334c0cec38 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -0,0 +1,115 @@ +from flask import url_for +import pytest +from isodate import parse_datetime, parse_duration + +from rq.job import Job + +from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.tests.utils import work_on_rq +from flexmeasures.data.services.scheduling import ( + handle_scheduling_exception, + get_data_source_for_job, +) + + +@pytest.mark.parametrize( + "message, asset_name", + [ + (message_for_trigger_schedule(), "Test battery"), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_asset_trigger_and_get_schedule( + app, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + add_charging_station_assets_fresh_db, + keep_scheduling_queue_empty, + message, + asset_name, + requesting_user, +): # noqa: C901 + # Include the price sensor and site-power-capacity in the flex-context explicitly, to test deserialization + price_sensor_id = add_market_prices_fresh_db["epex_da"].id + message["flex-context"] = { + "consumption-price-sensor": price_sensor_id, + "production-price-sensor": price_sensor_id, + "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities + } + + # trigger a schedule through the /assets//schedules/trigger [POST] api endpoint + assert len(app.queues["scheduling"]) == 0 + + sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset) + .filter(GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == asset_name) + .one_or_none() + ) + message["flex-model"]["sensor"] = sensor.id + message["flex-model"] = [ + message["flex-model"], + ] + + with app.test_client() as client: + print(message) + print(message["flex-model"]) + trigger_schedule_response = client.post( + url_for("AssetAPI:trigger_schedule", id=sensor.generic_asset.id), + json=message, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + assert trigger_schedule_response.status_code == 200 + job_id = trigger_schedule_response.json["schedule"] + + # look for scheduling jobs in queue + scheduled_jobs = app.queues["scheduling"].jobs + deferred_job_ids = app.queues["scheduling"].deferred_job_registry.get_job_ids() + assert len(scheduled_jobs) == len( + message["flex-model"] + ), "a scheduling job should be made for each sensor flex model" + assert ( + len(deferred_job_ids) == 1 + ), "only 1 job should be triggered when the last scheduling job is done" + scheduling_job = scheduled_jobs[0] + done_job_id = deferred_job_ids[0] + print(scheduling_job.kwargs) + assert scheduling_job.kwargs["asset_or_sensor"]["id"] == sensor.id + assert scheduling_job.kwargs["start"] == parse_datetime(message["start"]) + assert done_job_id == job_id + + # process the scheduling queue + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + assert ( + Job.fetch(job_id, connection=app.queues["scheduling"].connection).is_finished + is True + ) + + # Derive some expectations from the POSTed message + resolution = sensor.event_resolution + expected_length_of_schedule = parse_duration(message["duration"]) / resolution + + # check results are in the database + + # First, make sure the scheduler data source is now there + scheduling_job.refresh() # catch meta info that was added on this very instance + scheduler_source = get_data_source_for_job(scheduling_job) + assert scheduler_source is not None + + # try to retrieve the schedule through the /sensors//schedules/ [GET] api endpoint + get_schedule_response = client.get( + url_for( + "SensorAPI:get_schedule", id=sensor.id, uuid=scheduling_job.id + ), # todo: use (last?) job_id from trigger response + query_string={"duration": "PT48H"}, + ) + print("Server responded with:\n%s" % get_schedule_response.json) + assert get_schedule_response.status_code == 200 + # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" + assert len(get_schedule_response.json["values"]) == expected_length_of_schedule From f340a9530b91ce83567a1d9cc4ae8110b580004d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:45:20 +0200 Subject: [PATCH 051/162] docs: add fixture docstring explaining partial deserialization Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index da94b13a5a..8a7071144a 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -321,6 +321,10 @@ def smart_building(app, db, smart_building_types): @pytest.fixture def flex_description_sequential(smart_building, setup_markets): + """This is a partially deserialized flex model. + + Specifically, the main flex model is deserialized, while the sensors' individual flex models are still serialized. + """ assets, sensors = smart_building return { From 4c6c52ad5b87ae64716d7cbc2233d7768a04e20b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:45:33 +0200 Subject: [PATCH 052/162] style: black Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 537a7577a0..4ba23e145f 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -22,7 +22,7 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil asset_or_sensor=assets["Test Site"], scheduler_specs=scheduler_specs, enqueue=False, - **flex_description_sequential + **flex_description_sequential, ) assert len(jobs) == 3 From b328921928724420698b674ae3e541acca0ccf11 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:46:05 +0200 Subject: [PATCH 053/162] fix: in case no inflexible-device-sensors were part of the flex-context Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index e552a993db..be8be28b84 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -275,6 +275,8 @@ def create_sequential_scheduling_job( current_scheduler_kwargs = deepcopy(scheduler_kwargs) current_scheduler_kwargs["flex_model"] = child_flex_model["sensor_flex_model"] + if "inflexible-device-sensors" not in current_scheduler_kwargs["flex_context"]: + current_scheduler_kwargs["flex_context"]["inflexible-device-sensors"] = [] current_scheduler_kwargs["flex_context"]["inflexible-device-sensors"].extend( previous_sensors ) From b40d40e4ac32ebd8014ea083a382a52399c1a471 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:46:20 +0200 Subject: [PATCH 054/162] docs: fix inline comment Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index be8be28b84..f459e11b46 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -296,7 +296,7 @@ def create_sequential_scheduling_job( previous_sensors.append(sensor.id) previous_job = job - # create that triggers when the last job is done + # create job that triggers when the last job is done job = Job.create( func=cb_done_sequential_scheduling_job, args=([j.id for j in jobs],), From b7961f43520adbfc24ac379b5a284d878a71dfea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:49:23 +0200 Subject: [PATCH 055/162] fix: job enqueueing Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 4 ++-- flexmeasures/data/services/scheduling.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 4be917f816..6c36eb4c1a 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -496,7 +496,7 @@ def trigger_schedule( flex_context=flex_context, ) try: - job = create_sequential_scheduling_job( + jobs = create_sequential_scheduling_job( asset_or_sensor=asset, enqueue=True, **scheduler_kwargs ) except ValidationError as err: @@ -505,6 +505,6 @@ def trigger_schedule( return invalid_flex_config(str(err)) # todo: make a 'done job' and pass that job's ID here - response = dict(schedule=job.id) + response = dict(schedule=jobs[-1].id) d, s = request_processed() return dict(**response, **d), s diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index f459e11b46..4c1b72f189 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -288,7 +288,7 @@ def create_sequential_scheduling_job( scheduler_specs=scheduler_specs, requeue=requeue, job_id=job_id, - enqueue=enqueue, + enqueue=False, # we enqueue all jobs later in this method depends_on=previous_job, force_new_job_creation=force_new_job_creation, ) @@ -323,10 +323,10 @@ def create_sequential_scheduling_job( for job in jobs: current_app.queues["scheduling"].enqueue_job(job) current_app.job_cache.add( - asset_or_sensor["id"], + asset_or_sensor.id, job.id, queue="scheduling", - asset_or_sensor_type=asset_or_sensor["class"].lower(), + asset_or_sensor_type=str(type(asset_or_sensor)).lower(), ) return jobs From 429d97fa63c9dd1d4230c9b1ed66e01e66c2beb9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:59:51 +0200 Subject: [PATCH 056/162] style: black Signed-off-by: F.N. Claessen --- flexmeasures/api/common/utils/args_parsing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/utils/args_parsing.py b/flexmeasures/api/common/utils/args_parsing.py index 4c598130b8..4bc50567f2 100644 --- a/flexmeasures/api/common/utils/args_parsing.py +++ b/flexmeasures/api/common/utils/args_parsing.py @@ -58,6 +58,8 @@ def load_data(request, schema): # Avoid clashes for key in json_params: if key in newdata: - raise FMValidationError(message=f"{key} already set in the URL path or query parameters") + raise FMValidationError( + message=f"{key} already set in the URL path or query parameters" + ) newdata.update(json_params) return MultiDictProxy(newdata, schema) From 84235227b90eeb74467abd6a5cb150917b730788 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 12:06:23 +0200 Subject: [PATCH 057/162] refactor: rename argument; sequential scheduling is only for assets Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 +- flexmeasures/data/services/scheduling.py | 6 +++--- flexmeasures/data/tests/test_scheduling_sequential.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 6c36eb4c1a..6bc3be6170 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -497,7 +497,7 @@ def trigger_schedule( ) try: jobs = create_sequential_scheduling_job( - asset_or_sensor=asset, enqueue=True, **scheduler_kwargs + asset=asset, enqueue=True, **scheduler_kwargs ) except ValidationError as err: return invalid_flex_config(err.messages) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 4c1b72f189..9334be538c 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -256,7 +256,7 @@ def cb_done_sequential_scheduling_job(jobs_ids: list[str]): def create_sequential_scheduling_job( - asset_or_sensor: Asset | Sensor | None = None, + asset: Asset, job_id: str | None = None, enqueue: bool = True, requeue: bool = False, @@ -323,10 +323,10 @@ def create_sequential_scheduling_job( for job in jobs: current_app.queues["scheduling"].enqueue_job(job) current_app.job_cache.add( - asset_or_sensor.id, + asset.id, job.id, queue="scheduling", - asset_or_sensor_type=str(type(asset_or_sensor)).lower(), + asset_or_sensor_type="asset", ) return jobs diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 4ba23e145f..e00fb6e457 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -19,7 +19,7 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil flex_description_sequential["end"] = end.isoformat() jobs = create_sequential_scheduling_job( - asset_or_sensor=assets["Test Site"], + asset=assets["Test Site"], scheduler_specs=scheduler_specs, enqueue=False, **flex_description_sequential, From c60f5cf451022d9f8253c02cc9ec937644d7e75b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 12:07:36 +0200 Subject: [PATCH 058/162] docs: explain assert statement Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_sequential.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index e00fb6e457..5d025c452e 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -25,7 +25,9 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil **flex_description_sequential, ) - assert len(jobs) == 3 + assert ( + len(jobs) == 3 + ), "There should be 3 jobs: 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up." # The EV is scheduled firstly. assert jobs[0].kwargs["asset_or_sensor"] == { From 9f5eb6a1149b3125c903e56ee4bd4e0cb56d5205 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 12:10:36 +0200 Subject: [PATCH 059/162] style: black Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 8a7071144a..144bab940e 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -337,7 +337,7 @@ def flex_description_sequential(smart_building, setup_markets): "soc-unit": "MWh", "soc-min": 0.0, "soc-max": 0.05, # 50 kWh - } + }, }, { "sensor": sensors["Test Battery"], @@ -347,7 +347,7 @@ def flex_description_sequential(smart_building, setup_markets): "soc-unit": "MWh", "soc-min": 0.0, "soc-max": 0.1, # 100 kWh - } + }, }, ], "flex_context": { From 71da5c48dd6568250abfa21e9887a1b26d331700 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 5 Jun 2024 12:14:00 +0200 Subject: [PATCH 060/162] handle fallback and rescheduling Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/services/scheduling.py | 23 +++- flexmeasures/data/tests/conftest.py | 81 ++++++++----- .../data/tests/test_scheduling_sequential.py | 111 ++++++++++++++++-- 3 files changed, 174 insertions(+), 41 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 139c284bfa..fae4a0c876 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -9,7 +9,7 @@ import sys import importlib.util from importlib.abc import Loader -from typing import Type +from typing import Callable, Type import inspect from copy import deepcopy @@ -102,6 +102,15 @@ def load_custom_scheduler(scheduler_specs: dict) -> type: return scheduler_class +def success_callback(job, connection, result, *args, **kwargs): + queue = current_app.queues["scheduling"] + orginal_job = Job.fetch(job.meta["original_job_id"]) + + # requeue deferred jobs + for dependent_job_ids in orginal_job.dependent_ids: + queue.deferred_job_registry.requeue(dependent_job_ids) + + def trigger_optional_fallback(job, connection, type, value, traceback): """Create a fallback schedule job when the error is of type InfeasibleProblemException""" @@ -136,9 +145,16 @@ def trigger_optional_fallback(job, connection, type, value, traceback): force_new_job_creation=True, enqueue=False, scheduler_specs=scheduler_specs, + success_callback=Callback(success_callback), **scheduler_kwargs, ) + # keep track of the id of the original (non-fallback) job + fallback_job.meta["original_job_id"] = job.meta.get( + "original_job_id", job.id + ) + fallback_job.save_meta() + job.meta["fallback_job_id"] = fallback_job.id job.save_meta() current_app.queues["scheduling"].enqueue_job(fallback_job) @@ -154,6 +170,7 @@ def create_scheduling_job( force_new_job_creation: bool = False, scheduler_specs: dict | None = None, depends_on: Job | list[Job] | None = None, + success_callback: Callable | None = None, **scheduler_kwargs, ) -> Job: """ @@ -176,6 +193,7 @@ def create_scheduling_job( :param requeue: if True, requeues the job in case it is not new and had previously failed (this argument is used by the @job_cache decorator) :param force_new_job_creation: if True, this attribute forces a new job to be created (skipping cache) + :param success_callback: callback function that runs on success. (this argument is used by the @job_cache decorator) :returns: the job @@ -223,6 +241,7 @@ def create_scheduling_job( ).total_seconds() ), # NB job.cleanup docs says a negative number of seconds means persisting forever on_failure=Callback(trigger_optional_fallback), + on_success=success_callback, depends_on=depends_on, ) @@ -264,7 +283,7 @@ def create_sequential_scheduling_job( scheduler_specs: dict | None = None, depends_on: list[Job] | None = None, **scheduler_kwargs, -): +) -> list[Job]: flex_model = scheduler_kwargs["flex_model"] jobs = [] previous_sensors = [] diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 5f78d54faa..c60634f1cd 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -230,21 +230,25 @@ def _compute_report(self, **kwargs) -> list: @pytest.fixture -def smart_building_types(app, db): +def smart_building_types(app, fresh_db, setup_generic_asset_types_fresh_db): site = AssetType(name="site") - solar = AssetType(name="solar") building = AssetType(name="building") - battery = AssetType(name="battery") ev = AssetType(name="ev") - db.session.add_all([site, solar, building, battery, ev]) - db.session.flush() + fresh_db.session.add_all([site, building, ev]) + fresh_db.session.flush() - return (site, solar, building, battery, ev) + return ( + site, + setup_generic_asset_types_fresh_db["solar"], + building, + setup_generic_asset_types_fresh_db["battery"], + ev, + ) @pytest.fixture -def smart_building(app, db, smart_building_types): +def smart_building(app, fresh_db, smart_building_types): """ Topology of the sytstem: @@ -269,36 +273,39 @@ def smart_building(app, db, smart_building_types): site, solar, building, battery, ev = smart_building_types coordinates = {"latitude": 0, "longitude": 0} - test_site = Asset(name="Test Site", generic_asset_type=site, **coordinates) - db.session.add(test_site) - db.session.flush() + test_site = Asset(name="Test Site", generic_asset_type_id=site.id, **coordinates) + fresh_db.session.add(test_site) + fresh_db.session.flush() test_building = Asset( name="Test Building", - generic_asset_type=building, - parent_asset_id=site.id, + generic_asset_type_id=building.id, + parent_asset_id=test_site.id, **coordinates, ) test_solar = Asset( name="Test Solar", - generic_asset_type=solar, - parent_asset_id=site.id, + generic_asset_type_id=solar.id, + parent_asset_id=test_site.id, **coordinates, ) test_battery = Asset( name="Test Battery", - generic_asset_type=battery, - parent_asset_id=site.id, + generic_asset_type_id=battery.id, + parent_asset_id=test_site.id, **coordinates, ) test_ev = Asset( - name="Test EV", generic_asset_type=ev, parent_asset_id=site.id, **coordinates + name="Test EV", + generic_asset_type_id=ev.id, + parent_asset_id=test_site.id, + **coordinates, ) assets = (test_site, test_building, test_solar, test_battery, test_ev) - db.session.add_all(assets) - db.session.flush() + fresh_db.session.add_all(assets) + fresh_db.session.flush() sensors = [] @@ -307,47 +314,63 @@ def smart_building(app, db, smart_building_types): sensor = Sensor( name="power", unit="MW", - event_resolution="PT15M", + event_resolution=timedelta(minutes=15), generic_asset=asset, - # TODO: add knowledge horizon function? + timezone="Europe/Amsterdam", ) sensors.append(sensor) - db.session.add_all(sensors) - db.session.flush() + fresh_db.session.add_all(sensors) + fresh_db.session.flush() asset_names = [asset.name for asset in assets] return dict(zip(asset_names, assets)), dict(zip(asset_names, sensors)) @pytest.fixture -def flex_description_sequential(smart_building, setup_markets): +def flex_description_sequential( + smart_building, setup_markets_fresh_db, add_market_prices_fresh_db +): assets, sensors = smart_building return { "flex_model": [ { "sensor": sensors["Test EV"].id, + "consumption-capacity": "10kW", + "production-capacity": "0kW", "power-capacity": "10kW", - "soc-at-start": 0.01, # 10 kWh + "soc-at-start": 0.00, # 0 kWh "soc-unit": "MWh", "soc-min": 0.0, "soc-max": 0.05, # 50 kWh + "soc-targets": [ + { + "start": "2015-01-03T00:00:00+01:00", + "end": "2015-01-03T10:00:00+01:00", + "value": 0.0, + }, + {"datetime": "2015-01-03T23:45:00+01:00", "value": 0.05}, + ], }, { "sensor": sensors["Test Battery"].id, - "power-capacity": "20kW", - "soc-at-start": 0.01, # 10 kWh + "consumption-capacity": "0kW", + "production-capacity": "10kW", + "power-capacity": "10kW", + "soc-at-start": 0.1, # Batery is initially full (100 kWh) "soc-unit": "MWh", "soc-min": 0.0, "soc-max": 0.1, # 100 kWh }, ], "flex_context": { - "consumption-price-sensor": setup_markets["epex_da"].id, - "production-price-sensor": setup_markets["epex_da_production"].id, + "consumption-price-sensor": setup_markets_fresh_db["epex_da"].id, + "production-price-sensor": setup_markets_fresh_db["epex_da"].id, "inflexible-device-sensors": [ sensors["Test Solar"].id, sensors["Test Building"].id, ], + "site-production-capacity": "0kW", + "site-consumption-capacity": "100kW", }, } diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index ad183cad81..b0d3138aa3 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -1,12 +1,15 @@ +from unittest.mock import patch +from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException + import pandas as pd from flexmeasures.data.services.scheduling import create_sequential_scheduling_job - -# from flexmeasures.data.tests.utils import work_on_rq, exception_reporter +from flexmeasures.data.tests.utils import work_on_rq +from flexmeasures.data.services.scheduling import handle_scheduling_exception def test_create_sequential_jobs(db, app, flex_description_sequential, smart_building): assets, sensors = smart_building - # queue = app.queues["scheduling"] + queue = app.queues["scheduling"] start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") end = pd.Timestamp("2015-01-04").tz_localize("Europe/Amsterdam") @@ -15,14 +18,14 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil "class": "StorageScheduler", } - flex_description_sequential["start"] = start.isoformat() - flex_description_sequential["end"] = end.isoformat() + flex_description_sequential["start"] = start + flex_description_sequential["end"] = end jobs = create_sequential_scheduling_job( asset_or_sensor=assets["Test Site"], scheduler_specs=scheduler_specs, enqueue=False, - **flex_description_sequential + **flex_description_sequential, ) assert len(jobs) == 3 @@ -50,8 +53,96 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil sensors["Test EV"].id, ] - # TODO: enqueue jobs, let them run and check results - # for job in jobs: - # queue.enqueue_job(job) + ev_power = sensors["Test EV"].search_beliefs() + battery_power = sensors["Test Battery"].search_beliefs() + + # sensors are empty before running the schedule + assert ev_power.empty + assert battery_power.empty + + # enqueue all the tasks + for job in jobs: + queue.enqueue_job(job) + + # work tasks + work_on_rq(queue) + + # check that the jobs complete successfuly + assert jobs[0].get_status() == "finished" + assert jobs[1].get_status() == "finished" + + # check results + ev_power = sensors["Test EV"].search_beliefs() + assert ev_power.sources.unique()[0].model == "StorageScheduler" + ev_power = ev_power.droplevel([1, 2, 3]) + + battery_power = sensors["Test Battery"].search_beliefs() + assert battery_power.sources.unique()[0].model == "StorageScheduler" + battery_power = battery_power.droplevel([1, 2, 3]) + + start_charging = start + pd.Timedelta(hours=10) + end_charging = start + pd.Timedelta(hours=15) - sensors["Test EV"].event_resolution + + assert all(ev_power.loc[start_charging:end_charging] == -0.01) # 10 kW + assert all(battery_power.loc[start_charging:end_charging] == 0.01) # 10 kW + + +def test_create_sequential_jobs_fallback( + db, app, flex_description_sequential, smart_building +): + assets, sensors = smart_building + queue = app.queues["scheduling"] + + start = pd.Timestamp("2015-01-03").tz_localize("Europe/Amsterdam") + end = pd.Timestamp("2015-01-04").tz_localize("Europe/Amsterdam") + + scheduler_specs = { + "module": "flexmeasures.data.models.planning.storage", + "class": "StorageScheduler", + } + + flex_description_sequential["start"] = start + flex_description_sequential["end"] = end + + storage_module = "flexmeasures.data.models.planning.storage" + + with ( + patch(f"{storage_module}.StorageScheduler.persist_flex_model"), + patch(f"{storage_module}.StorageFallbackScheduler.persist_flex_model"), + patch( + f"{storage_module}.StorageScheduler.compute", + side_effect=iter([InfeasibleProblemException(), [], []]), + ), + ): + jobs = create_sequential_scheduling_job( + asset_or_sensor=assets["Test Site"], + scheduler_specs=scheduler_specs, + enqueue=False, + **flex_description_sequential, + ) + + assert len(jobs) == 3 + + # enqueue all the tasks + for job in jobs: + queue.enqueue_job(job) + + # work tasks + work_on_rq(queue, exc_handler=handle_scheduling_exception) + + # refresh jobs + for job in jobs: + job.refresh() + + finished_jobs = queue.finished_job_registry.get_job_ids() + failed_jobs = queue.failed_job_registry.get_job_ids() + + # First jobs failed + assert jobs[0].id in failed_jobs + + # The Fallback Job runs successfully + assert jobs[0].meta["fallback_job_id"] in finished_jobs - # work_on_rq(queue, exc_handler=exception_reporter) + # Jobs 1 and 2 run successfully + assert jobs[1].id in finished_jobs + assert jobs[2].id in finished_jobs From 07b44b67ede95c05bd4c2ecac560029b57c6a817 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 12:15:18 +0200 Subject: [PATCH 061/162] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 1 - flexmeasures/data/services/scheduling.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index b9bf1a1e02..b770d10303 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -14,7 +14,6 @@ from flexmeasures.data.schemas.utils import ( FMValidationError, MarshmallowClickMixin, - with_appcontext_if_needed, ) from flexmeasures.auth.policy import user_has_admin_access from flexmeasures.cli import is_running as running_as_cli diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 9334be538c..d1cb7cb0f7 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -16,7 +16,6 @@ from flask import current_app import click -from jsonschema import ValidationError from rq import get_current_job, Callback from rq.job import Job import timely_beliefs as tb @@ -32,7 +31,6 @@ from flexmeasures.data.models.generic_assets import GenericAsset as Asset from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.utils import get_data_source, save_to_db -from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.utils.time_utils import server_now from flexmeasures.data.services.utils import ( job_cache, From 5f9a4dcaf1e517e55e93bfe346d338bd5f1502af Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 12:26:21 +0200 Subject: [PATCH 062/162] fix: clarify join to avoid `sqlalchemy.exc.AmbiguousForeignKeysError` Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 334c0cec38..7919caaceb 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -47,8 +47,7 @@ def test_asset_trigger_and_get_schedule( sensor = ( Sensor.query.filter(Sensor.name == "power") - .join(GenericAsset) - .filter(GenericAsset.id == Sensor.generic_asset_id) + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) .filter(GenericAsset.name == asset_name) .one_or_none() ) From 8b0c0d0ba4f6a899070984ea421fa4bed6bee8ea Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 5 Jun 2024 12:59:35 +0200 Subject: [PATCH 063/162] fix merge Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/tests/conftest.py | 48 ++++++++++--------- .../data/tests/test_scheduling_sequential.py | 2 +- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index 52e8cff8c7..d41bce7763 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -340,31 +340,35 @@ def flex_description_sequential( "flex_model": [ { "sensor": sensors["Test EV"], - "consumption-capacity": "10kW", - "production-capacity": "0kW", - "power-capacity": "10kW", - "soc-at-start": 0.00, # 0 kWh - "soc-unit": "MWh", - "soc-min": 0.0, - "soc-max": 0.05, # 50 kWh - "soc-targets": [ - { - "start": "2015-01-03T00:00:00+01:00", - "end": "2015-01-03T10:00:00+01:00", - "value": 0.0, - }, - {"datetime": "2015-01-03T23:45:00+01:00", "value": 0.05}, - ], + "sensor_flex_model": { + "consumption-capacity": "10kW", + "production-capacity": "0kW", + "power-capacity": "10kW", + "soc-at-start": 0.00, # 0 kWh + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.05, # 50 kWh + "soc-targets": [ + { + "start": "2015-01-03T00:00:00+01:00", + "end": "2015-01-03T10:00:00+01:00", + "value": 0.0, + }, + {"datetime": "2015-01-03T23:45:00+01:00", "value": 0.05}, + ], + }, }, { "sensor": sensors["Test Battery"], - "consumption-capacity": "0kW", - "production-capacity": "10kW", - "power-capacity": "10kW", - "soc-at-start": 0.1, # Batery is initially full (100 kWh) - "soc-unit": "MWh", - "soc-min": 0.0, - "soc-max": 0.1, # 100 kWh + "sensor_flex_model": { + "consumption-capacity": "0kW", + "production-capacity": "10kW", + "power-capacity": "10kW", + "soc-at-start": 0.1, # Batery is initially full (100 kWh) + "soc-unit": "MWh", + "soc-min": 0.0, + "soc-max": 0.1, # 100 kWh + }, }, ], "flex_context": { diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 43ee6dae65..2527a6f637 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -117,7 +117,7 @@ def test_create_sequential_jobs_fallback( ), ): jobs = create_sequential_scheduling_job( - asset_or_sensor=assets["Test Site"], + asset=assets["Test Site"], scheduler_specs=scheduler_specs, enqueue=False, **flex_description_sequential, From 05064aeb5c0dbf4e43de02075847c96f40942a5b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 16:56:38 +0200 Subject: [PATCH 064/162] fix: remove 404, because the `asset` field should be seen as part of the POSTed payload to be validated Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index d7bc39ae85..f660487b84 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -52,7 +52,7 @@ def unwrap_envelope(self, data, **kwargs): class AssetTriggerSchema(Schema): - asset = GenericAssetIdField(data_key="id", status_if_not_found=404) + asset = GenericAssetIdField(data_key="id") start_of_schedule = AwareDateTimeField( data_key="start", format="iso", required=True ) From c7e58cbd8c58c62907fb7661b6f2275f947cec18 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 17:24:59 +0200 Subject: [PATCH 065/162] feature: avoid having magic number for status codes Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 33 +++++++++++++++++---- flexmeasures/data/schemas/generic_assets.py | 5 ++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 637daea77c..a1bb7047dc 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta +from http import HTTPStatus import json from flask import current_app @@ -165,7 +166,12 @@ def post(self, asset_data: dict): @route("/", methods=["GET"]) @use_kwargs( - {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path" + { + "asset": AssetIdField( + data_key="id", status_if_not_found=HTTPStatus.NOT_FOUND + ) + }, + location="path", ) @permission_required_for_context("read", ctx_arg_name="asset") @as_json @@ -203,7 +209,11 @@ def fetch_one(self, id, asset): @route("/", methods=["PATCH"]) @use_args(partial_asset_schema) @use_kwargs( - {"db_asset": AssetIdField(data_key="id", status_if_not_found=404)}, + { + "db_asset": AssetIdField( + data_key="id", status_if_not_found=HTTPStatus.NOT_FOUND + ) + }, location="path", ) @permission_required_for_context("update", ctx_arg_name="db_asset") @@ -265,7 +275,12 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): @route("/", methods=["DELETE"]) @use_kwargs( - {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, location="path" + { + "asset": AssetIdField( + data_key="id", status_if_not_found=HTTPStatus.NOT_FOUND + ) + }, + location="path", ) @permission_required_for_context("delete", ctx_arg_name="asset") @as_json @@ -293,7 +308,11 @@ def delete(self, id: int, asset: GenericAsset): @route("//chart", strict_slashes=False) # strict on next version? see #1014 @use_kwargs( - {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, + { + "asset": AssetIdField( + data_key="id", status_if_not_found=HTTPStatus.NOT_FOUND + ) + }, location="path", ) @use_kwargs( @@ -323,7 +342,11 @@ def get_chart(self, id: int, asset: GenericAsset, **kwargs): "//chart_data", strict_slashes=False ) # strict on next version? see #1014 @use_kwargs( - {"asset": AssetIdField(data_key="id", status_if_not_found=404)}, + { + "asset": AssetIdField( + data_key="id", status_if_not_found=HTTPStatus.NOT_FOUND + ) + }, location="path", ) @use_kwargs( diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index b770d10303..0f2aba5cff 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from http import HTTPStatus from flask import abort from marshmallow import validates, ValidationError, fields, validates_schema @@ -151,7 +152,7 @@ class Meta: class GenericAssetIdField(MarshmallowClickMixin, fields.Int): """Field that deserializes to a GenericAsset and serializes back to an integer.""" - def __init__(self, status_if_not_found: int | None = None, *args, **kwargs): + def __init__(self, status_if_not_found: HTTPStatus | None = None, *args, **kwargs): self.status_if_not_found = status_if_not_found super().__init__(*args, **kwargs) @@ -162,7 +163,7 @@ def _deserialize(self, value: int, attr, obj, **kwargs) -> GenericAsset: ).scalar_one_or_none() if generic_asset is None: message = f"No asset found with ID {value}." - if self.status_if_not_found == 404: + if self.status_if_not_found == HTTPStatus.NOT_FOUND: raise abort(404, message) else: raise FMValidationError(message) From a15792c114ffc4cc9fd9f5cbdfdcb16a18bc260c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 5 Jun 2024 11:45:20 +0200 Subject: [PATCH 066/162] fix: use new flex-context fields Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 7919caaceb..87585e5eb1 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -37,8 +37,8 @@ def test_asset_trigger_and_get_schedule( # Include the price sensor and site-power-capacity in the flex-context explicitly, to test deserialization price_sensor_id = add_market_prices_fresh_db["epex_da"].id message["flex-context"] = { - "consumption-price-sensor": price_sensor_id, - "production-price-sensor": price_sensor_id, + "consumption-price": {"sensor": price_sensor_id}, + "production-price": {"sensor": price_sensor_id}, "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities } From 75c80f52ae7683772dd56091970f42ae28b238c7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 14:20:21 +0200 Subject: [PATCH 067/162] chore: get rid of deprecation warning: The `sensor` keyword argument is deprecated. Please, consider using the argument `asset_or_sensor`. Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index fe607ed4c8..c10db4c388 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -339,7 +339,7 @@ def create_sequential_scheduling_job( previous_sensors ) current_scheduler_kwargs["resolution"] = sensor.event_resolution - current_scheduler_kwargs["sensor"] = sensor + current_scheduler_kwargs["asset_or_sensor"] = sensor job = create_scheduling_job( **current_scheduler_kwargs, From 74c3ff5b698374cb1f6933f8f081ababe53f2719 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 14:53:30 +0200 Subject: [PATCH 068/162] fix: return the ID of the wrap-up job Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 442ad87710..a562ad86db 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1057,7 +1057,7 @@ def trigger_schedule( flex_context=flex_context, ) try: - jobs = create_sequential_scheduling_job( + job = create_sequential_scheduling_job( asset=asset, enqueue=True, **scheduler_kwargs ) except ValidationError as err: @@ -1065,7 +1065,6 @@ def trigger_schedule( except ValueError as err: return invalid_flex_config(str(err)) - # todo: make a 'done job' and pass that job's ID here - response = dict(schedule=jobs[-1].id) + response = dict(schedule=job.id) d, s = request_processed() return dict(**response, **d), s From 0b6ab40ac274ccf29e47ee5112d0ef4a5f5369ce Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 14:58:19 +0200 Subject: [PATCH 069/162] docs: move API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 7e5ee70c05..48fc6c7628 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,12 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-24 | 2025-06-10 +"""""""""""""""""""" +- New API endpoint `[POST] /assets/(id)/schedules/trigger `_ to schedule a site with multiple flexible devices. +- Updated message for 404 Not Found on endpoints for managing assets: `/assets` (GET, POST) and `/assets/` (GET, PATCH, DELETE). + + v3.0-23 | 2025-04-08 """""""""""""""""""" @@ -114,12 +120,6 @@ v3.0-19 | 2024-08-09 - ``soc-usage`` -v3.0-19 | 2024-06-03 -"""""""""""""""""""" -- New API endpoint `[POST] /assets/(id)/schedules/trigger `_ to schedule a site with multiple flexible devices. -- Updated message for 404 Not Found on endpoints for managing assets: `/assets` (GET, POST) and `/assets/` (GET, PATCH, DELETE). - - v3.0-18 | 2024-03-07 """""""""""""""""""" From d14cb93cd957e4fba44c65b4aa8ae039600784f3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 15:05:57 +0200 Subject: [PATCH 070/162] docs: mention the multi-device scheduler handles inflexible devices, too Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index a562ad86db..c9d6b30aa1 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -916,7 +916,7 @@ def trigger_schedule( **kwargs, ): """ - Trigger FlexMeasures to create a schedule for a collection of flexible devices. + Trigger FlexMeasures to create a schedule for a collection of flexible and inflexible devices. .. :quickref: Schedule; Trigger scheduling job for multiple devices From c8eb88e844f56447d8888bf5d0eb4c0fcc0620d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 17:04:20 +0200 Subject: [PATCH 071/162] fix: GenericAsset becomes asset_or_sensor_type=="genericasset" in the job cache, whereas we expect "asset" Signed-off-by: F.N. Claessen --- flexmeasures/data/services/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py index 2f8efbfffe..3452ffa4a6 100644 --- a/flexmeasures/data/services/utils.py +++ b/flexmeasures/data/services/utils.py @@ -38,7 +38,11 @@ def get_scheduler_instance( def get_asset_or_sensor_ref(asset_or_sensor: Asset | Sensor) -> dict: - return {"id": asset_or_sensor.id, "class": asset_or_sensor.__class__.__name__} + class_name = asset_or_sensor.__class__.__name__ + # todo: remove these two lines after renaming the GenericAsset class to Asset + if class_name == "GenericAsset": + class_name = "Asset" + return {"id": asset_or_sensor.id, "class": class_name} def get_asset_or_sensor_from_ref(asset_or_sensor: dict): From 559c0ea894968d68ba53f0ca959bdb8535267f46 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 17:14:15 +0200 Subject: [PATCH 072/162] feat: test getting the schedule for each flexible device described in the flex-model Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 87585e5eb1..7f0a001e70 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -101,14 +101,16 @@ def test_asset_trigger_and_get_schedule( scheduler_source = get_data_source_for_job(scheduling_job) assert scheduler_source is not None - # try to retrieve the schedule through the /sensors//schedules/ [GET] api endpoint - get_schedule_response = client.get( - url_for( - "SensorAPI:get_schedule", id=sensor.id, uuid=scheduling_job.id - ), # todo: use (last?) job_id from trigger response - query_string={"duration": "PT48H"}, - ) - print("Server responded with:\n%s" % get_schedule_response.json) - assert get_schedule_response.status_code == 200 - # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" - assert len(get_schedule_response.json["values"]) == expected_length_of_schedule + # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint + for flex_model in message["flex-model"]: + sensor_id = flex_model["sensor"] + get_schedule_response = client.get( + url_for( + "SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id + ), # todo: use (last?) job_id from trigger response + query_string={"duration": "PT48H"}, + ) + print("Server responded with:\n%s" % get_schedule_response.json) + assert get_schedule_response.status_code == 200 + # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" + assert len(get_schedule_response.json["values"]) == expected_length_of_schedule From d46a8aa21ad7fee15f2bbee988c6f9b35fc1e9b8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 17:15:08 +0200 Subject: [PATCH 073/162] docs: add clarifying inline comment Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 7f0a001e70..f23cf78742 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -52,6 +52,8 @@ def test_asset_trigger_and_get_schedule( .one_or_none() ) message["flex-model"]["sensor"] = sensor.id + + # Convert the flex-model to a multi-asset flex-model message["flex-model"] = [ message["flex-model"], ] From 7ecab719c099a54d09235011b9a1ac4b7c5609df Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 20:08:59 +0200 Subject: [PATCH 074/162] feat: create charging hub (hierarchical structure of two existing CP assets) Signed-off-by: F.N. Claessen --- flexmeasures/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index b444ac100b..b4156c057c 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -1000,11 +1000,21 @@ def create_charging_station_assets( """Add uni- and bi-directional charging station assets, set their capacity value and their initial SOC.""" oneway_evse = GenericAssetType(name="one-way_evse") twoway_evse = GenericAssetType(name="two-way_evse") + charging_hub = GenericAssetType(name="charging_hub") + + charging_hub = GenericAsset( + name="Test charging hub", + owner=setup_accounts["Prosumer"], + generic_asset_type=charging_hub, + latitude=10, + longitude=100, + ) charging_station = GenericAsset( name="Test charging station", owner=setup_accounts["Prosumer"], generic_asset_type=oneway_evse, + parent_asset=charging_hub, latitude=10, longitude=100, flex_context={ @@ -1040,6 +1050,7 @@ def create_charging_station_assets( name="Test charging station (bidirectional)", owner=setup_accounts["Prosumer"], generic_asset_type=twoway_evse, + parent_asset=charging_hub, latitude=10, longitude=100, flex_context={ From 2e91265444fcca03cb074744550716600fcd798a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 20:10:05 +0200 Subject: [PATCH 075/162] feat: add util function to sort jobs by time of creation Signed-off-by: F.N. Claessen --- flexmeasures/data/services/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py index 3452ffa4a6..41b1da3307 100644 --- a/flexmeasures/data/services/utils.py +++ b/flexmeasures/data/services/utils.py @@ -10,6 +10,7 @@ import click from sqlalchemy import JSON, String, cast, literal from flask import current_app +from rq import Queue from rq.job import Job from sqlalchemy import select @@ -248,3 +249,13 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + +def sort_jobs(queue: Queue, jobs: list[str | Job]) -> list[Job]: + """Sort jobs in chronological order of creation, and return Job objects.""" + jobs = [queue.fetch_job(job) if isinstance(job, str) else job for job in jobs] + jobs = [ + job for job in jobs if job is not None + ] # Remove any None entries (in case some jobs don’t exist) + sorted_jobs = sorted(jobs, key=lambda job: job.created_at) + return sorted_jobs From 83ab8bd50bcb14553fd843a08b9c753bcd9b68d0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 20:28:07 +0200 Subject: [PATCH 076/162] feat: test a multi-asset flex-model comprising more than 1 flexible device Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 74 ++++++++++++------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index f23cf78742..1717883ec2 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -2,22 +2,26 @@ import pytest from isodate import parse_datetime, parse_duration +import pandas as pd from rq.job import Job from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule -from flexmeasures.data.models.generic_assets import GenericAsset -from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import ( handle_scheduling_exception, get_data_source_for_job, ) +from flexmeasures.data.services.utils import sort_jobs @pytest.mark.parametrize( - "message, asset_name", + "message, charge_point_message, asset_name", [ - (message_for_trigger_schedule(), "Test battery"), + ( + message_for_trigger_schedule(), + message_for_trigger_schedule(with_targets=True), + "Test battery", + ), ], ) @pytest.mark.parametrize( @@ -26,11 +30,11 @@ def test_asset_trigger_and_get_schedule( app, add_market_prices_fresh_db, - add_battery_assets_fresh_db, - battery_soc_sensor_fresh_db, + setup_roles_users_fresh_db, add_charging_station_assets_fresh_db, keep_scheduling_queue_empty, message, + charge_point_message, asset_name, requesting_user, ): # noqa: C901 @@ -42,27 +46,37 @@ def test_asset_trigger_and_get_schedule( "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities } - # trigger a schedule through the /assets//schedules/trigger [POST] api endpoint - assert len(app.queues["scheduling"]) == 0 + # Set up flex-model for CP 1 + CP_1_flex_model = message["flex-model"] + bidirectional_charging_station = add_charging_station_assets_fresh_db[ + "Test charging station (bidirectional)" + ] + sensor_1 = bidirectional_charging_station.sensors[0] + assert sensor_1.name == "power", "expecting to schedule a power sensor" - sensor = ( - Sensor.query.filter(Sensor.name == "power") - .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) - .filter(GenericAsset.name == asset_name) - .one_or_none() - ) - message["flex-model"]["sensor"] = sensor.id + # Set up flex-model for CP 2 + charging_station = add_charging_station_assets_fresh_db["Test charging station"] + CP_2_flex_model = charge_point_message["flex-model"] + sensor_2 = charging_station.sensors[0] + assert sensor_2.name == "power", "expecting to schedule a power sensor" - # Convert the flex-model to a multi-asset flex-model + # Convert the two flex-models to a single multi-asset flex-model + CP_1_flex_model["sensor"] = sensor_1.id + CP_2_flex_model["sensor"] = sensor_2.id message["flex-model"] = [ - message["flex-model"], + CP_1_flex_model, + CP_2_flex_model, ] + # trigger a schedule through the /assets//schedules/trigger [POST] api endpoint + assert len(app.queues["scheduling"]) == 0 with app.test_client() as client: print(message) print(message["flex-model"]) trigger_schedule_response = client.post( - url_for("AssetAPI:trigger_schedule", id=sensor.generic_asset.id), + url_for( + "AssetAPI:trigger_schedule", id=sensor_1.generic_asset.parent_asset.id + ), json=message, ) print("Server responded with:\n%s" % trigger_schedule_response.json) @@ -72,16 +86,16 @@ def test_asset_trigger_and_get_schedule( # look for scheduling jobs in queue scheduled_jobs = app.queues["scheduling"].jobs deferred_job_ids = app.queues["scheduling"].deferred_job_registry.get_job_ids() - assert len(scheduled_jobs) == len( + deferred_jobs = sort_jobs(app.queues["scheduling"], deferred_job_ids) + + assert len(scheduled_jobs) == 1, "one scheduling job should be queued" + assert len(deferred_jobs) == len( message["flex-model"] - ), "a scheduling job should be made for each sensor flex model" - assert ( - len(deferred_job_ids) == 1 - ), "only 1 job should be triggered when the last scheduling job is done" + ), "a scheduling job should be made for each sensor flex model (1 was already queued, but there is also 1 wrap-up job that should be triggered when the last scheduling job is done" scheduling_job = scheduled_jobs[0] - done_job_id = deferred_job_ids[0] + done_job_id = deferred_jobs[-1].id print(scheduling_job.kwargs) - assert scheduling_job.kwargs["asset_or_sensor"]["id"] == sensor.id + assert scheduling_job.kwargs["asset_or_sensor"]["id"] == sensor_1.id assert scheduling_job.kwargs["start"] == parse_datetime(message["start"]) assert done_job_id == job_id @@ -93,7 +107,7 @@ def test_asset_trigger_and_get_schedule( ) # Derive some expectations from the POSTed message - resolution = sensor.event_resolution + resolution = sensor_1.event_resolution expected_length_of_schedule = parse_duration(message["duration"]) / resolution # check results are in the database @@ -105,6 +119,14 @@ def test_asset_trigger_and_get_schedule( # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint for flex_model in message["flex-model"]: + + # We expect a longer schedule if the targets exceeds the original duration in the trigger + if "soc-targets" in flex_model: + for t in flex_model["soc-targets"]: + duration = pd.Timestamp(t["datetime"]) - pd.Timestamp(message["start"]) + if duration > pd.Timedelta(message["duration"]): + expected_length_of_schedule = duration / resolution + sensor_id = flex_model["sensor"] get_schedule_response = client.get( url_for( From 29af4d3ab6a31554111e0260e23eade139960539 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 20:49:13 +0200 Subject: [PATCH 077/162] feat: use StorageScheduler by default Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index c10db4c388..bdc61d0299 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -625,19 +625,10 @@ def find_scheduler_class(asset_or_sensor: Asset | Sensor) -> type: else: asset = asset_or_sensor - if asset.generic_asset_type.name in ( - "battery", - "one-way_evse", - "two-way_evse", - ): - scheduler_class = StorageScheduler - elif asset.generic_asset_type.name in ("process", "load"): + if asset.generic_asset_type.name in ("process", "load"): scheduler_class = ProcessScheduler else: - raise ValueError( - "Scheduling is not (yet) supported for asset type %s." - % asset.generic_asset_type - ) + scheduler_class = StorageScheduler return scheduler_class From e44d472c8c659c35812d514b112e632dab25eecf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 21:54:40 +0200 Subject: [PATCH 078/162] feat: allow to toggle between sequential and simultaneous scheduling, and schedule simultaneously by default Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 10 +++++++--- flexmeasures/data/schemas/scheduling/__init__.py | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index c9d6b30aa1..95a0a8a8e2 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -35,6 +35,7 @@ from flexmeasures.data.schemas.scheduling import AssetTriggerSchema from flexmeasures.data.services.scheduling import ( create_sequential_scheduling_job, + create_simultaneous_scheduling_job, ) from flexmeasures.api.common.responses import ( invalid_flex_config, @@ -913,6 +914,7 @@ def trigger_schedule( belief_time: datetime | None = None, flex_model: dict | None = None, flex_context: dict | None = None, + sequential: bool = False, **kwargs, ): """ @@ -1056,10 +1058,12 @@ def trigger_schedule( flex_model=flex_model, flex_context=flex_context, ) + if sequential: + f = create_sequential_scheduling_job + else: + f = create_simultaneous_scheduling_job try: - job = create_sequential_scheduling_job( - asset=asset, enqueue=True, **scheduler_kwargs - ) + job = f(asset=asset, enqueue=True, **scheduler_kwargs) except ValidationError as err: return invalid_flex_config(err.messages) except ValueError as err: diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 7458e5c77d..3bc9eb6538 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -486,6 +486,7 @@ class AssetTriggerSchema(Schema): data_key="flex-model", ) flex_context = fields.Dict(required=False, data_key="flex-context") + sequential = fields.Bool(load_default=False) @validates_schema def check_flex_model_sensors(self, data, **kwargs): From df0a2f7d6ea2362e2528e788a617322e87d7814b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 22:02:02 +0200 Subject: [PATCH 079/162] fix: prepare to rename GenericAsset class to Asset Signed-off-by: F.N. Claessen --- flexmeasures/data/services/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py index 41b1da3307..689190f370 100644 --- a/flexmeasures/data/services/utils.py +++ b/flexmeasures/data/services/utils.py @@ -65,13 +65,13 @@ def get_asset_or_sensor_from_ref(asset_or_sensor: dict): Sensor(id=2) """ - if asset_or_sensor["class"] == Asset.__name__: + if asset_or_sensor["class"] in (Asset.__name__, "Asset"): klass = Asset elif asset_or_sensor["class"] == Sensor.__name__: klass = Sensor else: raise ValueError( - f"Unrecognized class `{asset_or_sensor['class']}`. Please, consider using GenericAsset or Sensor." + f"Unrecognized class `{asset_or_sensor['class']}`. Please, consider using Asset or Sensor." ) return db.session.get(klass, asset_or_sensor["id"]) From 3c4113a08ace2fb8d38490f46464a99986c1cfe1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 22:03:42 +0200 Subject: [PATCH 080/162] feat: test simultaneous and sequential scheduling via API Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 1717883ec2..914779a53e 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -15,7 +15,7 @@ @pytest.mark.parametrize( - "message, charge_point_message, asset_name", + "message_without_targets, message_with_targets, asset_name", [ ( message_for_trigger_schedule(), @@ -24,6 +24,7 @@ ), ], ) +@pytest.mark.parametrize("sequential", [True, False]) @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) @@ -33,21 +34,22 @@ def test_asset_trigger_and_get_schedule( setup_roles_users_fresh_db, add_charging_station_assets_fresh_db, keep_scheduling_queue_empty, - message, - charge_point_message, + message_without_targets, + message_with_targets, asset_name, + sequential, requesting_user, ): # noqa: C901 # Include the price sensor and site-power-capacity in the flex-context explicitly, to test deserialization price_sensor_id = add_market_prices_fresh_db["epex_da"].id - message["flex-context"] = { + message_without_targets["flex-context"] = { "consumption-price": {"sensor": price_sensor_id}, "production-price": {"sensor": price_sensor_id}, "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities } # Set up flex-model for CP 1 - CP_1_flex_model = message["flex-model"] + CP_1_flex_model = message_without_targets["flex-model"].copy() bidirectional_charging_station = add_charging_station_assets_fresh_db[ "Test charging station (bidirectional)" ] @@ -56,17 +58,19 @@ def test_asset_trigger_and_get_schedule( # Set up flex-model for CP 2 charging_station = add_charging_station_assets_fresh_db["Test charging station"] - CP_2_flex_model = charge_point_message["flex-model"] + CP_2_flex_model = message_with_targets["flex-model"].copy() sensor_2 = charging_station.sensors[0] assert sensor_2.name == "power", "expecting to schedule a power sensor" # Convert the two flex-models to a single multi-asset flex-model CP_1_flex_model["sensor"] = sensor_1.id CP_2_flex_model["sensor"] = sensor_2.id + message = message_without_targets.copy() message["flex-model"] = [ CP_1_flex_model, CP_2_flex_model, ] + message["sequential"] = sequential # trigger a schedule through the /assets//schedules/trigger [POST] api endpoint assert len(app.queues["scheduling"]) == 0 @@ -89,13 +93,28 @@ def test_asset_trigger_and_get_schedule( deferred_jobs = sort_jobs(app.queues["scheduling"], deferred_job_ids) assert len(scheduled_jobs) == 1, "one scheduling job should be queued" - assert len(deferred_jobs) == len( - message["flex-model"] - ), "a scheduling job should be made for each sensor flex model (1 was already queued, but there is also 1 wrap-up job that should be triggered when the last scheduling job is done" + if sequential: + assert len(deferred_jobs) == len( + message["flex-model"] + ), "a scheduling job should be made for each sensor flex model (1 was already queued, but there is also 1 wrap-up job that should be triggered when the last scheduling job is done" + done_job_id = deferred_jobs[-1].id + else: + assert ( + len(deferred_jobs) == 0 + ), "the whole scheduling job is handled as a single job (simultaneous scheduling)" + done_job_id = scheduled_jobs[0].id scheduling_job = scheduled_jobs[0] - done_job_id = deferred_jobs[-1].id + print(scheduling_job.kwargs) - assert scheduling_job.kwargs["asset_or_sensor"]["id"] == sensor_1.id + if sequential: + assert ( + scheduling_job.kwargs["asset_or_sensor"]["id"] == sensor_1.id + ), "first queued job is for scheduling the first sensor" + else: + assert ( + scheduling_job.kwargs["asset_or_sensor"]["id"] + == sensor_1.generic_asset.parent_asset.id + ), "first queued job is the one for the top-level asset" assert scheduling_job.kwargs["start"] == parse_datetime(message["start"]) assert done_job_id == job_id @@ -121,7 +140,7 @@ def test_asset_trigger_and_get_schedule( for flex_model in message["flex-model"]: # We expect a longer schedule if the targets exceeds the original duration in the trigger - if "soc-targets" in flex_model: + if sequential and "soc-targets" in flex_model: for t in flex_model["soc-targets"]: duration = pd.Timestamp(t["datetime"]) - pd.Timestamp(message["start"]) if duration > pd.Timedelta(message["duration"]): From fba0f6e61c54008216c0c757aa7373ffe77b4477 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Jun 2025 22:04:13 +0200 Subject: [PATCH 081/162] chore: clean up todo Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 914779a53e..bc38123c30 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -148,9 +148,7 @@ def test_asset_trigger_and_get_schedule( sensor_id = flex_model["sensor"] get_schedule_response = client.get( - url_for( - "SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id - ), # todo: use (last?) job_id from trigger response + url_for("SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id), query_string={"duration": "PT48H"}, ) print("Server responded with:\n%s" % get_schedule_response.json) From 5b3c06ea6bc78922a49a613fbfa2c7057bbcedf6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 09:32:11 +0200 Subject: [PATCH 082/162] fix: update test Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_simultaneous.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/test_scheduling_simultaneous.py b/flexmeasures/data/tests/test_scheduling_simultaneous.py index 95f39cf145..5e31e598f1 100644 --- a/flexmeasures/data/tests/test_scheduling_simultaneous.py +++ b/flexmeasures/data/tests/test_scheduling_simultaneous.py @@ -38,7 +38,7 @@ def test_create_simultaneous_jobs( # The EV is scheduled firstly. assert job.kwargs["asset_or_sensor"] == { "id": assets["Test Site"].id, - "class": "GenericAsset", + "class": "Asset", } # It uses the inflexible-device-sensors that are defined in the flex-context, exclusively. assert job.kwargs["flex_context"]["inflexible-device-sensors"] == [ From d23fc7cb450ca5cd4115fe38850937ac0c094376 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 09:35:42 +0200 Subject: [PATCH 083/162] fix: correct type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/generic_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index f673eac398..f2fcc15d1d 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -272,7 +272,7 @@ def __init__(self, status_if_not_found: HTTPStatus | None = None, *args, **kwarg self.status_if_not_found = status_if_not_found super().__init__(*args, **kwargs) - def _deserialize(self, value: int, attr, obj, **kwargs) -> GenericAsset: + def _deserialize(self, value: int | str, attr, obj, **kwargs) -> GenericAsset: """Turn a generic asset id into a GenericAsset.""" generic_asset: GenericAsset = db.session.execute( select(GenericAsset).filter_by(id=int(value)) From 29144b1b46efb18e778c43ec96adb5fc9ae75a06 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 09:43:09 +0200 Subject: [PATCH 084/162] fix: point API users to pick up their schedules for each power sensor individually Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 95a0a8a8e2..0018455566 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1029,7 +1029,7 @@ def trigger_schedule( This message indicates that the scheduling request has been processed without any error. A scheduling job has been created with some Universally Unique Identifier (UUID), which will be picked up by a worker. - The given UUID may be used to obtain the resulting schedule: see /assets//schedules/. + The given UUID may be used to obtain the resulting schedule: see /sensors//schedules/. .. sourcecode:: json From 0a9f8114c52ef7473fd47234c7c65544cbf5d7c0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 10:10:49 +0200 Subject: [PATCH 085/162] docs: update endpoint documentation in docstring Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 58 ++++++++------------------------- 1 file changed, 13 insertions(+), 45 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 0018455566..4da290b29e 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -934,9 +934,10 @@ def trigger_schedule( For details on flexibility model and context, see :ref:`describing_flexibility`. Below, we'll also list some examples. - .. note:: This endpoint support scheduling an EMS with multiple flexible sensors at once, - but internally, it does so sequentially + .. note:: This endpoint support scheduling an EMS with multiple flexible devices at once. + Internally, it can do so or jointly (the default) or sequentially (considering already scheduled sensors as inflexible). + To use sequential scheduling, use ``sequential=true`` in the JSON body. The length of the schedule can be set explicitly through the 'duration' field. Otherwise, it is set by the config setting :ref:`planning_horizon_config`, which defaults to 48 hours. @@ -949,70 +950,37 @@ def trigger_schedule( If you have ideas for algorithms that should be part of FlexMeasures, let us know: https://flexmeasures.io/get-in-touch/ - **Example request A** - - This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh. - - .. code-block:: json - - { - "start": "2015-06-02T10:00:00+00:00", - "flex-model": [ - { - "sensor": 931, - "soc-at-start": 12.1, - "soc-unit": "kWh" - } - ] - } - - **Example request B** + **Example request** - This message triggers a 24-hour schedule for a storage asset, starting at 10.00am, - at which the state of charge (soc) is 12.1 kWh, with a target state of charge of 25 kWh at 4.00pm. + This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh, + together with a curtailable production asset, whose production forecasts are recorded under sensor 760. - The charging efficiency is constant (120%) and the discharging efficiency is determined by the contents of sensor - with id 98. If just the ``roundtrip-efficiency`` is known, it can be described with its own field. - The global minimum and maximum soc are set to 10 and 25 kWh, respectively. - To guarantee a minimum SOC in the period prior, the sensor with ID 300 contains beliefs at 2.00pm and 3.00pm, for 15kWh and 20kWh, respectively. - Storage efficiency is set to 99.99%, denoting the state of charge left after each time step equal to the sensor's resolution. Aggregate consumption (of all devices within this EMS) should be priced by sensor 9, and aggregate production should be priced by sensor 10, where the aggregate power flow in the EMS is described by the sum over sensors 13, 14 and 15 - (plus the flexible sensor being optimized, of course). - + (plus the two sensors for the flexible devices being optimized, of course). The battery consumption power capacity is limited by sensor 42 and the production capacity is constant (30 kW). Finally, the site consumption capacity is limited by sensor 32. - Note that, if forecasts for sensors 13, 14 and 15 are not available, a schedule cannot be computed. - .. code-block:: json { "start": "2015-06-02T10:00:00+00:00", - "duration": "PT24H", "flex-model": [ { "sensor": 931, "soc-at-start": 12.1, "soc-unit": "kWh", - "soc-targets": [ - { - "value": 25, - "datetime": "2015-06-02T16:00:00+00:00" - }, - ], - "soc-minima": {"sensor" : 300}, - "soc-min": 10, - "soc-max": 25, - "charging-efficiency": "120%", - "discharging-efficiency": {"sensor": 98}, - "storage-efficiency": 0.9999, "power-capacity": "25kW", "consumption-capacity" : {"sensor": 42}, "production-capacity" : "30 kW" }, + { + "sensor": 760, + "consumption-capacity": "0 kW", + "production-capacity": {"sensor": 760}, + } ], "flex-context": { "consumption-price-sensor": 9, @@ -1029,7 +997,7 @@ def trigger_schedule( This message indicates that the scheduling request has been processed without any error. A scheduling job has been created with some Universally Unique Identifier (UUID), which will be picked up by a worker. - The given UUID may be used to obtain the resulting schedule: see /sensors//schedules/. + The given UUID may be used to obtain the resulting schedule for each flexible device: see /sensors//schedules/. .. sourcecode:: json From 36da302089388c876249b9c3bc01490a678275b9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 10:13:46 +0200 Subject: [PATCH 086/162] docs: update notes regarding API endpoint(s) for triggering schedule computation Signed-off-by: F.N. Claessen --- documentation/features/scheduling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index c0c78cffdc..45cd8e6920 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -44,7 +44,7 @@ For more details on the possible formats for field values, see :ref:`variable_qu Where should you set these fields? Within requests to the API or by editing the relevant asset in the UI. -If they are not sent in via the API (the endpoint triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset. +If they are not sent in via the API (one of the endpoints triggering schedule computation), the scheduler will look them up on the `flex-context` field of the asset. And if the asset belongs to a larger system (a hierarchy of assets), the scheduler will also search if parent assets have them set. @@ -156,7 +156,7 @@ The process scheduler is suitable for shiftable, breakable and inflexible loads, We describe the respective flex models below. -At the moment, they have to be sent through the API (the endpoint to trigger schedule computation, or using the FlexMeasures client) or through the CLI (the command to add schedules). +At the moment, they have to be sent through the API (one of the endpoints to trigger schedule computation, or using the FlexMeasures client) or through the CLI (the command to add schedules). We will soon work on the possibility to store (a subset of) these fields on the data model and edit them in the UI. From 9afcdef9d95b4c87fd215a3b403547dc0be0d0c1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 11:42:03 +0200 Subject: [PATCH 087/162] fix: scheduler_kwargs should be JSON serializable Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 6 +++++- flexmeasures/data/services/utils.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index bdc61d0299..c109285bc3 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -5,6 +5,7 @@ from __future__ import annotations from datetime import datetime, timedelta +import json import os import sys import importlib.util @@ -40,6 +41,7 @@ get_asset_or_sensor_ref, get_asset_or_sensor_from_ref, get_scheduler_instance, + json_isoformat, ) @@ -247,7 +249,9 @@ def create_scheduling_job( ) job.meta["asset_or_sensor"] = asset_or_sensor - job.meta["scheduler_kwargs"] = scheduler_kwargs + job.meta["scheduler_kwargs"] = json.loads( + json.dumps(scheduler_kwargs, default=json_isoformat) + ) job.save_meta() # in case the function enqueues it diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py index 689190f370..f9b3bee16d 100644 --- a/flexmeasures/data/services/utils.py +++ b/flexmeasures/data/services/utils.py @@ -259,3 +259,23 @@ def sort_jobs(queue: Queue, jobs: list[str | Job]) -> list[Job]: ] # Remove any None entries (in case some jobs don’t exist) sorted_jobs = sorted(jobs, key=lambda job: job.created_at) return sorted_jobs + + +def json_isoformat(obj): + """JSON serializer that attempts to isoformat objects that aren't serializable by default. + + Especially suitable for timedelta and datetime objects. + + Usage: + + json.dumps(dict_with_various_objects, default=json_isoformat) + + """ + + try: + if hasattr(obj, "isoformat"): + return obj.isoformat() + else: + return str(obj) + except Exception: + raise TypeError(f"Object of type {type(obj)} is not JSON serializable") From f156981e6383ea0add2171c3a29b719576ad9e18 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 13:16:32 +0200 Subject: [PATCH 088/162] fix: don't validate storage constraints on a non-storage device Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 499e1594e5..772f87e358 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -686,6 +686,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 soc_max[d], soc_min[d], ) + else: + # No need to validate non-existing storage constraints + skip_validation = True power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], From 4fdb4a6cbf7b355b9e9543d9757b955207e27d3c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 14:59:13 +0200 Subject: [PATCH 089/162] docs: update main example on posting flex states Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 6eb63b6419..f4be4fe4ea 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -271,14 +271,18 @@ There is one more crucial kind of data that FlexMeasures needs to know about: Wh For example, a battery has a certain state of charge, which is relevant to describe the flexibility that the battery currently has. In our terminology, this is called the "flex model" and you can read more at :ref:`describing_flexibility`. -Owners of such devices can post the flex model along with triggering the creation of a new schedule, to `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_. +Owners of such devices can post the flex model along with triggering the creation of a new schedule, to one of two endpoints: + +1. `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ - for scheduling multiple devices +2. `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ - for scheduling a single device (which can also be done with the first endpoint) + The URL might look like this: .. code-block:: html - https://company.flexmeasures.io/api//sensors/10/schedules/trigger + https://company.flexmeasures.io/api//assets/10/schedules/trigger -The following example triggers a schedule for a power sensor (with ID 10) of a battery asset, asking to take into account the battery's current state of charge. +The following example triggers a schedule for a power sensor (with ID 15) of a battery asset (with ID 10), asking to take into account the battery's current state of charge. From this, FlexMeasures derives the energy flexibility this battery has in the next 48 hours and computes an optimal charging schedule. The endpoint also allows to limit the flexibility range and also to set target values. @@ -286,9 +290,12 @@ The endpoint also allows to limit the flexibility range and also to set target v { "start": "2015-06-02T10:00:00+00:00", - "flex-model": { - "soc-at-start": "12.1 kWh" - } + "flex-model": [ + { + "sensor": 15, + "soc-at-start": "12.1 kWh" + } + ] } .. note:: More details on supported flex models can be found in :ref:`flex_models_and_schedulers`. From 9b7f8bd7bd6d1414b271e3af60f9b50d86d9a707 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:02:11 +0200 Subject: [PATCH 090/162] docs: update comment on persisting flexibility states Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index f4be4fe4ea..d657ca5c27 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -300,6 +300,9 @@ The endpoint also allows to limit the flexibility range and also to set target v .. note:: More details on supported flex models can be found in :ref:`flex_models_and_schedulers`. -.. note:: Flexibility states are persisted on sensor attributes. To record a more complete history of the state of charge, set up a separate sensor and post data to it using `[POST] /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ (see :ref:`posting_sensor_data`). +.. note:: + Flexibility states posted in trigger messages are only stored temporarily to describe the scheduling job. + To record a more complete history of the state of charge, set up a separate sensor and post data to it using `[POST] /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ (see :ref:`posting_sensor_data`). + Then reference that sensor in your flex model. In :ref:`how_queue_scheduling`, we'll cover what happens when FlexMeasures is triggered to create a new schedule, and how those schedules can be retrieved via the API, so they can be used to steer assets. \ No newline at end of file From c99045cb6786af1fc9d8117f650a2c3988f31172 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:06:15 +0200 Subject: [PATCH 091/162] docs: update the trigger endpoints referenced in the scheduling section Signed-off-by: F.N. Claessen --- documentation/features/scheduling.rst | 2 +- documentation/tut/forecasting_scheduling.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 45cd8e6920..5c00eef970 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -275,7 +275,7 @@ depending on the first target state of charge and the capabilities of the asset. Of course, we also log a failure in the scheduling job, so it's important to take note of these failures. Often, mis-configured flex models are the reason. -For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_. +For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_. Finally, are you interested in the linear programming details behind the storage scheduler? Then head over to :ref:`storage_device_scheduler`! diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index f7a1cfb1f9..e5925137d8 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -99,7 +99,7 @@ It usually involves a linear program that combines a state of energy flexibility There are two ways to queue a scheduling job: First, we can add a scheduling job to the queue via the API. -We already learned about the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ endpoint in :ref:`posting_flex_states`, where we saw how to post a flexibility state (in this case, the state of charge of a battery at a certain point in time). +We already learned about the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ endpoint in :ref:`posting_flex_states`, where we saw how to post a flexibility state (in this case, the state of charge of a battery at a certain point in time). Here, we extend that (storage) example with an additional target value, representing a desired future state of charge. @@ -182,7 +182,7 @@ We saw above how FlexMeasures can create optimised schedules with control signal https://company.flexmeasures.io/api//sensors//schedules/ -Here, the schedule's Universally Unique Identifier (UUID) should be filled in that is returned in the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ response. +Here, the schedule's Universally Unique Identifier (UUID) should be filled in that is returned in the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ response. Schedules can be queried by their UUID for up to 1 week after they were triggered (ask your host if you need to keep them around longer). Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. From 85c144b253cf47b666f22a859143b254e64bb519 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:12:32 +0200 Subject: [PATCH 092/162] docs: update quickref Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index a89aa25199..32593ae881 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -577,7 +577,7 @@ def get_schedule( # noqa: C901 ): """Get a schedule from FlexMeasures. - .. :quickref: Schedule; Download schedule from the platform + .. :quickref: Schedule; Download schedule for one device **Optional fields** From f7e2843cabcb1f5c08638d88949ba00c18f67972 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:18:30 +0200 Subject: [PATCH 093/162] docs: update note Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 32593ae881..2febb0e5a1 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -411,9 +411,8 @@ def trigger_schedule( For details on flexibility model and context, see :ref:`describing_flexibility`. Below, we'll also list some examples. - .. note:: This endpoint does not support to schedule an EMS with multiple flexible sensors at once. This will happen in another endpoint. - See https://github.com/FlexMeasures/flexmeasures/issues/485. Until then, it is possible to call this endpoint for one flexible endpoint at a time - (considering already scheduled sensors as inflexible). + .. note:: To schedule an EMS with multiple flexible sensors at once, + use `this endpoint <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ instead. The length of the schedule can be set explicitly through the 'duration' field. Otherwise, it is set by the config setting :ref:`planning_horizon_config`, which defaults to 48 hours. From afed80667c1c6711d0aa7a9b189681240f50d2b7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:19:51 +0200 Subject: [PATCH 094/162] docs: fix grammar Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 4da290b29e..f8a5c4c412 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -934,9 +934,9 @@ def trigger_schedule( For details on flexibility model and context, see :ref:`describing_flexibility`. Below, we'll also list some examples. - .. note:: This endpoint support scheduling an EMS with multiple flexible devices at once. - Internally, it can do so or jointly (the default) or sequentially - (considering already scheduled sensors as inflexible). + .. note:: This endpoint supports scheduling an EMS with multiple flexible devices at once. + It can do so jointly (the default) or sequentially + (considering previously scheduled sensors as inflexible). To use sequential scheduling, use ``sequential=true`` in the JSON body. The length of the schedule can be set explicitly through the 'duration' field. From 217a632ed5a21b47b8fa306260cb7edff31263be Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 15:20:17 +0200 Subject: [PATCH 095/162] docs: show different texts Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index d657ca5c27..f3e3e075eb 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -273,8 +273,8 @@ In our terminology, this is called the "flex model" and you can read more at :re Owners of such devices can post the flex model along with triggering the creation of a new schedule, to one of two endpoints: -1. `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ - for scheduling multiple devices -2. `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ - for scheduling a single device (which can also be done with the first endpoint) +1. `[POST] /assets//schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ - for scheduling multiple devices +2. `[POST] /sensors//schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ - for scheduling a single device (which can also be done with the first endpoint) The URL might look like this: From 7963c54d4de9cb60794c39b752c10227d7cd9111 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 16:02:30 +0200 Subject: [PATCH 096/162] docs: clarify assumed sensor/asset structure Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index f8a5c4c412..9bbd0c664e 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -923,7 +923,8 @@ def trigger_schedule( .. :quickref: Schedule; Trigger scheduling job for multiple devices Trigger FlexMeasures to create a schedule for this asset. - The assumption is that this is a flexible asset containing multiple power sensors. + The power sensors of flexible devices that are referenced in the flex-model must belong the given asset, + either directly or indirectly, by being assigned to one of the asset's (grand)children. In this request, you can describe: From f95dae6d2e1080f668fffbb09043a828cac9e61b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 16:04:21 +0200 Subject: [PATCH 097/162] docs: mention single-asset scheduling can still take into account inflexible devices Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 2febb0e5a1..37c405ddcc 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -395,7 +395,7 @@ def trigger_schedule( **kwargs, ): """ - Trigger FlexMeasures to create a schedule for a single flexible device. + Trigger FlexMeasures to create a schedule for a single flexible device, possibly taking into account inflexible devices. .. :quickref: Schedule; Trigger scheduling job for one device From 7162ff04772269989c2ddac5df8e4ac24776dce1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 16:42:09 +0200 Subject: [PATCH 098/162] docs: add code examples for API and flexmeasures-client for triggering a schedule, to the expanded toy tutorial Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 74 ++++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 5f084bb056..852e9a2ad1 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -77,13 +77,75 @@ Trigger an updated schedule Now, we'll reschedule the battery while taking into account the solar production. This will have an effect on the available headroom for the battery, given the ``site-power-capacity`` limit discussed earlier. -.. code-block:: bash +.. tabs:: + + .. tab:: CLI + + .. code-block:: bash + + $ flexmeasures add schedule for-storage --sensor 2 --consumption-price-sensor 1 \ + --inflexible-device-sensor 3 \ + --start ${TOMORROW}T07:00+02:00 --duration PT12H \ + --soc-at-start 50% --roundtrip-efficiency 90% + New schedule is stored. + + .. tab:: API + + Example POST call to http://localhost:5000/api/assets/2/schedules/trigger (replace the start date): + + .. code-block:: json + + { + "start": "2025-06-11T07:00+02:00", + "duration": "PT12H", + "flex-model": [ + "soc-at-start": "50%", + "roundtrip-efficiency": "90%" + ], + "flex-context": { + "inflexible-device-sensors": [3] + } + } + + .. tab:: flexmeasures-client + + Using the flexmeasures-client: + + .. code-block:: bash + + pip install flexmeasures-client + + .. code-block:: python + + import asyncio + from datetime import date + from flexmeasures_client import FlexMeasuresClient as Client + + async def client_script(): + client = Client( + email="toy-user@flexmeasures.io", + password="toy-password", + host="localhost:5000", + ) + schedule = await client.trigger_and_get_schedule( + asset_id=2, # Toy building (asset) + start=f"{date.today().isoformat()}T07:00+02:00", + duration="PT12H", + flex_model=[ + { + "soc-at-start": "50%", + "roundtrip-efficiency": "90%", + }, + ], + flex_context={ + "inflexible-device-sensors": [3], # solar production (sensor) + }, + ) + print(schedule) + await client.close() + + asyncio.run(client_script()) - $ flexmeasures add schedule for-storage --sensor 2 --consumption-price-sensor 1 \ - --inflexible-device-sensor 3 \ - --start ${TOMORROW}T07:00+02:00 --duration PT12H \ - --soc-at-start 50% --roundtrip-efficiency 90% - New schedule is stored. We can see the updated scheduling in the `FlexMeasures UI `_ : From 26f7f456a5179b4543f77440a1c31293aba100b8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 16:46:33 +0200 Subject: [PATCH 099/162] docs: add link to flexmeasures-client on PyPI Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 852e9a2ad1..45b11bcb51 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -109,7 +109,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: flexmeasures-client - Using the flexmeasures-client: + Using the `flexmeasures-client Date: Wed, 11 Jun 2025 17:14:06 +0200 Subject: [PATCH 100/162] docs: add missing sensor IDs to multi-asset flex-model Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 45b11bcb51..5f03ae2558 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -99,6 +99,7 @@ Now, we'll reschedule the battery while taking into account the solar production "start": "2025-06-11T07:00+02:00", "duration": "PT12H", "flex-model": [ + "sensor": 2, "soc-at-start": "50%", "roundtrip-efficiency": "90%" ], @@ -133,6 +134,7 @@ Now, we'll reschedule the battery while taking into account the solar production duration="PT12H", flex_model=[ { + "sensor": 2, "soc-at-start": "50%", "roundtrip-efficiency": "90%", }, From 56663216734a3b4a0b7e95d3a003172d957aeb84 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:16:11 +0200 Subject: [PATCH 101/162] docs: fix RST syntax Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 5f03ae2558..eb6f7303e5 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -63,7 +63,7 @@ Setting the data source type to "forecaster" helps FlexMeasures to visually dist $ flexmeasures add beliefs --sensor 3 --source 4 solar-tomorrow.csv --timezone Europe/Amsterdam Successfully created beliefs -The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. We can see solar production in the `FlexMeasures UI `_ : +The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. We can see solar production in the `FlexMeasures UI `_: .. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-production.png :align: center @@ -110,7 +110,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: flexmeasures-client - Using the `flexmeasures-client `_: .. code-block:: bash @@ -149,7 +149,7 @@ Now, we'll reschedule the battery while taking into account the solar production asyncio.run(client_script()) -We can see the updated scheduling in the `FlexMeasures UI `_ : +We can see the updated scheduling in the `FlexMeasures UI `_: .. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-charging-with-solar.png :align: center @@ -181,7 +181,7 @@ In the case of the scheduler that we ran in the previous tutorial, which did not .. note:: You can add arbitrary sensors to a chart using the asset UI or the attribute ``sensors_to_show``. See :ref:`view_asset-data` for more. -A nice feature is that you can check the data connectivity status of your building asset. Now that we have made the schedule, both lamps are green. You can also view it in `FlexMeasures UI `_ : +A nice feature is that you can check the data connectivity status of your building asset. Now that we have made the schedule, both lamps are green. You can also view it in `FlexMeasures UI `_: .. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/screenshot_building_status.png :align: center From 588ae1a7bfc32a4de238a4ae39f4d28b2e21a118 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:25:26 +0200 Subject: [PATCH 102/162] docs: improve capitalization Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 +- documentation/tut/toy-example-setup.rst | 2 +- flexmeasures/cli/data_show.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 30fc907f4e..69bbf92222 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -19,7 +19,7 @@ Make a schedule After going through the setup, we can finally create the schedule, which is the main benefit of FlexMeasures (smart real-time control). -We'll ask FlexMeasures for a schedule for our (dis)charging sensor (ID 2). We also need to specify what to optimize against. Here we pass the Id of our market price sensor (ID 1). +We'll ask FlexMeasures for a schedule for our (dis)charging sensor (ID 2). We also need to specify what to optimize against. Here we pass the ID of our market price sensor (ID 1). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). .. code-block:: bash diff --git a/documentation/tut/toy-example-setup.rst b/documentation/tut/toy-example-setup.rst index 0fdae58421..3fa8cce0da 100644 --- a/documentation/tut/toy-example-setup.rst +++ b/documentation/tut/toy-example-setup.rst @@ -210,7 +210,7 @@ If you want, you can inspect what you created: Child assets of toy-building (ID: 2) ==================================== - Id Name Type + ID Name Type ------- ----------------- ---------------------------- 3 toy-battery battery 4 toy-solar solar diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index aa09d90450..bc529aff2e 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -248,7 +248,7 @@ def show_generic_asset(asset): click.echo(f"Child assets of {asset.name} (ID: {asset.id})") click.echo(f"======{len(asset.name) * '='}===================\n") if child_asset_data: - click.echo(tabulate(child_asset_data, headers=["Id", "Name", "Type"])) + click.echo(tabulate(child_asset_data, headers=["ID", "Name", "Type"])) else: click.secho("No children assets ...", **MsgStyle.WARN) From 67d3f051c9293649067f50fd5ca339927c600b75 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:29:41 +0200 Subject: [PATCH 103/162] docs: add code examples for API and flexmeasures-client for triggering a schedule, to the toy-example-from-scratch tutorial Signed-off-by: F.N. Claessen --- .../tut/toy-example-from-scratch.rst | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 69bbf92222..8f631acfd2 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -22,11 +22,69 @@ After going through the setup, we can finally create the schedule, which is the We'll ask FlexMeasures for a schedule for our (dis)charging sensor (ID 2). We also need to specify what to optimize against. Here we pass the ID of our market price sensor (ID 1). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). -.. code-block:: bash +.. tabs:: + + .. tab:: CLI + + .. code-block:: bash + + $ flexmeasures add schedule for-storage --sensor 2 --start ${TOMORROW}T07:00+01:00 --duration PT12H \ + --soc-at-start 50% --roundtrip-efficiency 90% + New schedule is stored. + + .. tab:: API + + Example POST call to http://localhost:5000/api/assets/2/schedules/trigger (replace the start date): + + .. code-block:: json + + { + "start": "2025-06-11T07:00+01:00", + "duration": "PT12H", + "flex-model": [ + "sensor": 2, + "soc-at-start": "50%", + "roundtrip-efficiency": "90%" + ] + } + + .. tab:: flexmeasures-client + + Using the `flexmeasures-client `_: + + .. code-block:: bash + + pip install flexmeasures-client + + .. code-block:: python + + import asyncio + from datetime import date + from flexmeasures_client import FlexMeasuresClient as Client + + async def client_script(): + client = Client( + email="toy-user@flexmeasures.io", + password="toy-password", + host="localhost:5000", + ) + schedule = await client.trigger_and_get_schedule( + asset_id=2, # Toy building (asset) + start=f"{date.today().isoformat()}T07:00+01:00", + duration="PT12H", + flex_model=[ + { + "sensor": 2, + "soc-at-start": "50%", + "roundtrip-efficiency": "90%", + }, + ], + ) + print(schedule) + await client.close() + + asyncio.run(client_script()) - $ flexmeasures add schedule for-storage --sensor 2 --start ${TOMORROW}T07:00+01:00 --duration PT12H \ - --soc-at-start 50% --roundtrip-efficiency 90% - New schedule is stored. Great. Let's see what we made: From 9fb381e001e7fa168bbb4c552c2650d58e41e98c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:30:28 +0200 Subject: [PATCH 104/162] docs: sync timezone offsets across tutorials Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index eb6f7303e5..415821b883 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -85,7 +85,7 @@ Now, we'll reschedule the battery while taking into account the solar production $ flexmeasures add schedule for-storage --sensor 2 --consumption-price-sensor 1 \ --inflexible-device-sensor 3 \ - --start ${TOMORROW}T07:00+02:00 --duration PT12H \ + --start ${TOMORROW}T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% New schedule is stored. @@ -96,7 +96,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. code-block:: json { - "start": "2025-06-11T07:00+02:00", + "start": "2025-06-11T07:00+01:00", "duration": "PT12H", "flex-model": [ "sensor": 2, @@ -130,7 +130,7 @@ Now, we'll reschedule the battery while taking into account the solar production ) schedule = await client.trigger_and_get_schedule( asset_id=2, # Toy building (asset) - start=f"{date.today().isoformat()}T07:00+02:00", + start=f"{date.today().isoformat()}T07:00+01:00", duration="PT12H", flex_model=[ { From 650077b6141a1cd76b3f6f0b249f97d44c5fab6c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:41:41 +0200 Subject: [PATCH 105/162] docs: fix flex-model in API example Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 415821b883..22b76b4b5b 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -99,9 +99,11 @@ Now, we'll reschedule the battery while taking into account the solar production "start": "2025-06-11T07:00+01:00", "duration": "PT12H", "flex-model": [ - "sensor": 2, - "soc-at-start": "50%", - "roundtrip-efficiency": "90%" + { + "sensor": 2, + "soc-at-start": "50%", + "roundtrip-efficiency": "90%" + } ], "flex-context": { "inflexible-device-sensors": [3] From d877518f0aa10aaa9841bde479fae67766380f15 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 11 Jun 2025 17:43:33 +0200 Subject: [PATCH 106/162] docs: add multi-asset example for PV curtailment in API and flexmeasures-client Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 50 +++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 22b76b4b5b..838106dda6 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -110,6 +110,28 @@ Now, we'll reschedule the battery while taking into account the solar production } } + Alternatively, if the solar production is curtailable, move the solar production to the flex-model: + + .. code-block:: json + + { + "start": "2025-06-11T07:00+01:00", + "duration": "PT12H", + "flex-model": [ + { + "sensor": 2, + "soc-at-start": "50%", + "roundtrip-efficiency": "90%" + }, + { + "sensor": 3, + "consumption-capacity": "0 kW", + "production-capacity": {"sensor": 3}, + } + ], + "flex-context": {} + } + .. tab:: flexmeasures-client Using the `flexmeasures-client `_: @@ -131,12 +153,12 @@ Now, we'll reschedule the battery while taking into account the solar production host="localhost:5000", ) schedule = await client.trigger_and_get_schedule( - asset_id=2, # Toy building (asset) + asset_id=2, # Toy building (asset ID) start=f"{date.today().isoformat()}T07:00+01:00", duration="PT12H", flex_model=[ { - "sensor": 2, + "sensor": 2, # battery power (sensor ID) "soc-at-start": "50%", "roundtrip-efficiency": "90%", }, @@ -150,6 +172,30 @@ Now, we'll reschedule the battery while taking into account the solar production asyncio.run(client_script()) + Alternatively, if the solar production is curtailable, move the solar production to the flex-model: + + .. code-block:: python + + schedule = await client.trigger_and_get_schedule( + asset_id=2, # Toy building (asset ID) + start=f"{date.today().isoformat()}T07:00+01:00", + duration="PT12H", + flex_model=[ + { + "sensor": 2, # battery power (sensor ID) + "soc-at-start": "50%", + "roundtrip-efficiency": "90%", + }, + { + "sensor": 3, # solar production (sensor ID) + "consumption-capacity": "0 kW", + "production-capacity": {"sensor": 3}, + }, + ], + flex_context={}, + ) + + We can see the updated scheduling in the `FlexMeasures UI `_: From 43e3b4ad14ac02713f6865edaa56bbca6b105aa4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 09:41:28 +0200 Subject: [PATCH 107/162] fix: serializable job kwargs Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 28 +++++++++++++++++++----- flexmeasures/data/services/utils.py | 20 ----------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index c109285bc3..7c85756150 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -5,7 +5,6 @@ from __future__ import annotations from datetime import datetime, timedelta -import json import os import sys import importlib.util @@ -41,7 +40,6 @@ get_asset_or_sensor_ref, get_asset_or_sensor_from_ref, get_scheduler_instance, - json_isoformat, ) @@ -125,6 +123,17 @@ def trigger_optional_fallback(job, connection, type, value, traceback): scheduler_kwargs = job.meta["scheduler_kwargs"] + # Deserialize start and end + timezone = "UTC" + if hasattr(asset_or_sensor, "timezone"): + timezone = asset_or_sensor.timezone + scheduler_kwargs["start"] = pd.Timestamp(scheduler_kwargs["start"]).tz_convert( + timezone + ) + scheduler_kwargs["end"] = pd.Timestamp(scheduler_kwargs["end"]).tz_convert( + timezone + ) + if ("scheduler_specs" in job.kwargs) and ( job.kwargs["scheduler_specs"] is not None ): @@ -203,7 +212,7 @@ def create_scheduling_job( """ # We first create a scheduler and check if deserializing works, so the flex config is checked # and errors are raised before the job is enqueued (so users get a meaningful response right away). - # Note: We are putting still serialized scheduler_kwargs into the job! + # Note: We should put only serializable scheduler_kwargs into the job! if sensor is not None: current_app.logger.warning( @@ -249,9 +258,16 @@ def create_scheduling_job( ) job.meta["asset_or_sensor"] = asset_or_sensor - job.meta["scheduler_kwargs"] = json.loads( - json.dumps(scheduler_kwargs, default=json_isoformat) - ) + job.meta["scheduler_kwargs"] = scheduler_kwargs + + # Serialize start and end + job.meta["scheduler_kwargs"]["start"] = job.meta["scheduler_kwargs"][ + "start" + ].isoformat() + job.meta["scheduler_kwargs"]["end"] = job.meta["scheduler_kwargs"][ + "end" + ].isoformat() + job.save_meta() # in case the function enqueues it diff --git a/flexmeasures/data/services/utils.py b/flexmeasures/data/services/utils.py index f9b3bee16d..689190f370 100644 --- a/flexmeasures/data/services/utils.py +++ b/flexmeasures/data/services/utils.py @@ -259,23 +259,3 @@ def sort_jobs(queue: Queue, jobs: list[str | Job]) -> list[Job]: ] # Remove any None entries (in case some jobs don’t exist) sorted_jobs = sorted(jobs, key=lambda job: job.created_at) return sorted_jobs - - -def json_isoformat(obj): - """JSON serializer that attempts to isoformat objects that aren't serializable by default. - - Especially suitable for timedelta and datetime objects. - - Usage: - - json.dumps(dict_with_various_objects, default=json_isoformat) - - """ - - try: - if hasattr(obj, "isoformat"): - return obj.isoformat() - else: - return str(obj) - except Exception: - raise TypeError(f"Object of type {type(obj)} is not JSON serializable") From e9c65a52b5b6d9805e0e904e6aedab3a8c9b4c16 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 10:28:33 +0200 Subject: [PATCH 108/162] docs: add inline notes explaining the IDs in the flexmeasures-client examples Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 838106dda6..b8d63a7d51 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -164,7 +164,7 @@ Now, we'll reschedule the battery while taking into account the solar production }, ], flex_context={ - "inflexible-device-sensors": [3], # solar production (sensor) + "inflexible-device-sensors": [3], # solar production (sensor ID) }, ) print(schedule) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 8f631acfd2..09fde96dd8 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -69,12 +69,12 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, host="localhost:5000", ) schedule = await client.trigger_and_get_schedule( - asset_id=2, # Toy building (asset) + asset_id=2, # Toy building (asset ID) start=f"{date.today().isoformat()}T07:00+01:00", duration="PT12H", flex_model=[ { - "sensor": 2, + "sensor": 2, # battery power (sensor ID) "soc-at-start": "50%", "roundtrip-efficiency": "90%", }, From acbc49bba7899fadf2e8ab9088da842b0c957eca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 10:42:33 +0200 Subject: [PATCH 109/162] docs: fix url to trigger endpoint Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index b8d63a7d51..5f217f4c88 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -91,7 +91,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: API - Example POST call to http://localhost:5000/api/assets/2/schedules/trigger (replace the start date): + Example POST call to http://localhost:5000/api/v3_0/assets/2/schedules/trigger (replace the start date): .. code-block:: json diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 09fde96dd8..bd9ec26ee3 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -34,7 +34,7 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, .. tab:: API - Example POST call to http://localhost:5000/api/assets/2/schedules/trigger (replace the start date): + Example POST call to http://localhost:5000/api/v3_0/assets/2/schedules/trigger (replace the start date): .. code-block:: json From 4bc892c2a7361aef1200d5ae42203257eee79d71 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 10:46:49 +0200 Subject: [PATCH 110/162] docs: link to the endpoint docs instead of the actual endpoint, which would just result in a 405 Method Not Allowed Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 5f217f4c88..a5fb1350f2 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -91,7 +91,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: API - Example POST call to http://localhost:5000/api/v3_0/assets/2/schedules/trigger (replace the start date): + Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (replace the start date): .. code-block:: json diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index bd9ec26ee3..a0707c8022 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -34,7 +34,7 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, .. tab:: API - Example POST call to http://localhost:5000/api/v3_0/assets/2/schedules/trigger (replace the start date): + Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (replace the start date): .. code-block:: json From de3574741fdcfb65d207f599d49188807245f6d8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 10:47:59 +0200 Subject: [PATCH 111/162] docs: better phrasing Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- documentation/tut/toy-example-from-scratch.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index a5fb1350f2..4e0fb29dbe 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -91,7 +91,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: API - Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (replace the start date): + Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (update the start date to tomorrow): .. code-block:: json diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index a0707c8022..f357ecd2e4 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -34,7 +34,7 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, .. tab:: API - Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (replace the start date): + Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (update the start date to tomorrow): .. code-block:: json From 1bd3b5a2f552ce7d13c217c7a93f1a5d79a59384 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:12:19 +0200 Subject: [PATCH 112/162] style: emphasize new lines with respect to previous examples Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 4e0fb29dbe..d1672c03f2 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -82,6 +82,7 @@ Now, we'll reschedule the battery while taking into account the solar production .. tab:: CLI .. code-block:: bash + :emphasize-lines: 2 $ flexmeasures add schedule for-storage --sensor 2 --consumption-price-sensor 1 \ --inflexible-device-sensor 3 \ @@ -94,6 +95,7 @@ Now, we'll reschedule the battery while taking into account the solar production Example call: `[POST] http://localhost:5000/api/v3_0/assets/2/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ (update the start date to tomorrow): .. code-block:: json + :emphasize-lines: 11-13 { "start": "2025-06-11T07:00+01:00", @@ -113,6 +115,7 @@ Now, we'll reschedule the battery while taking into account the solar production Alternatively, if the solar production is curtailable, move the solar production to the flex-model: .. code-block:: json + :emphasize-lines: 10-14,16 { "start": "2025-06-11T07:00+01:00", @@ -141,6 +144,7 @@ Now, we'll reschedule the battery while taking into account the solar production pip install flexmeasures-client .. code-block:: python + :emphasize-lines: 22-24 import asyncio from datetime import date @@ -175,6 +179,7 @@ Now, we'll reschedule the battery while taking into account the solar production Alternatively, if the solar production is curtailable, move the solar production to the flex-model: .. code-block:: python + :emphasize-lines: 11-15,17 schedule = await client.trigger_and_get_schedule( asset_id=2, # Toy building (asset ID) From 48aaf0a54799cd1ca8ccdab604f846afe8c63ee5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:15:33 +0200 Subject: [PATCH 113/162] docs: add comment hinting at the parameter name we are introducing in the example code Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index d1672c03f2..92b0f759bb 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -75,7 +75,8 @@ The one-hour CSV data is automatically resampled to the 15-minute resolution of Trigger an updated schedule ---------------------------- -Now, we'll reschedule the battery while taking into account the solar production. This will have an effect on the available headroom for the battery, given the ``site-power-capacity`` limit discussed earlier. +Now, we'll reschedule the battery while taking into account the solar production as an inflexible device. +This will have an effect on the available headroom for the battery, given the ``site-power-capacity`` limit discussed earlier. .. tabs:: From 29385fc9209f3dab9f992e0d73b0591e6fbe086d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:18:05 +0200 Subject: [PATCH 114/162] style: use a new line for each option Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 12 ++++++++---- documentation/tut/toy-example-from-scratch.rst | 8 ++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 92b0f759bb..04c30b7591 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -83,12 +83,16 @@ This will have an effect on the available headroom for the battery, given the `` .. tab:: CLI .. code-block:: bash - :emphasize-lines: 2 + :emphasize-lines: 4 - $ flexmeasures add schedule for-storage --sensor 2 --consumption-price-sensor 1 \ + $ flexmeasures add schedule for-storage \ + --sensor 2 \ + --consumption-price-sensor 1 \ --inflexible-device-sensor 3 \ - --start ${TOMORROW}T07:00+01:00 --duration PT12H \ - --soc-at-start 50% --roundtrip-efficiency 90% + --start ${TOMORROW}T07:00+01:00 \ + --duration PT12H \ + --soc-at-start 50% \ + --roundtrip-efficiency 90% New schedule is stored. .. tab:: API diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index f357ecd2e4..22d8136dc3 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -28,8 +28,12 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, .. code-block:: bash - $ flexmeasures add schedule for-storage --sensor 2 --start ${TOMORROW}T07:00+01:00 --duration PT12H \ - --soc-at-start 50% --roundtrip-efficiency 90% + $ flexmeasures add schedule for-storage \ + --sensor 2 \ + --start ${TOMORROW}T07:00+01:00 \ + --duration PT12H \ + --soc-at-start 50% \ + --roundtrip-efficiency 90% New schedule is stored. .. tab:: API From 9df13929967ab3ccec0d236af70b1d8915b9a730 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:50:33 +0200 Subject: [PATCH 115/162] fix: remove outdated comment (the consumption-price sensor is preconfigured on asset already, since #1417) Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 22d8136dc3..952373a158 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -19,7 +19,7 @@ Make a schedule After going through the setup, we can finally create the schedule, which is the main benefit of FlexMeasures (smart real-time control). -We'll ask FlexMeasures for a schedule for our (dis)charging sensor (ID 2). We also need to specify what to optimize against. Here we pass the ID of our market price sensor (ID 1). +We'll ask FlexMeasures for a schedule for our (dis)charging sensor (ID 2). To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, the scheduler should know what the state of charge of the battery is when the schedule starts (50%) and what its roundtrip efficiency is (90%). .. tabs:: From 8c141efe555f9e8054d896d19b22383f02d7cd7d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:52:23 +0200 Subject: [PATCH 116/162] feat: simplify expanded CLI example (the consumption-price sensor is preconfigured on asset already) Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 04c30b7591..83d2918ec9 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -83,11 +83,10 @@ This will have an effect on the available headroom for the battery, given the `` .. tab:: CLI .. code-block:: bash - :emphasize-lines: 4 + :emphasize-lines: 3 $ flexmeasures add schedule for-storage \ --sensor 2 \ - --consumption-price-sensor 1 \ --inflexible-device-sensor 3 \ --start ${TOMORROW}T07:00+01:00 \ --duration PT12H \ From 075d61a80bdd974269005c93b7ce297f757aaf5d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:53:26 +0200 Subject: [PATCH 117/162] docs: refer to multiple schedules Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-setup.rst b/documentation/tut/toy-example-setup.rst index 3fa8cce0da..501309bb44 100644 --- a/documentation/tut/toy-example-setup.rst +++ b/documentation/tut/toy-example-setup.rst @@ -19,7 +19,7 @@ Below are the ``flexmeasures`` CLI commands we'll run, and which we'll explain s # setup an account with a user, assets for battery & solar and an energy market (ID 1) $ flexmeasures add toy-account - # load prices to optimise the schedule against + # load prices to optimize schedules against $ flexmeasures add beliefs --sensor 1 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam From e85d511b484e4df76b528ef52faf236a01d93f35 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 11:56:08 +0200 Subject: [PATCH 118/162] docs: add note regarding the lack of flex-context in the example Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-from-scratch.rst | 1 + documentation/tut/toy-example-setup.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index 952373a158..c3ce822f80 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -89,6 +89,7 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, asyncio.run(client_script()) +.. note:: We already specified what to optimize against by having set the consumption price sensor in the flex-context of the battery (see :ref:`tut_load_data`). Great. Let's see what we made: diff --git a/documentation/tut/toy-example-setup.rst b/documentation/tut/toy-example-setup.rst index 501309bb44..c07bd0d2de 100644 --- a/documentation/tut/toy-example-setup.rst +++ b/documentation/tut/toy-example-setup.rst @@ -131,6 +131,8 @@ Install Flexmeasures and the database .. include:: ../notes/macOS-port-note.rst +.. _tut_load_data: + Add some structural data --------------------------------------- From becdf12533e73c52f4667902b8e17bb5a3c6cb19 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:03:29 +0200 Subject: [PATCH 119/162] docs: highlight consumption-price in flex-context Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-setup.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/tut/toy-example-setup.rst b/documentation/tut/toy-example-setup.rst index c07bd0d2de..ea2eb028ce 100644 --- a/documentation/tut/toy-example-setup.rst +++ b/documentation/tut/toy-example-setup.rst @@ -197,6 +197,7 @@ If you want, you can inspect what you created: 4 toy-solar solar (52.374, 4.88969) .. code-block:: bash + :emphasize-lines: 30 $ flexmeasures show asset --id 2 From b874ddd5a73259b561a122e23ce454fdb76a75ea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:11:44 +0200 Subject: [PATCH 120/162] docs: stop using the --consumption-price-sensor option in the ProcessScheduler examples Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-process.rst | 6 +++--- flexmeasures/cli/data_add.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-process.rst b/documentation/tut/toy-example-process.rst index bcf9a93376..f75aefdacf 100644 --- a/documentation/tut/toy-example-process.rst +++ b/documentation/tut/toy-example-process.rst @@ -59,7 +59,7 @@ Now we are ready to schedule a process. Let's start with the INFLEXIBLE policy, .. code-block:: bash - flexmeasures add schedule for-process --sensor 4 --consumption-price-sensor 1\ + flexmeasures add schedule for-process --sensor 4 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type INFLEXIBLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" @@ -70,7 +70,7 @@ Following the INFLEXIBLE policy, we'll schedule the same 4h block using a BREAKA .. code-block:: bash - flexmeasures add schedule for-process --sensor 5 --consumption-price-sensor 1\ + flexmeasures add schedule for-process --sensor 5 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type BREAKABLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" @@ -81,7 +81,7 @@ Finally, we'll schedule the process using the SHIFTABLE policy. .. code-block:: bash - flexmeasures add schedule for-process --sensor 6 --consumption-price-sensor 1\ + flexmeasures add schedule for-process --sensor 6 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type SHIFTABLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index dced19df6a..e5865bdd4c 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -2178,18 +2178,21 @@ def create_asset_with_one_sensor( "toy-process", "process", "Power (Inflexible)", + flex_context={"consumption-price": {"sensor": day_ahead_sensor.id}}, ) breakable_power = create_asset_with_one_sensor( "toy-process", "process", "Power (Breakable)", + flex_context={"consumption-price": {"sensor": day_ahead_sensor.id}}, ) shiftable_power = create_asset_with_one_sensor( "toy-process", "process", "Power (Shiftable)", + flex_context={"consumption-price": {"sensor": day_ahead_sensor.id}}, ) db.session.flush() From 99eae7545f05fa646e9495e752a42099259073e8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:15:21 +0200 Subject: [PATCH 121/162] docs: reduce indentation of CLI command output Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-process.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/documentation/tut/toy-example-process.rst b/documentation/tut/toy-example-process.rst index f75aefdacf..8623ae2647 100644 --- a/documentation/tut/toy-example-process.rst +++ b/documentation/tut/toy-example-process.rst @@ -36,15 +36,15 @@ Before moving forward, we'll add the `process` asset and three sensors to store $ flexmeasures add toy-account --kind process - User with email toy-user@flexmeasures.io already exists in account Docker Toy Account. - The sensor recording day-ahead prices is day-ahead prices (ID: 1). - Created - Created - Created - Created - The sensor recording the power of the inflexible load is Power (Inflexible) (ID: 4). - The sensor recording the power of the breakable load is Power (Breakable) (ID: 5). - The sensor recording the power of the shiftable load is Power (Shiftable) (ID: 6). + User with email toy-user@flexmeasures.io already exists in account Docker Toy Account. + The sensor recording day-ahead prices is day-ahead prices (ID: 1). + Created + Created + Created + Created + The sensor recording the power of the inflexible load is Power (Inflexible) (ID: 4). + The sensor recording the power of the breakable load is Power (Breakable) (ID: 5). + The sensor recording the power of the shiftable load is Power (Shiftable) (ID: 6). Trigger an updated schedule From 8040e90dcf72171f632bc510d42a067907ee81a7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:16:49 +0200 Subject: [PATCH 122/162] docs: add missing $ Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-process.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/tut/toy-example-process.rst b/documentation/tut/toy-example-process.rst index 8623ae2647..f293a4fa58 100644 --- a/documentation/tut/toy-example-process.rst +++ b/documentation/tut/toy-example-process.rst @@ -59,7 +59,7 @@ Now we are ready to schedule a process. Let's start with the INFLEXIBLE policy, .. code-block:: bash - flexmeasures add schedule for-process --sensor 4 \ + $ flexmeasures add schedule for-process --sensor 4 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type INFLEXIBLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" @@ -70,7 +70,7 @@ Following the INFLEXIBLE policy, we'll schedule the same 4h block using a BREAKA .. code-block:: bash - flexmeasures add schedule for-process --sensor 5 \ + $ flexmeasures add schedule for-process --sensor 5 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type BREAKABLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" @@ -81,7 +81,7 @@ Finally, we'll schedule the process using the SHIFTABLE policy. .. code-block:: bash - flexmeasures add schedule for-process --sensor 6 \ + $ flexmeasures add schedule for-process --sensor 6 \ --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --process-duration PT4H \ --process-power 0.2MW --process-type SHIFTABLE \ --forbid "{\"start\" : \"${TOMORROW}T15:00:00+02:00\", \"duration\" : \"PT1H\"}" From 07dc5bf27d50fc0f68ad7862e205e5ac3a57e562 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:27:19 +0200 Subject: [PATCH 123/162] docs: rerun some tutorial steps to get the right indentation (updating the datetimes in the examples is a nice bonus) Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-reporter.rst | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/documentation/tut/toy-example-reporter.rst b/documentation/tut/toy-example-reporter.rst index 9fb895f9fb..d6a73f5ca8 100644 --- a/documentation/tut/toy-example-reporter.rst +++ b/documentation/tut/toy-example-reporter.rst @@ -48,29 +48,29 @@ Run the command below to show the values for our newly-created `grid connection $ flexmeasures show beliefs --sensor 7 --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --resolution PT1H Beliefs for Sensor 'grid connection capacity' (ID 7). - Data spans a day and starts at 2023-08-14 00:00:00+02:00. - The time resolution (x-axis) is an hour. - ┌────────────────────────────────────────────────────────────┐ - │ │ - │ │ - │ │ - │ │ - │ │ 1.0MW - │ │ - │ │ - │ │ - │▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀│ 0.5MW - │ │ - │ │ - │ │ - │▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁│ 0.0MW - │ │ - │ │ - │ │ - │ │ -0.5MW - └────────────────────────────────────────────────────────────┘ - 5 10 15 20 - ██ grid connection capacity + Data spans a day and starts at 2025-06-13 00:00:00+02:00. + The time resolution (x-axis) is an hour. + ┌────────────────────────────────────────────────────────────┐ + │ │ + │ │ + │ │ + │ │ + │ │ 1.0MW + │ │ + │ │ + │ │ + │▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄│ 0.5MW + │ │ + │ │ + │ │ + │▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁│ 0.0MW + │ │ + │ │ + │ │ + │ │ -0.5MW + └────────────────────────────────────────────────────────────┘ + 06:00 12:00 18:00 + ██ grid connection capacity (toy-building) Moreover, we can check the freshly created source ``, which defines the `ProfitOrLossReporter` with the required configuration. @@ -81,19 +81,19 @@ That's because reporters belong to a bigger category of classes that also contai $ flexmeasures show data-sources --show-attributes --id 6 - type: reporter - ======== - - ID Name User ID Model Version Attributes - ---- ------------ --------- -------------------- --------- ------------------------------------------ - 6 FlexMeasures ProfitOrLossReporter { - "data_generator": { - "config": { - "consumption_price_sensor": 1, - "loss_is_positive": true - } + type: reporter + ======== + + ID Name User ID Model Version Attributes + ---- ------------ --------- -------------------- --------- ------------------------------------------ + 6 FlexMeasures ProfitOrLossReporter { + "data_generator": { + "config": { + "consumption_price_sensor": 1, + "loss_is_positive": true } } + } Compute headroom From 903b9c76b01420f7b187b53282b65fe23d1cc499 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 12 Jun 2025 12:52:39 +0200 Subject: [PATCH 124/162] docs: fix header navigation Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-process.rst | 2 +- documentation/tut/toy-example-reporter.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/toy-example-process.rst b/documentation/tut/toy-example-process.rst index f293a4fa58..3fcf9ce440 100644 --- a/documentation/tut/toy-example-process.rst +++ b/documentation/tut/toy-example-process.rst @@ -27,7 +27,7 @@ Moreover, we'll touch upon the use of time restrictions to avoid scheduling a pr Setup -..... +----- Before moving forward, we'll add the `process` asset and three sensors to store the schedules resulting from following three different policies. diff --git a/documentation/tut/toy-example-reporter.rst b/documentation/tut/toy-example-reporter.rst index d6a73f5ca8..bcf968e784 100644 --- a/documentation/tut/toy-example-reporter.rst +++ b/documentation/tut/toy-example-reporter.rst @@ -22,7 +22,7 @@ In the second part, we'll use the `ProfitOrLossReporter` to compute the costs of Before getting to the meat of the tutorial, we need to set up up all the entities. Instead of having to do that manually (e.g. using commands such as ``flexmeasures add sensor``), we have prepared a command that does that automatically. Setup -..... +----- Just as in previous sections, we need to run the command ``flexmeasures add toy-account``, but this time with a different value for *kind*: From b130b1e76ca7a45b6e9d80a56a630253d51654ae Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 09:40:06 +0200 Subject: [PATCH 125/162] docs: add example setting Signed-off-by: F.N. Claessen --- documentation/tut/posting_data.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index f3e3e075eb..04a5786908 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -302,7 +302,15 @@ The endpoint also allows to limit the flexibility range and also to set target v .. note:: Flexibility states posted in trigger messages are only stored temporarily to describe the scheduling job. - To record a more complete history of the state of charge, set up a separate sensor and post data to it using `[POST] /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ (see :ref:`posting_sensor_data`). - Then reference that sensor in your flex model. + To record a more complete history of the flexibility state, set up separate sensors and post data to them using `[POST] /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ (see :ref:`posting_sensor_data`). + Then reference those sensors in your flex model. + For example, say you use sensor 82 to record the power-to-heat efficiency of a heating system, then use this sensor reference in your flex model: + + .. code-block:: json + + { + "charging-efficiency": {"sensor": 82} + } + In :ref:`how_queue_scheduling`, we'll cover what happens when FlexMeasures is triggered to create a new schedule, and how those schedules can be retrieved via the API, so they can be used to steer assets. \ No newline at end of file From 822516c97d1f10c694f1f8dbf5110617f78c2926 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 10:13:22 +0200 Subject: [PATCH 126/162] docs: suggested changes Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 2 +- flexmeasures/api/v3_0/assets.py | 14 ++++++++------ flexmeasures/api/v3_0/sensors.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index 83d2918ec9..f4f8db92a0 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -75,7 +75,7 @@ The one-hour CSV data is automatically resampled to the 15-minute resolution of Trigger an updated schedule ---------------------------- -Now, we'll reschedule the battery while taking into account the solar production as an inflexible device. +Now, we'll reschedule the battery while taking into account the solar production (forecast) as an inflexible device. This will have an effect on the available headroom for the battery, given the ``site-power-capacity`` limit discussed earlier. .. tabs:: diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 9bbd0c664e..0d5d12890b 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -923,7 +923,7 @@ def trigger_schedule( .. :quickref: Schedule; Trigger scheduling job for multiple devices Trigger FlexMeasures to create a schedule for this asset. - The power sensors of flexible devices that are referenced in the flex-model must belong the given asset, + The flex-model references the power sensors of flexible devices, which must belong to the given asset, either directly or indirectly, by being assigned to one of the asset's (grand)children. In this request, you can describe: @@ -953,13 +953,15 @@ def trigger_schedule( **Example request** - This message triggers a schedule for a storage asset, starting at 10.00am, at which the state of charge (soc) is 12.1 kWh, - together with a curtailable production asset, whose production forecasts are recorded under sensor 760. + This message triggers a schedule for a storage asset (with power sensor 931), + starting at 10.00am, when the state of charge (soc) should be assumed to be 12.1 kWh, + and also schedules a curtailable production asset (with power sensor 932), + whose production forecasts are recorded under sensor 760. Aggregate consumption (of all devices within this EMS) should be priced by sensor 9, and aggregate production should be priced by sensor 10, - where the aggregate power flow in the EMS is described by the sum over sensors 13, 14 and 15 - (plus the two sensors for the flexible devices being optimized, of course). + where the aggregate power flow in the EMS is described by the sum over sensors 13, 14, 15, + and the two power sensors (931 and 932) of the flexible devices being optimized (referenced in the flex-model). The battery consumption power capacity is limited by sensor 42 and the production capacity is constant (30 kW). Finally, the site consumption capacity is limited by sensor 32. @@ -978,7 +980,7 @@ def trigger_schedule( "production-capacity" : "30 kW" }, { - "sensor": 760, + "sensor": 932, "consumption-capacity": "0 kW", "production-capacity": {"sensor": 760}, } diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 37c405ddcc..7636899879 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -450,8 +450,8 @@ def trigger_schedule( Storage efficiency is set to 99.99%, denoting the state of charge left after each time step equal to the sensor's resolution. Aggregate consumption (of all devices within this EMS) should be priced by sensor 9, and aggregate production should be priced by sensor 10, - where the aggregate power flow in the EMS is described by the sum over sensors 13, 14 and 15 - (plus the flexible sensor being optimized, of course). + where the aggregate power flow in the EMS is described by the sum over sensors 13, 14, 15, + and the power sensor of the flexible device being optimized (referenced in the endpoint URL). The battery consumption power capacity is limited by sensor 42 and the production capacity is constant (30 kW). From 84f9cf988af1bf40cb0cddd9180d9804c072b3e9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 10:24:23 +0200 Subject: [PATCH 127/162] docs: clarify curtailment model Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index f4f8db92a0..cb042eac27 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -116,7 +116,8 @@ This will have an effect on the available headroom for the battery, given the `` } } - Alternatively, if the solar production is curtailable, move the solar production to the flex-model: + Alternatively, if the solar production is curtailable, move the solar production to the flex-model. + There, we tell the scheduler to pick any production value between 0 and the production forecast recorded on sensor 3, and to store the resulting schedule on sensor 3 as well (the FlexMeasures UI will still be able to distinguish forecasts from schedules): .. code-block:: json :emphasize-lines: 10-14,16 From 81a54e03e082e40ee9e775dd1fc606d5885a19c9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 10:33:35 +0200 Subject: [PATCH 128/162] docs: rewrite quickref Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 0d5d12890b..81d8c8f6c2 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -920,7 +920,7 @@ def trigger_schedule( """ Trigger FlexMeasures to create a schedule for a collection of flexible and inflexible devices. - .. :quickref: Schedule; Trigger scheduling job for multiple devices + .. :quickref: Schedule; Trigger scheduling job for any number of devices Trigger FlexMeasures to create a schedule for this asset. The flex-model references the power sensors of flexible devices, which must belong to the given asset, From 71dff5fdf111561d03bf309267e6cdd2d60bf096 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 10:37:13 +0200 Subject: [PATCH 129/162] fix: revert accidental revision from resolving merge conflicts Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 7636899879..69073e576d 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -555,6 +555,8 @@ def trigger_schedule( except ValueError as err: return invalid_flex_config(str(err)) + db.session.commit() + response = dict(schedule=job.id) d, s = request_processed() return dict(**response, **d), s From ca6bb86d5826569f1945de204797e33aeacc6fac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 15:41:54 +0200 Subject: [PATCH 130/162] feat: check resulting schedule Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index bc38123c30..dc84f1ef12 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -2,6 +2,7 @@ import pytest from isodate import parse_datetime, parse_duration +from numpy.testing import assert_equal import pandas as pd from rq.job import Job @@ -155,3 +156,74 @@ def test_asset_trigger_and_get_schedule( assert get_schedule_response.status_code == 200 # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" assert len(get_schedule_response.json["values"]) == expected_length_of_schedule + + if sequential: + if sensor_id == sensor_1.id: + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -0.047911, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, -0.0, -0.158368, -0.0, -0.0, # noqa: E128 + -0.0, 0.161632, 1.6e-05, 1.6e-05, 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + 0.161632, -0.158384, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, -0.0, 3.2e-05, -0.158384, -0.0, # noqa: E128 + -0.0, -0.0, 0.161632, -0.0, 3.2e-05, 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, # noqa: E128 + -0.158384, -0.0, -0.0, -0.0, 0.161632, -0.0, 3.2e-05, -0.0, -0.0, 4.8e-05, 1.6e-05, # noqa: E128 + 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, -0.0, -0.0, 4.8e-05, -0.0, 3.2e-05, # noqa: E128 + 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, # noqa: E128 + ] + # fmt: on + ) + else: + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -0.0, -0.0, -0.0, -0.0, 0.054017, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + ] + # fmt: on + ) + else: + if sensor_id == sensor_1.id: + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -1.323137, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -1.667577, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -1.309551, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + ] + # fmt: on + ) + else: + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + ] + # fmt: on + ) From 252af4c45dd45f6aa1d71c5a2a7643ff7fc190bb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 19:32:54 +0200 Subject: [PATCH 131/162] fix: extending scheduling horizon with simultaneous scheduling was silently failing Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 2 +- flexmeasures/data/models/planning/storage.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index dc84f1ef12..60ad9b99f2 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -141,7 +141,7 @@ def test_asset_trigger_and_get_schedule( for flex_model in message["flex-model"]: # We expect a longer schedule if the targets exceeds the original duration in the trigger - if sequential and "soc-targets" in flex_model: + if "soc-targets" in flex_model: for t in flex_model["soc-targets"]: duration = pd.Timestamp(t["datetime"]) - pd.Timestamp(message["start"]) if duration > pd.Timedelta(message["duration"]): diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 772f87e358..6a146258f8 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1058,8 +1058,8 @@ def deserialize_flex_config(self): # Extend schedule period in case a target exceeds its end self.possibly_extend_end( - soc_targets=sensor_flex_model.get("soc_targets"), - sensor=sensor_flex_model["sensor"], + soc_targets=self.flex_model[d].get("soc_targets"), + sensor=self.flex_model[d]["sensor"], ) else: From 0b098900c64559c7c3ceafcf9f4ed88c8ef1dfee Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Jun 2025 21:07:15 +0200 Subject: [PATCH 132/162] fix: update test accordingly Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 87 ++++++++++++++----- 1 file changed, 64 insertions(+), 23 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 60ad9b99f2..a4b6a30a83 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from flask import url_for import pytest from isodate import parse_datetime, parse_duration @@ -6,6 +8,7 @@ import pandas as pd from rq.job import Job +from flexmeasures import Sensor from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import ( @@ -137,16 +140,43 @@ def test_asset_trigger_and_get_schedule( scheduler_source = get_data_source_for_job(scheduling_job) assert scheduler_source is not None - # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint - for flex_model in message["flex-model"]: + def compute_expected_length( + message: dict, sensors: list[Sensor], sequential: bool + ) -> list[int]: + """We expect a longer schedule if the targets exceeds the original duration in the trigger. + + If the planning happens sequentially, individual schedules may be extended to accommodate for far-away targets. + If the planning happens jointly, we expect all schedules to be extended. + """ + expected_durations = [pd.Timedelta(message["duration"])] * len(sensors) + for d, (sensor, flex_model) in enumerate(zip(sensors, message["flex-model"])): + assert ( + flex_model["sensor"] == sensor.id + ), "make sure we are dealing with the assumed sensor" + if "soc-targets" in flex_model: + for t in flex_model["soc-targets"]: + duration = pd.Timestamp(t["datetime"]) - pd.Timestamp( + message["start"] + ) + if duration > expected_durations[d]: + if sequential: + expected_durations[d] = duration + else: + expected_durations = [duration] * len(sensors) + + # Convert duration to number of steps in the sensor's resolution + expected_lengths = [ + expected_durations[d] / sensor.event_resolution + for d, sensor in enumerate(sensors) + ] - # We expect a longer schedule if the targets exceeds the original duration in the trigger - if "soc-targets" in flex_model: - for t in flex_model["soc-targets"]: - duration = pd.Timestamp(t["datetime"]) - pd.Timestamp(message["start"]) - if duration > pd.Timedelta(message["duration"]): - expected_length_of_schedule = duration / resolution + return expected_lengths + sensors = [sensor_1, sensor_2] + expected_length_of_schedule = compute_expected_length(message, sensors, sequential) + + # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint + for d, flex_model in enumerate(message["flex-model"]): sensor_id = flex_model["sensor"] get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id), @@ -155,7 +185,9 @@ def test_asset_trigger_and_get_schedule( print("Server responded with:\n%s" % get_schedule_response.json) assert get_schedule_response.status_code == 200 # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" - assert len(get_schedule_response.json["values"]) == expected_length_of_schedule + assert ( + len(get_schedule_response.json["values"]) == expected_length_of_schedule[d] + ) if sequential: if sensor_id == sensor_1.id: @@ -202,13 +234,17 @@ def test_asset_trigger_and_get_schedule( get_schedule_response.json["values"], # fmt: off [ - -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -1.323137, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -1.667577, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -1.309551, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + 2.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, # noqa: E128 + -1.784316, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -1.694362, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 ] # fmt: on ) @@ -217,13 +253,18 @@ def test_asset_trigger_and_get_schedule( get_schedule_response.json["values"], # fmt: off [ - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + 2.0, 2.0, 1.749626, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) From 21d3f8194a11803ad924bc38bed21c68bf7fcc2f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 15 Jun 2025 22:06:17 +0200 Subject: [PATCH 133/162] fix: test charging during cheapest hour Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 66 +++++++++++-------- flexmeasures/data/models/planning/utils.py | 2 +- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index a4b6a30a83..4e07ce55e4 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -189,42 +189,54 @@ def compute_expected_length( len(get_schedule_response.json["values"]) == expected_length_of_schedule[d] ) + prices = add_market_prices_fresh_db["epex_da"].search_beliefs( + event_starts_after=message["start"], + event_ends_before=pd.Timestamp(message["start"]) + + pd.Timedelta(message["duration"]), + ) if sequential: if sensor_id == sensor_1.id: assert_equal( get_schedule_response.json["values"], # fmt: off [ - -0.047911, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, -0.0, -0.158368, -0.0, -0.0, # noqa: E128 - -0.0, 0.161632, 1.6e-05, 1.6e-05, 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - 0.161632, -0.158384, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, -0.0, 3.2e-05, -0.158384, -0.0, # noqa: E128 - -0.0, -0.0, 0.161632, -0.0, 3.2e-05, 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, # noqa: E128 - -0.158384, -0.0, -0.0, -0.0, 0.161632, -0.0, 3.2e-05, -0.0, -0.0, 4.8e-05, 1.6e-05, # noqa: E128 - 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, -0.0, -0.0, 4.8e-05, -0.0, 3.2e-05, # noqa: E128 - 1.6e-05, 1.6e-05, 1.6e-05, 1.6e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.047911, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, -0.0, 3.2e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, 0.161632, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161632, -0.158384, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, 0.161632, -0.158384, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 8.1e-05, -0.158384, -0.0, -0.0, -0.0, 0.161632, -0.0, 3.2e-05, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, 8.1e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 0.161632, -0.0, -0.0, 4.8e-05, -0.0, 3.2e-05, 1.6e-05, 1.6e-05, -0.0, -0.0, # noqa: E128 + 4.8e-05, -0.0, -0.158368, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) else: + assert ( + sum( + get_schedule_response.json["values"][ + prices.values.argmin() + * 4 : (prices.values.argmin() + 1) + * 4 + ] + ) + > 0 + ), "we expect to charge in the cheapest hour" assert_equal( get_schedule_response.json["values"], # fmt: off [ - -0.0, -0.0, -0.0, -0.0, 0.054017, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) @@ -253,18 +265,18 @@ def compute_expected_length( get_schedule_response.json["values"], # fmt: off [ - -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - 2.0, 2.0, 1.749626, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, 2.0, 2.0, 1.743315, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, # noqa: E128 + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index ee893f3cce..82e5046613 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -69,7 +69,7 @@ def initialize_index( def add_tiny_price_slope( - orig_prices: pd.DataFrame, col_name: str = "event_value", d: float = 10**-3 + orig_prices: pd.DataFrame, col_name: str = "event_value", d: float = 10**-4 ) -> pd.DataFrame: """Add tiny price slope to col_name to represent e.g. inflation as a simple linear price increase. This is meant to break ties, when multiple time slots have equal prices, in favour of acting sooner. From a2b134cae53dcb928ab28370be41a05885bb9db5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 12:55:36 +0200 Subject: [PATCH 134/162] fix: use soc-unit in multi-asset flex-models Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 6a146258f8..44243918d9 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1052,7 +1052,11 @@ def deserialize_flex_config(self): ) for d, sensor_flex_model in enumerate(self.flex_model): self.flex_model[d] = StorageFlexModelSchema( - start=self.start, sensor=sensor_flex_model["sensor"] + start=self.start, + sensor=sensor_flex_model["sensor"], + default_soc_unit=sensor_flex_model["sensor_flex_model"].get( + "soc-unit" + ), ).load(sensor_flex_model["sensor_flex_model"]) self.flex_model[d]["sensor"] = sensor_flex_model["sensor"] From 2174073344a2d425786a0d630a1aeb725ac74440 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 12:59:26 +0200 Subject: [PATCH 135/162] fix: revise test expectations accordingly Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 4e07ce55e4..6806e74cf4 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -184,6 +184,9 @@ def compute_expected_length( ) print("Server responded with:\n%s" % get_schedule_response.json) assert get_schedule_response.status_code == 200 + assert ( + get_schedule_response.json["unit"] == "MW" + ), "by default, the schedules are expected in the sensor unit" # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" assert ( len(get_schedule_response.json["values"]) == expected_length_of_schedule[d] @@ -200,13 +203,14 @@ def compute_expected_length( get_schedule_response.json["values"], # fmt: off [ - -0.047911, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, -0.0, 3.2e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, 0.161632, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161632, -0.158384, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, 0.161632, -0.158384, -0.0, -0.0, -0.0, 0.161632, 1.6e-05, 1.6e-05, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, 8.1e-05, -0.158384, -0.0, -0.0, -0.0, 0.161632, -0.0, 3.2e-05, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, 8.1e-05, -0.158384, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, 0.161632, -0.0, -0.0, 4.8e-05, -0.0, 3.2e-05, 1.6e-05, 1.6e-05, -0.0, -0.0, # noqa: E128 - 4.8e-05, -0.0, -0.158368, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.047515, -0.0, -0.0, -0.0, 0.161228, 1.6e-05, -0.0, 3.2e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, 0.161229, -0.157988, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157988, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, 0.000113, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, # noqa: E128 + 1.6e-05, -0.0, -0.0, 4.8e-05, -0.0, -0.157972, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161228, 1.6e-05, -0.0, 3.2e-05, -0.0, 3.2e-05, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, 8.1e-05, -0.0, -0.157972, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) @@ -246,37 +250,50 @@ def compute_expected_length( get_schedule_response.json["values"], # fmt: off [ - 2.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, 2.0, 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, # noqa: E128 - -1.784316, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -1.694362, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 - -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, -2.0, # noqa: E128 + -0.047515, -0.0, -0.0, -0.0, 0.161228, -0.0, -0.0, 4.8e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 + 0.161228, -0.0, -0.0, 4.9e-05, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, 1.6e-05, # noqa: E128 + -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, -0.0, 4.8e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 + 0.161228, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157877, -0.0, -0.0, -0.0, 0.161228, # noqa: E128 + -0.0, 3.2e-05, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157909, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, -0.0, -0.0, # noqa: E128 + -0.0, 6.5e-05, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157909, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, 0.161228, -0.0, -0.0, 4.9e-05, -0.157986, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157987, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 ] # fmt: on ) else: + assert ( + sum( + get_schedule_response.json["values"][ + prices.values.argmin() + * 4 : (prices.values.argmin() + 1) + * 4 + ] + ) + > 0 + ), "we expect to charge in the cheapest hour" assert_equal( get_schedule_response.json["values"], # fmt: off [ - -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, 2.0, 2.0, 1.743315, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 2.0, 2.0, # noqa: E128 - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, ] # fmt: on ) From a47efed264169708356759f97655028dd704192f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 13:05:02 +0200 Subject: [PATCH 136/162] refactor: use explanatory util variable Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_asset_schedules_fresh_db.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 6806e74cf4..fdf031ab95 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -197,6 +197,7 @@ def compute_expected_length( event_ends_before=pd.Timestamp(message["start"]) + pd.Timedelta(message["duration"]), ) + cheapest_hour = prices.values.argmin() if sequential: if sensor_id == sensor_1.id: assert_equal( @@ -218,9 +219,7 @@ def compute_expected_length( assert ( sum( get_schedule_response.json["values"][ - prices.values.argmin() - * 4 : (prices.values.argmin() + 1) - * 4 + cheapest_hour * 4 : (cheapest_hour + 1) * 4 ] ) > 0 @@ -271,9 +270,7 @@ def compute_expected_length( assert ( sum( get_schedule_response.json["values"][ - prices.values.argmin() - * 4 : (prices.values.argmin() + 1) - * 4 + cheapest_hour * 4 : (cheapest_hour + 1) * 4 ] ) > 0 From 0eabc38ee1926848be75eacedf092e86bdd4fef3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 13:11:10 +0200 Subject: [PATCH 137/162] refactor: remove obsolete argument Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 44243918d9..e52cebc707 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1117,15 +1117,14 @@ def get_min_max_targets( return min_target, max_target def get_min_max_soc_on_sensor( - self, adjust_unit: bool = False, deserialized_names: bool = True + self, adjust_unit: bool = False ) -> tuple[float | None, float | None]: soc_min_sensor: float | None = self.sensor.get_attribute("min_soc_in_mwh") soc_max_sensor: float | None = self.sensor.get_attribute("max_soc_in_mwh") - soc_unit_label = "soc_unit" if deserialized_names else "soc-unit" if adjust_unit: - if soc_min_sensor and self.flex_model.get(soc_unit_label) == "kWh": + if soc_min_sensor and self.flex_model.get("soc_unit") == "kWh": soc_min_sensor *= 1000 # later steps assume soc data is kWh - if soc_max_sensor and self.flex_model.get(soc_unit_label) == "kWh": + if soc_max_sensor and self.flex_model.get("soc_unit") == "kWh": soc_max_sensor *= 1000 return soc_min_sensor, soc_max_sensor @@ -1134,9 +1133,9 @@ def ensure_soc_min_max(self): Make sure we have min and max SOC. If not passed directly, then get default from sensor or targets. """ - _, max_target = self.get_min_max_targets(deserialized_names=False) + _, max_target = self.get_min_max_targets() soc_min_sensor, soc_max_sensor = self.get_min_max_soc_on_sensor( - adjust_unit=True, deserialized_names=False + adjust_unit=True ) if "soc-min" not in self.flex_model or self.flex_model["soc-min"] is None: # Default is 0 - can't drain the storage by more than it contains From 9aedfc6383e459cd4cea70f6e6e9f89ef8669e23 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 13:31:48 +0200 Subject: [PATCH 138/162] feat: check constraints Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 25 ++++++++++++++++++- .../data/models/planning/tests/utils.py | 14 +++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index fdf031ab95..f4c8cd14ca 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -10,12 +10,14 @@ from flexmeasures import Sensor from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule +from flexmeasures.data.models.planning.tests.utils import check_constraints from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import ( handle_scheduling_exception, get_data_source_for_job, ) from flexmeasures.data.services.utils import sort_jobs +from flexmeasures.utils.unit_utils import ur @pytest.mark.parametrize( @@ -176,7 +178,7 @@ def compute_expected_length( expected_length_of_schedule = compute_expected_length(message, sensors, sequential) # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint - for d, flex_model in enumerate(message["flex-model"]): + for d, (sensor, flex_model) in enumerate(zip(sensors, message["flex-model"])): sensor_id = flex_model["sensor"] get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id), @@ -192,6 +194,27 @@ def compute_expected_length( len(get_schedule_response.json["values"]) == expected_length_of_schedule[d] ) + check_constraints( + sensor=sensor, + schedule=pd.Series( + data=get_schedule_response.json["values"], + index=pd.date_range( + start=get_schedule_response.json["start"], + periods=len(get_schedule_response.json["values"]), + freq=sensor.event_resolution, + ), + ), + soc_at_start=flex_model["soc-at-start"], + soc_min=flex_model["soc-min"], + soc_max=flex_model["soc-max"], + roundtrip_efficiency=ur.Quantity(flex_model["roundtrip-efficiency"]) + .to("dimensionless") + .magnitude, + storage_efficiency=ur.Quantity(flex_model["storage-efficiency"]) + .to("dimensionless") + .magnitude, + ) + prices = add_market_prices_fresh_db["epex_da"].search_beliefs( event_starts_after=message["start"], event_ends_before=pd.Timestamp(message["start"]) diff --git a/flexmeasures/data/models/planning/tests/utils.py b/flexmeasures/data/models/planning/tests/utils.py index b72e5dd590..4e8127d7e1 100644 --- a/flexmeasures/data/models/planning/tests/utils.py +++ b/flexmeasures/data/models/planning/tests/utils.py @@ -27,6 +27,8 @@ def check_constraints( roundtrip_efficiency: float = 1, storage_efficiency: float = 1, tolerance: float = 0.00001, + soc_min: float | None = None, + soc_max: float | None = None, ) -> pd.Series: soc_schedule = integrate_time_series( schedule, @@ -45,8 +47,16 @@ def check_constraints( assert min(schedule.values) >= capacity * -1 - tolerance assert max(schedule.values) <= capacity + tolerance for soc in soc_schedule.values: - assert soc >= sensor.get_attribute("min_soc_in_mwh") - assert soc <= sensor.get_attribute("max_soc_in_mwh") + assert ( + soc >= soc_min + if soc_min is not None + else sensor.get_attribute("min_soc_in_mwh") + ) + assert ( + soc <= soc_max + if soc_max is not None + else sensor.get_attribute("max_soc_in_mwh") + ) return soc_schedule From 47fc431628093d88549cdfc84d98d9f71f1d5575 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 13:37:05 +0200 Subject: [PATCH 139/162] fix: only check the CP with a target Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 145 +++++++----------- 1 file changed, 53 insertions(+), 92 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index f4c8cd14ca..2008310d5f 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -221,99 +221,60 @@ def compute_expected_length( + pd.Timedelta(message["duration"]), ) cheapest_hour = prices.values.argmin() - if sequential: - if sensor_id == sensor_1.id: - assert_equal( - get_schedule_response.json["values"], - # fmt: off - [ - -0.047515, -0.0, -0.0, -0.0, 0.161228, 1.6e-05, -0.0, 3.2e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, 0.161229, -0.157988, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157988, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, 0.000113, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, # noqa: E128 - 1.6e-05, -0.0, -0.0, 4.8e-05, -0.0, -0.157972, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161228, 1.6e-05, -0.0, 3.2e-05, -0.0, 3.2e-05, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, 8.1e-05, -0.0, -0.157972, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, # noqa: E128 - ] - # fmt: on - ) - else: - assert ( - sum( - get_schedule_response.json["values"][ - cheapest_hour * 4 : (cheapest_hour + 1) * 4 - ] - ) - > 0 - ), "we expect to charge in the cheapest hour" - assert_equal( - get_schedule_response.json["values"], - # fmt: off - [ - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - ] - # fmt: on - ) - else: - if sensor_id == sensor_1.id: - assert_equal( - get_schedule_response.json["values"], - # fmt: off - [ - -0.047515, -0.0, -0.0, -0.0, 0.161228, -0.0, -0.0, 4.8e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 - 0.161228, -0.0, -0.0, 4.9e-05, -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, 1.6e-05, # noqa: E128 - -0.157988, -0.0, -0.0, -0.0, 0.161228, -0.0, -0.0, 4.8e-05, -0.157988, -0.0, -0.0, -0.0, # noqa: E128 - 0.161228, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157877, -0.0, -0.0, -0.0, 0.161228, # noqa: E128 - -0.0, 3.2e-05, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157909, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161228, -0.0, 3.2e-05, -0.0, -0.0, # noqa: E128 - -0.0, 6.5e-05, -0.0, -0.0, -0.0, -0.0, -0.0, -0.157909, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, 0.161228, -0.0, -0.0, 4.9e-05, -0.157986, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.161229, -0.157987, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + if sequential and sensor_id == sensor_2.id: + assert ( + sum( + get_schedule_response.json["values"][ + cheapest_hour * 4 : (cheapest_hour + 1) * 4 ] - # fmt: on ) - else: - assert ( - sum( - get_schedule_response.json["values"][ - cheapest_hour * 4 : (cheapest_hour + 1) * 4 - ] - ) - > 0 - ), "we expect to charge in the cheapest hour" - assert_equal( - get_schedule_response.json["values"], - # fmt: off - [ - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, + > 0 + ), "we expect to charge in the cheapest hour" + print(get_schedule_response.json["values"]) + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + ] + # fmt: on + ) + elif not sequential and sensor_id == sensor_2.id: + assert ( + sum( + get_schedule_response.json["values"][ + cheapest_hour * 4 : (cheapest_hour + 1) * 4 ] - # fmt: on ) + > 0 + ), "we expect to charge in the cheapest hour" + assert_equal( + get_schedule_response.json["values"], + # fmt: off + [ + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 + -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, + ] + # fmt: on + ) From 56f490b4ce486531745d23a822973392b3cf46d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 15:19:21 +0200 Subject: [PATCH 140/162] fix: if/else logic Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/utils.py b/flexmeasures/data/models/planning/tests/utils.py index 4e8127d7e1..b518d5f549 100644 --- a/flexmeasures/data/models/planning/tests/utils.py +++ b/flexmeasures/data/models/planning/tests/utils.py @@ -50,12 +50,12 @@ def check_constraints( assert ( soc >= soc_min if soc_min is not None - else sensor.get_attribute("min_soc_in_mwh") + else soc >= sensor.get_attribute("min_soc_in_mwh") ) assert ( soc <= soc_max if soc_max is not None - else sensor.get_attribute("max_soc_in_mwh") + else soc <= sensor.get_attribute("max_soc_in_mwh") ) return soc_schedule From 9354979ef672e27f5650ccf8e9dea00e767459e3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 15:20:08 +0200 Subject: [PATCH 141/162] refactor: simplify Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/utils.py b/flexmeasures/data/models/planning/tests/utils.py index b518d5f549..323b722c8e 100644 --- a/flexmeasures/data/models/planning/tests/utils.py +++ b/flexmeasures/data/models/planning/tests/utils.py @@ -47,15 +47,11 @@ def check_constraints( assert min(schedule.values) >= capacity * -1 - tolerance assert max(schedule.values) <= capacity + tolerance for soc in soc_schedule.values: - assert ( - soc >= soc_min - if soc_min is not None - else soc >= sensor.get_attribute("min_soc_in_mwh") + assert soc >= ( + soc_min if soc_min is not None else sensor.get_attribute("min_soc_in_mwh") ) - assert ( - soc <= soc_max - if soc_max is not None - else soc <= sensor.get_attribute("max_soc_in_mwh") + assert soc <= ( + soc_max if soc_max is not None else sensor.get_attribute("max_soc_in_mwh") ) return soc_schedule From 1cc657486f194124379271fe2c100d47166ad56e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 15:29:28 +0200 Subject: [PATCH 142/162] refactor: remove obsolete argument (second part) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index e52cebc707..4934b7d875 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1093,26 +1093,20 @@ def possibly_extend_end(self, soc_targets, sensor: Sensor = None): else: self.end = max_target_datetime - def get_min_max_targets( - self, deserialized_names: bool = True - ) -> tuple[float | None, float | None]: + def get_min_max_targets(self) -> tuple[float | None, float | None]: min_target = None max_target = None - soc_targets_label = "soc_targets" if deserialized_names else "soc-targets" # if the SOC targets are defined as a Sensor, we don't get min max values - if isinstance(self.flex_model.get(soc_targets_label), dict): + if isinstance(self.flex_model.get("soc-targets"), dict): return None, None - if ( - soc_targets_label in self.flex_model - and len(self.flex_model[soc_targets_label]) > 0 - ): + if "soc-targets" in self.flex_model and len(self.flex_model["soc-targets"]) > 0: min_target = min( - [target["value"] for target in self.flex_model[soc_targets_label]] + [target["value"] for target in self.flex_model["soc-targets"]] ) max_target = max( - [target["value"] for target in self.flex_model[soc_targets_label]] + [target["value"] for target in self.flex_model["soc-targets"]] ) return min_target, max_target @@ -1122,9 +1116,9 @@ def get_min_max_soc_on_sensor( soc_min_sensor: float | None = self.sensor.get_attribute("min_soc_in_mwh") soc_max_sensor: float | None = self.sensor.get_attribute("max_soc_in_mwh") if adjust_unit: - if soc_min_sensor and self.flex_model.get("soc_unit") == "kWh": + if soc_min_sensor and self.flex_model.get("soc-unit") == "kWh": soc_min_sensor *= 1000 # later steps assume soc data is kWh - if soc_max_sensor and self.flex_model.get("soc_unit") == "kWh": + if soc_max_sensor and self.flex_model.get("soc-unit") == "kWh": soc_max_sensor *= 1000 return soc_min_sensor, soc_max_sensor From 26ae60c116b5c414aceb835dd81f8da342198d28 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 15:51:45 +0200 Subject: [PATCH 143/162] refactor: remove another obsolete argument Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4934b7d875..cc19833db7 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1094,6 +1094,7 @@ def possibly_extend_end(self, soc_targets, sensor: Sensor = None): self.end = max_target_datetime def get_min_max_targets(self) -> tuple[float | None, float | None]: + """This happens before deserializing the flex-model.""" min_target = None max_target = None @@ -1110,27 +1111,24 @@ def get_min_max_targets(self) -> tuple[float | None, float | None]: ) return min_target, max_target - def get_min_max_soc_on_sensor( - self, adjust_unit: bool = False - ) -> tuple[float | None, float | None]: + def get_min_max_soc_on_sensor(self) -> tuple[float | None, float | None]: + """This happens before deserializing the flex-model.""" soc_min_sensor: float | None = self.sensor.get_attribute("min_soc_in_mwh") soc_max_sensor: float | None = self.sensor.get_attribute("max_soc_in_mwh") - if adjust_unit: - if soc_min_sensor and self.flex_model.get("soc-unit") == "kWh": - soc_min_sensor *= 1000 # later steps assume soc data is kWh - if soc_max_sensor and self.flex_model.get("soc-unit") == "kWh": - soc_max_sensor *= 1000 + if soc_min_sensor and self.flex_model.get("soc-unit") == "kWh": + soc_min_sensor *= 1000 # later steps assume soc data is kWh + if soc_max_sensor and self.flex_model.get("soc-unit") == "kWh": + soc_max_sensor *= 1000 return soc_min_sensor, soc_max_sensor def ensure_soc_min_max(self): """ Make sure we have min and max SOC. If not passed directly, then get default from sensor or targets. + This happens before deserializing the flex-model. """ _, max_target = self.get_min_max_targets() - soc_min_sensor, soc_max_sensor = self.get_min_max_soc_on_sensor( - adjust_unit=True - ) + soc_min_sensor, soc_max_sensor = self.get_min_max_soc_on_sensor() if "soc-min" not in self.flex_model or self.flex_model["soc-min"] is None: # Default is 0 - can't drain the storage by more than it contains self.flex_model["soc-min"] = soc_min_sensor if soc_min_sensor else 0 From 55c6f5f5446e7aa7c818442f78e19386a5d37fb3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 16:08:41 +0200 Subject: [PATCH 144/162] refactor: assign intermediate result to variable Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 2008310d5f..1166b6de7c 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -190,17 +190,16 @@ def compute_expected_length( get_schedule_response.json["unit"] == "MW" ), "by default, the schedules are expected in the sensor unit" # assert get_schedule_response.json["type"] == "GetDeviceMessageResponse" - assert ( - len(get_schedule_response.json["values"]) == expected_length_of_schedule[d] - ) + power_schedule = get_schedule_response.json["values"] + assert len(power_schedule) == expected_length_of_schedule[d] check_constraints( sensor=sensor, schedule=pd.Series( - data=get_schedule_response.json["values"], + data=power_schedule, index=pd.date_range( start=get_schedule_response.json["start"], - periods=len(get_schedule_response.json["values"]), + periods=len(power_schedule), freq=sensor.event_resolution, ), ), @@ -223,16 +222,11 @@ def compute_expected_length( cheapest_hour = prices.values.argmin() if sequential and sensor_id == sensor_2.id: assert ( - sum( - get_schedule_response.json["values"][ - cheapest_hour * 4 : (cheapest_hour + 1) * 4 - ] - ) - > 0 + sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - print(get_schedule_response.json["values"]) + print(power_schedule) assert_equal( - get_schedule_response.json["values"], + power_schedule, # fmt: off [ -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 @@ -252,15 +246,10 @@ def compute_expected_length( ) elif not sequential and sensor_id == sensor_2.id: assert ( - sum( - get_schedule_response.json["values"][ - cheapest_hour * 4 : (cheapest_hour + 1) * 4 - ] - ) - > 0 + sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" assert_equal( - get_schedule_response.json["values"], + power_schedule, # fmt: off [ -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 From d1433380224445d9050fcd743248fba17f178f47 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 16:36:16 +0200 Subject: [PATCH 145/162] feat: test fetching SoC schedules Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 53 ++++++++++++++++++- flexmeasures/conftest.py | 18 +++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 1166b6de7c..d8ecb393b3 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -68,6 +68,11 @@ def test_asset_trigger_and_get_schedule( sensor_2 = charging_station.sensors[0] assert sensor_2.name == "power", "expecting to schedule a power sensor" + uni_soc_sensor = add_charging_station_assets_fresh_db["uni-soc"] + bi_soc_sensor = add_charging_station_assets_fresh_db["bi-soc"] + CP_1_flex_model["state-of-charge"] = {"sensor": bi_soc_sensor.id} + CP_2_flex_model["state-of-charge"] = {"sensor": uni_soc_sensor.id} + # Convert the two flex-models to a single multi-asset flex-model CP_1_flex_model["sensor"] = sensor_1.id CP_2_flex_model["sensor"] = sensor_2.id @@ -175,10 +180,15 @@ def compute_expected_length( return expected_lengths sensors = [sensor_1, sensor_2] + soc_sensors = [bi_soc_sensor, uni_soc_sensor] expected_length_of_schedule = compute_expected_length(message, sensors, sequential) # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint - for d, (sensor, flex_model) in enumerate(zip(sensors, message["flex-model"])): + for d, (sensor, soc_sensor, flex_model) in enumerate( + zip(sensors, soc_sensors, message["flex-model"]) + ): + + # Fetch power schedule sensor_id = flex_model["sensor"] get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=sensor_id, uuid=scheduling_job.id), @@ -214,6 +224,47 @@ def compute_expected_length( .magnitude, ) + # Fetch SoC schedule + get_schedule_response = client.get( + url_for("SensorAPI:get_schedule", id=soc_sensor.id, uuid=scheduling_job.id), + query_string={"duration": "PT48H"}, + ) + print("Server responded with:\n%s" % get_schedule_response.json) + assert get_schedule_response.status_code == 200 + assert ( + get_schedule_response.json["unit"] == "MWh" + ), "by default, the schedules are expected in the sensor unit" + soc_schedule = get_schedule_response.json["values"] + assert ( + len(soc_schedule) + == expected_length_of_schedule[d] + + 1 # +1 because the SoC schedule is end-inclusive + ) + assert soc_schedule[0] * 1000 == flex_model["soc-at-start"] + + # Check for cycling and final state + if sensor_id == sensor_1.id: + # We expect cycling for the bi-directional Charge Point + assert any( + [s == flex_model["soc-min"] / 1000 for s in soc_schedule] + ), "we should stay above soc-min" + assert any( + [s == flex_model["soc-max"] / 1000 for s in soc_schedule] + ), "we should stay below soc-max" + assert ( + soc_schedule[-1] * 1000 == flex_model["soc-min"] + ), "we should end empty" + else: + # We expect no cycling for the uni-directional Charge Point + assert ( + 1 - (-pd.Series(soc_schedule).diff() / pd.Series(soc_schedule)).max() + ) * 100 <= ur.Quantity(flex_model["storage-efficiency"]).to( + "%" + ).magnitude, "all downwards SoC should be attributable to storage losses" + assert ( + soc_schedule[-1] * 1000 == flex_model["soc-targets"][0]["value"] + ), "we should end on target" + prices = add_market_prices_fresh_db["epex_da"].search_beliefs( event_starts_after=message["start"], event_ends_before=pd.Timestamp(message["start"]) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index b4156c057c..97a90a724d 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -1081,9 +1081,27 @@ def create_charging_station_assets( ), ) db.session.add(bidirectional_charging_station_power_sensor) + bi_soc = Sensor( + name="bi-soc", + generic_asset=bidirectional_charging_station, + unit="MWh", + event_resolution=timedelta(minutes=0), + attributes={"consumption_is_positive": True}, + ) + uni_soc = Sensor( + name="uni-soc", + generic_asset=charging_station, + unit="MWh", + event_resolution=timedelta(minutes=0), + attributes={"consumption_is_positive": True}, + ) + db.session.add(bi_soc) + db.session.add(uni_soc) return { "Test charging station": charging_station, "Test charging station (bidirectional)": bidirectional_charging_station, + "bi-soc": bi_soc, + "uni-soc": uni_soc, } From 54be9493ab47026f06d0661f63ca8dbd4b1fcd68 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 16:38:25 +0200 Subject: [PATCH 146/162] refactor: clarifying variable Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_asset_schedules_fresh_db.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index d8ecb393b3..7024e06737 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -256,11 +256,13 @@ def compute_expected_length( ), "we should end empty" else: # We expect no cycling for the uni-directional Charge Point + soc_as_percentage_of_previous_soc = ( + 1 - (-pd.Series(soc_schedule).diff() / pd.Series(soc_schedule)) + ) * 100 assert ( - 1 - (-pd.Series(soc_schedule).diff() / pd.Series(soc_schedule)).max() - ) * 100 <= ur.Quantity(flex_model["storage-efficiency"]).to( - "%" - ).magnitude, "all downwards SoC should be attributable to storage losses" + soc_as_percentage_of_previous_soc.min() + <= ur.Quantity(flex_model["storage-efficiency"]).to("%").magnitude + ), "all downwards SoC should be attributable to storage losses" assert ( soc_schedule[-1] * 1000 == flex_model["soc-targets"][0]["value"] ), "we should end on target" From 45a13f8908cde12de198b0fa706c31cd97fa8346 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 17:33:11 +0200 Subject: [PATCH 147/162] fix: API should only switch sign if the scheduling service switched it; this streamlines the conditions in two places (see #1345) Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 4 +++- flexmeasures/conftest.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 69073e576d..5378b46b05 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -710,7 +710,9 @@ def get_schedule( # noqa: C901 ) sign = 1 - if sensor.get_attribute("consumption_is_positive", True): + if sensor.measures_power and sensor.get_attribute( + "consumption_is_positive", True + ): sign = -1 # For consumption schedules, positive values denote consumption. For the db, consumption is negative diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 97a90a724d..2ffb5e356b 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -1086,14 +1086,12 @@ def create_charging_station_assets( generic_asset=bidirectional_charging_station, unit="MWh", event_resolution=timedelta(minutes=0), - attributes={"consumption_is_positive": True}, ) uni_soc = Sensor( name="uni-soc", generic_asset=charging_station, unit="MWh", event_resolution=timedelta(minutes=0), - attributes={"consumption_is_positive": True}, ) db.session.add(bi_soc) db.session.add(uni_soc) From 8ba2b50202bb1e473fca9f53fbfbcde524a2c981 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:01:48 +0200 Subject: [PATCH 148/162] dev: remove print statement Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 7024e06737..f04cd3c486 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -277,7 +277,6 @@ def compute_expected_length( assert ( sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - print(power_schedule) assert_equal( power_schedule, # fmt: off From 7fd964c20f621a861294202c325fe913211e5280 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:05:18 +0200 Subject: [PATCH 149/162] fix: "%" units not supported in Python 3.8? Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_asset_schedules_fresh_db.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index f04cd3c486..9031783595 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -256,12 +256,11 @@ def compute_expected_length( ), "we should end empty" else: # We expect no cycling for the uni-directional Charge Point - soc_as_percentage_of_previous_soc = ( - 1 - (-pd.Series(soc_schedule).diff() / pd.Series(soc_schedule)) - ) * 100 + s = pd.Series(soc_schedule) + soc_as_percentage_of_previous_soc = 1 + s.diff() / s assert ( soc_as_percentage_of_previous_soc.min() - <= ur.Quantity(flex_model["storage-efficiency"]).to("%").magnitude + <= ur.Quantity(flex_model["storage-efficiency"]).to("").magnitude ), "all downwards SoC should be attributable to storage losses" assert ( soc_schedule[-1] * 1000 == flex_model["soc-targets"][0]["value"] From 10f7f1bbb3faa257d987e032c2c99a60256d6e44 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:22:12 +0200 Subject: [PATCH 150/162] docs: add paragraph on interpreting schedules and making them suit specific power levels Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index e5925137d8..22bec28773 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -212,3 +212,8 @@ However, because the targets values represent averages over 15-minute time inter For example, the battery might start to consume with 2.1 MW at 10.00am and increase its consumption to 2.25 at 10.10am, increase its consumption to 5 MW at 10.15am and decrease its consumption to 2 MW at 10.20am. That should result in the same average values for each quarter-hour. + +Likewise, the control signals can be used to schedule devices that only run at specific power levels. +For example, let's assume the battery only supports running at an integer number of MW. +In that case, the battery could start to consume with 2 MW at 10.00am, increase its consumption to 3 MW at (15 seconds before) 10.12am, and decrease its consumption to 2 MW at 10.30am. +Again, this results in the same average values for each quarter-hour. From 22ba034931f4a240fb0b2c565211e39b15ec76ff Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:44:49 +0200 Subject: [PATCH 151/162] docs: use the same sensor ID as in the previous example from posting_data.rst that we are continuing Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 22bec28773..6eeb6b8c6e 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -108,6 +108,7 @@ Here, we extend that (storage) example with an additional target value, represen { "start": "2015-06-02T10:00:00+00:00", "flex-model": { + "sensor": 15, "soc-at-start": "12.1 kWh", "soc-targets": [ { @@ -130,7 +131,7 @@ A second way to add scheduling jobs is via the CLI, so this is available for peo .. code-block:: bash - $ flexmeasures add schedule for-storage --sensor 1 --consumption-price-sensor 2 \ + $ flexmeasures add schedule for-storage --sensor 15 --consumption-price-sensor 2 \ --start 2022-07-05T07:00+01:00 --duration PT12H \ --soc-at-start 50% --roundtrip-efficiency 90% --as-job From 59f10c94daa97b43d32b8f5e83fdf533e5753918 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:45:13 +0200 Subject: [PATCH 152/162] docs: add note on retrieving the SoC schedule Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 6eeb6b8c6e..a25daf02a5 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -187,6 +187,17 @@ Here, the schedule's Universally Unique Identifier (UUID) should be filled in th Schedules can be queried by their UUID for up to 1 week after they were triggered (ask your host if you need to keep them around longer). Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. +.. note:: Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. + If a ``state-of-charge`` sensor was also referenced in the flex-model, the scheduled state of charge can also be retrieved using the same endpoint and UUID. + Simply replace the power sensor ID with the state-of-charge sensor ID. + + .. code-block:: json + + "flex-model": { + "sensor": 15, + "state-of-charge": {"sensor": 16} + } + The following example response indicates that FlexMeasures planned ahead 45 minutes for the requested battery power sensor. The list of consecutive power values represents the target consumption of the battery (negative values for production). Each value represents the average power over a 15 minute time interval. From 673b37809f68ac8df7198abe5f8d90e3f0d1b999 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:46:16 +0200 Subject: [PATCH 153/162] docs: reference the sensor ID in the main body of text Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index a25daf02a5..ecc7e7ea7a 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -186,9 +186,9 @@ We saw above how FlexMeasures can create optimised schedules with control signal Here, the schedule's Universally Unique Identifier (UUID) should be filled in that is returned in the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ response. Schedules can be queried by their UUID for up to 1 week after they were triggered (ask your host if you need to keep them around longer). Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. +Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. -.. note:: Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. - If a ``state-of-charge`` sensor was also referenced in the flex-model, the scheduled state of charge can also be retrieved using the same endpoint and UUID. +.. note:: If a ``state-of-charge`` sensor was also referenced in the flex-model, the scheduled state of charge can also be retrieved using the same endpoint and UUID. Simply replace the power sensor ID with the state-of-charge sensor ID. .. code-block:: json From 363ad0a413291420e91ff375009c089b69887fa6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:51:20 +0200 Subject: [PATCH 154/162] docs: fill in API version 3 Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 4 ++-- documentation/tut/posting_data.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index ecc7e7ea7a..bbceb56549 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -155,7 +155,7 @@ A prognosis can be requested at a URL looking like this: .. code-block:: html - https://company.flexmeasures.io/api//sensors/data + https://company.flexmeasures.io/api/v3_0/sensors/data This example requests a prognosis for 24 hours, with a rolling horizon of 6 hours before realisation. @@ -181,7 +181,7 @@ We saw above how FlexMeasures can create optimised schedules with control signal .. code-block:: html - https://company.flexmeasures.io/api//sensors//schedules/ + https://company.flexmeasures.io/api/v3_0/sensors//schedules/ Here, the schedule's Universally Unique Identifier (UUID) should be filled in that is returned in the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ response. Schedules can be queried by their UUID for up to 1 week after they were triggered (ask your host if you need to keep them around longer). diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 04a5786908..3452a9a671 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -46,7 +46,7 @@ The exact URL will depend on your domain name, and will look approximately like .. code-block:: html - [POST] https://company.flexmeasures.io/api//sensors/data + [POST] https://company.flexmeasures.io/api/v3_0/sensors/data This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. @@ -280,7 +280,7 @@ The URL might look like this: .. code-block:: html - https://company.flexmeasures.io/api//assets/10/schedules/trigger + https://company.flexmeasures.io/api/v3_0/assets/10/schedules/trigger The following example triggers a schedule for a power sensor (with ID 15) of a battery asset (with ID 10), asking to take into account the battery's current state of charge. From this, FlexMeasures derives the energy flexibility this battery has in the next 48 hours and computes an optimal charging schedule. From 96db6e1e6136f27b467c26b31559861ac1ecbf02 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 20:51:53 +0200 Subject: [PATCH 155/162] docs: also also Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index bbceb56549..6c76a8d0ec 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -188,7 +188,7 @@ Schedules can be queried by their UUID for up to 1 week after they were triggere Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. -.. note:: If a ``state-of-charge`` sensor was also referenced in the flex-model, the scheduled state of charge can also be retrieved using the same endpoint and UUID. +.. note:: If a ``state-of-charge`` sensor was referenced in the flex-model, the scheduled state of charge can be retrieved using the same endpoint and UUID. Simply replace the power sensor ID with the state-of-charge sensor ID. .. code-block:: json From 9befe96e5dcffbb939e23d162d5fcd99a526497b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 16 Jun 2025 21:14:53 +0200 Subject: [PATCH 156/162] docs: follow capitalization of package itself Signed-off-by: F.N. Claessen --- documentation/tut/toy-example-expanded.rst | 4 ++-- documentation/tut/toy-example-from-scratch.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/toy-example-expanded.rst b/documentation/tut/toy-example-expanded.rst index cb042eac27..7a66fd5fad 100644 --- a/documentation/tut/toy-example-expanded.rst +++ b/documentation/tut/toy-example-expanded.rst @@ -140,9 +140,9 @@ This will have an effect on the available headroom for the battery, given the `` "flex-context": {} } - .. tab:: flexmeasures-client + .. tab:: FlexMeasures Client - Using the `flexmeasures-client `_: + Using the `FlexMeasures Client `_: .. code-block:: bash diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index c3ce822f80..5ee37867aa 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -52,9 +52,9 @@ To keep it short, we'll only ask for a 12-hour window starting at 7am. Finally, ] } - .. tab:: flexmeasures-client + .. tab:: FlexMeasures Client - Using the `flexmeasures-client `_: + Using the `FlexMeasures Client `_: .. code-block:: bash From edc099e5b091faf256fdd58b8da771b14146a341 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 19 Jun 2025 00:13:42 +0200 Subject: [PATCH 157/162] refactor: simplify expected schedule Signed-off-by: F.N. Claessen --- .../tests/test_asset_schedules_fresh_db.py | 45 +++---------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 9031783595..172a5d6666 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -183,6 +183,11 @@ def compute_expected_length( soc_sensors = [bi_soc_sensor, uni_soc_sensor] expected_length_of_schedule = compute_expected_length(message, sensors, sequential) + # The 72nd quarterhour is the first quarterhour within the cheapest hour. + # That's when we expect all charging for the uni-directional CP. + expected_uni_schedule = [0] * 188 + expected_uni_schedule[72] = 0.053651 + # try to retrieve the schedule for each sensor through the /sensors//schedules/ [GET] api endpoint for d, (sensor, soc_sensor, flex_model) in enumerate( zip(sensors, soc_sensors, message["flex-model"]) @@ -276,45 +281,9 @@ def compute_expected_length( assert ( sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - assert_equal( - power_schedule, - # fmt: off - [ - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - ] - # fmt: on - ) + assert_equal(power_schedule, expected_uni_schedule) elif not sequential and sensor_id == sensor_2.id: assert ( sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - assert_equal( - power_schedule, - # fmt: off - [ - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.053651, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, # noqa: E128 - -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, - ] - # fmt: on - ) + assert_equal(power_schedule, expected_uni_schedule) From 867e1ae6bd03952d45a7fc20c43ad101e3a02687 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 19 Jun 2025 00:14:15 +0200 Subject: [PATCH 158/162] docs: clarify note Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 6c76a8d0ec..2ded06eb05 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -188,8 +188,7 @@ Schedules can be queried by their UUID for up to 1 week after they were triggere Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. -.. note:: If a ``state-of-charge`` sensor was referenced in the flex-model, the scheduled state of charge can be retrieved using the same endpoint and UUID. - Simply replace the power sensor ID with the state-of-charge sensor ID. +.. note:: If a ``state-of-charge`` sensor was referenced in the flex-model (like in the example below), the scheduled state of charge can be retrieved using the same endpoint and UUID, but then using the state-of-charge sensor ID. .. code-block:: json @@ -198,6 +197,8 @@ Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which "state-of-charge": {"sensor": 16} } + Simply replace the power sensor ID in the URL with the state-of-charge sensor ID when fetching the schedule. + The following example response indicates that FlexMeasures planned ahead 45 minutes for the requested battery power sensor. The list of consecutive power values represents the target consumption of the battery (negative values for production). Each value represents the average power over a 15 minute time interval. From d0f4e02adabe4bb2310634818891f17b61ad4d4e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 19 Jun 2025 00:15:46 +0200 Subject: [PATCH 159/162] docs: clarify asserts Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 172a5d6666..f9a324331e 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -252,10 +252,10 @@ def compute_expected_length( # We expect cycling for the bi-directional Charge Point assert any( [s == flex_model["soc-min"] / 1000 for s in soc_schedule] - ), "we should stay above soc-min" + ), "we should reach soc-min at least once" assert any( [s == flex_model["soc-max"] / 1000 for s in soc_schedule] - ), "we should stay below soc-max" + ), "we should reach soc-max at least once" assert ( soc_schedule[-1] * 1000 == flex_model["soc-min"] ), "we should end empty" From 4fa292d2a82467fda85419123426b8d003dda69c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 19 Jun 2025 09:43:38 +0200 Subject: [PATCH 160/162] fix: np.assert_equal does not treat -0.0 the same as 0 (or as 0.0) Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_asset_schedules_fresh_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index f9a324331e..230d46fd3d 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -4,7 +4,7 @@ import pytest from isodate import parse_datetime, parse_duration -from numpy.testing import assert_equal +from numpy.testing import assert_almost_equal import pandas as pd from rq.job import Job @@ -281,9 +281,9 @@ def compute_expected_length( assert ( sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - assert_equal(power_schedule, expected_uni_schedule) + assert_almost_equal(power_schedule, expected_uni_schedule) elif not sequential and sensor_id == sensor_2.id: assert ( sum(power_schedule[cheapest_hour * 4 : (cheapest_hour + 1) * 4]) > 0 ), "we expect to charge in the cheapest hour" - assert_equal(power_schedule, expected_uni_schedule) + assert_almost_equal(power_schedule, expected_uni_schedule) From ecb355f293b838f9d5d926179b8cd568a7901ac6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 Jun 2025 14:34:24 +0200 Subject: [PATCH 161/162] docs: rephrase note Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 2ded06eb05..fecd9c265b 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -186,9 +186,9 @@ We saw above how FlexMeasures can create optimised schedules with control signal Here, the schedule's Universally Unique Identifier (UUID) should be filled in that is returned in the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-(id)-schedules-trigger>`_ response. Schedules can be queried by their UUID for up to 1 week after they were triggered (ask your host if you need to keep them around longer). Afterwards, the exact schedule can still be retrieved through the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_, using precise filter values for ``start``, ``prior`` and ``source``. -Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex-model. +Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which is the sensor ID of one of the power sensors that was referenced in the flex model. -.. note:: If a ``state-of-charge`` sensor was referenced in the flex-model (like in the example below), the scheduled state of charge can be retrieved using the same endpoint and UUID, but then using the state-of-charge sensor ID. +.. note:: If a ``state-of-charge`` sensor was referenced in the flex model (like in the example below), the scheduled state of charge can be retrieved using the same endpoint and UUID, but then using the state-of-charge sensor ID. .. code-block:: json @@ -197,7 +197,7 @@ Besides the UUID, the endpoint for retrieving schedules takes a sensor ID, which "state-of-charge": {"sensor": 16} } - Simply replace the power sensor ID in the URL with the state-of-charge sensor ID when fetching the schedule. + For instance, if the above snippet represents the flex model used by FlexMeasures to compute the schedule, then to fetch the scheduled state of charge you simply replace the power sensor ID in the URL of the `[GET] /sensors/data <../api/v3_0.html#get--api-v3_0-sensors-data>`_ endpoint with the state-of-charge sensor ID. The following example response indicates that FlexMeasures planned ahead 45 minutes for the requested battery power sensor. The list of consecutive power values represents the target consumption of the battery (negative values for production). From 094bf05be2ba88bbf00918a1a51f169596099dd6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 20 Jun 2025 14:35:52 +0200 Subject: [PATCH 162/162] docs: explain test expectations Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_asset_schedules_fresh_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 230d46fd3d..db5f82fcdc 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -249,13 +249,13 @@ def compute_expected_length( # Check for cycling and final state if sensor_id == sensor_1.id: - # We expect cycling for the bi-directional Charge Point + # We expect cycling fully for the bi-directional Charge Point assert any( [s == flex_model["soc-min"] / 1000 for s in soc_schedule] - ), "we should reach soc-min at least once" + ), "we should reach soc-min at least once, because we expect at least one full cycle" assert any( [s == flex_model["soc-max"] / 1000 for s in soc_schedule] - ), "we should reach soc-max at least once" + ), "we should reach soc-max at least once, because we expect at least one full cycle" assert ( soc_schedule[-1] * 1000 == flex_model["soc-min"] ), "we should end empty"