diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 38770e144..d11dc50df 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -107,6 +107,8 @@ def setup_markets(db): market_type=day_ahead, event_resolution=timedelta(hours=1), unit="EUR/MWh", + knowledge_horizon_fnc="determine_ex_ante_knowledge_horizon_for_x_days_ago_at_y_oclock", + knowledge_horizon_par={"x": 1, "y": 12, "z": "Europe/Paris"}, ) db.session.add(epex_da) diff --git a/flexmeasures/data/models/assets.py b/flexmeasures/data/models/assets.py index 0c9d3150c..2f3a35a8e 100644 --- a/flexmeasures/data/models/assets.py +++ b/flexmeasures/data/models/assets.py @@ -1,8 +1,7 @@ from typing import Dict, List, Tuple, Union -from datetime import timedelta import isodate - +import timely_beliefs as tb from sqlalchemy.orm import Query from flexmeasures.data.config import db @@ -68,10 +67,9 @@ def __repr__(self): return "" % self.name -class Asset(db.Model): +class Asset(db.Model, tb.SensorDBMixin): """Each asset is an energy- consuming or producing hardware. """ - id = db.Column(db.Integer, primary_key=True) # The name name = db.Column(db.String(80), default="", unique=True) # The name we want to see (don't unnecessarily capitalize, so it can be used in a sentence) @@ -80,13 +78,6 @@ class Asset(db.Model): asset_type_name = db.Column( db.String(80), db.ForeignKey("asset_type.name"), nullable=False ) - unit = db.Column(db.String(80), default="", nullable=False) - # Expected resolution of time series for this sensor. - # Defaults to zero, as it can't be None (used for calculations during - # query building). You should set this to a realistic value! - event_resolution = db.Column( - db.Interval(), nullable=False, default=timedelta(minutes=0) - ) # How many MW at peak usage capacity_in_mw = db.Column(db.Float, nullable=False) # State of charge in MWh and its datetime and udi event diff --git a/flexmeasures/data/models/markets.py b/flexmeasures/data/models/markets.py index bc56e4550..41afa907e 100644 --- a/flexmeasures/data/models/markets.py +++ b/flexmeasures/data/models/markets.py @@ -1,6 +1,7 @@ from typing import Dict -from datetime import timedelta +import timely_beliefs as tb +from timely_beliefs.sensors.func_store import knowledge_horizons from sqlalchemy.orm import Query from flexmeasures.data.config import db @@ -40,21 +41,23 @@ def __repr__(self): return "" % self.name -class Market(db.Model): +class Market(db.Model, tb.SensorDBMixin): """Each market is a pricing service.""" - id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(80), unique=True) display_name = db.Column(db.String(80), default="", unique=True) market_type_name = db.Column( db.String(80), db.ForeignKey("market_type.name"), nullable=False ) - unit = db.Column(db.String(80), default="", nullable=False) - event_resolution = db.Column( - db.Interval(), nullable=False, default=timedelta(minutes=0) - ) def __init__(self, **kwargs): + # Set default knowledge horizon function for an economic sensor + if "knowledge_horizon_fnc" not in kwargs: + kwargs["knowledge_horizon_fnc"] = knowledge_horizons.ex_ante.__name__ + if "knowledge_horizon_par" not in kwargs: + kwargs["knowledge_horizon_par"] = { + knowledge_horizons.ex_ante.__code__.co_varnames[1]: "PT0H" + } super(Market, self).__init__(**kwargs) self.name = self.name.replace(" ", "_").lower() if "display_name" not in kwargs: diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 5cc68cd07..8d88eee85 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -1,7 +1,7 @@ from typing import Dict, Tuple -from datetime import timedelta import math +import timely_beliefs as tb from sqlalchemy.orm import Query from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.sql.expression import func @@ -35,20 +35,15 @@ def __repr__(self): return "" % self.name -class WeatherSensor(db.Model): +class WeatherSensor(db.Model, tb.SensorDBMixin): """A weather sensor has a location on Earth and measures weather values of a certain weather sensor type, such as temperature, wind speed and radiation.""" - id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(80), unique=True) display_name = db.Column(db.String(80), default="", unique=False) weather_sensor_type_name = db.Column( db.String(80), db.ForeignKey("weather_sensor_type.name"), nullable=False ) - unit = db.Column(db.String(80), default="", nullable=False) - event_resolution = db.Column( - db.Interval(), nullable=False, default=timedelta(minutes=0) - ) # latitude is the North/South coordinate latitude = db.Column(db.Float, nullable=False) # longitude is the East/West coordinate diff --git a/flexmeasures/data/services/forecasting.py b/flexmeasures/data/services/forecasting.py index a79d4f542..12035d8b2 100644 --- a/flexmeasures/data/services/forecasting.py +++ b/flexmeasures/data/services/forecasting.py @@ -72,9 +72,9 @@ def create_forecasting_jobs( 2) forecast each quarter-hour from 9pm to 11pm, i.e. the 6h forecast 3) forecast each quarter-hour from 3pm to 5pm the next day, i.e. the 1d forecast - If not given, relevant horizons are deduced from the resolution of the posted data. + If not given, relevant horizons are derived from the resolution of the posted data. - The job needs a model configurator, for which you can supply a model search term. If ommited, the + The job needs a model configurator, for which you can supply a model search term. If omitted, the current default model configuration will be used. It's possible to customize model parameters, but this feature is (currently) meant to only diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index b1c60df01..299e06476 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -181,11 +181,7 @@ def query_time_series_data( if current_app.config.get("FLEXMEASURES_MODE", "") == "demo": df.index = df.index.map(lambda t: t.replace(year=datetime.now().year)) - flexmeasures_sensor = find_sensor_by_name(name=generic_asset_name) - sensor = tb.Sensor( - name=generic_asset_name, - event_resolution=flexmeasures_sensor.event_resolution, - ) + sensor = find_sensor_by_name(name=generic_asset_name) bdf = tb.BeliefsDataFrame(df.reset_index(), sensor=sensor) # re-sample data to the resolution we need to serve diff --git a/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py b/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py new file mode 100644 index 000000000..e99ac01c2 --- /dev/null +++ b/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py @@ -0,0 +1,165 @@ +"""mix in timely beliefs sensor with asset, market and weather sensor; introduce knowledge horizons + +Revision ID: 22ce09690d23 +Revises: 564e8df4e3a9 +Create Date: 2021-01-31 14:31:16.370110 + +""" +from alembic import op +import json +import sqlalchemy as sa +from timely_beliefs.sensors.func_store import knowledge_horizons + + +# revision identifiers, used by Alembic. +revision = "22ce09690d23" +down_revision = "564e8df4e3a9" +branch_labels = None +depends_on = None + +# set default parameters for the two default knowledge horizon functions +ex_ante_default_par = {knowledge_horizons.ex_ante.__code__.co_varnames[0]: "PT0H"} +ex_post_default_par = {knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"} + + +def upgrade(): + + # Mix in timely_beliefs.Sensor with flexmeasures.Asset + op.add_column( + "asset", + sa.Column( + "knowledge_horizon_fnc", + sa.String(length=80), + nullable=True, + default=knowledge_horizons.ex_post.__name__, + ), + ) + op.execute( + f"update asset set knowledge_horizon_fnc = '{knowledge_horizons.ex_post.__name__}';" + ) # default assumption that power measurements are known right after the fact + op.alter_column("asset", "knowledge_horizon_fnc", nullable=False) + + op.add_column( + "asset", + sa.Column( + "knowledge_horizon_par", + sa.JSON(), + nullable=True, + default={knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"}, + ), + ) + op.execute( + f"""update asset set knowledge_horizon_par = '{json.dumps(ex_post_default_par)}';""" + ) + op.alter_column("asset", "knowledge_horizon_par", nullable=False) + + op.add_column("asset", sa.Column("timezone", sa.String(length=80), nullable=True)) + op.execute("update asset set timezone = 'Asia/Seoul';") + op.alter_column("asset", "timezone", nullable=False) + + # Mix in timely_beliefs.Sensor with flexmeasures.Market + op.add_column( + "market", + sa.Column( + "knowledge_horizon_fnc", + sa.String(length=80), + nullable=True, + default=knowledge_horizons.ex_ante.__name__, + ), + ) + op.execute( + f"update market set knowledge_horizon_fnc = '{knowledge_horizons.ex_ante.__name__}';" + ) # default assumption that prices are known before a transaction + op.execute( + f"update market set knowledge_horizon_fnc = '{knowledge_horizons.at_date.__name__}' where name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');" + ) + op.alter_column("market", "knowledge_horizon_fnc", nullable=False) + + op.add_column( + "market", + sa.Column( + "knowledge_horizon_par", + sa.JSON(), + nullable=True, + default=ex_ante_default_par, + ), + ) + op.execute( + f"""update market set knowledge_horizon_par = '{json.dumps(ex_ante_default_par)}';""" + ) + op.execute( + """update market set knowledge_horizon_par = '{"knowledge_time": "2014-12-31 00:00:00+00:00"}' where name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');""" + ) # tariff publication date (unofficial) + op.alter_column("market", "knowledge_horizon_par", nullable=False) + op.execute( + "update price set horizon = interval '0 hours' from market where market_id = market.id and market.name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');" + ) # 0 hours after fixed knowledge time (i.e. at publication date) + + op.add_column("market", sa.Column("timezone", sa.String(length=80), nullable=True)) + op.execute("update market set timezone = 'UTC';") + op.execute("update market set timezone = 'Europe/Paris' where name='epex_da';") + op.execute("update market set timezone = 'Asia/Seoul' where unit='KRW/kWh';") + op.alter_column("market", "timezone", nullable=False) + + # Mix in timely_beliefs.Sensor with flexmeasures.WeatherSensor + op.add_column( + "weather_sensor", + sa.Column( + "knowledge_horizon_fnc", + sa.String(length=80), + nullable=True, + default=knowledge_horizons.ex_post.__name__, + ), + ) + op.execute( + f"update weather_sensor set knowledge_horizon_fnc = '{knowledge_horizons.ex_post.__name__}';" + ) # default assumption that weather measurements are known right after the fact + op.alter_column("weather_sensor", "knowledge_horizon_fnc", nullable=False) + + op.add_column( + "weather_sensor", + sa.Column( + "knowledge_horizon_par", + sa.JSON(), + nullable=True, + default={knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"}, + ), + ) + op.execute( + f"""update weather_sensor set knowledge_horizon_par = '{json.dumps(ex_post_default_par)}';""" + ) + op.alter_column("weather_sensor", "knowledge_horizon_par", nullable=False) + + op.add_column( + "weather_sensor", sa.Column("timezone", sa.String(length=80), nullable=True) + ) + op.execute("update weather_sensor set timezone = 'Asia/Seoul';") + op.alter_column("weather_sensor", "timezone", nullable=False) + + # todo: execute after adding relevant tests and updating our strategy to create forecasting jobs when new information arrives + # op.execute( + # f"update market set knowledge_horizon_fnc = '{knowledge_horizons.x_days_ago_at_y_oclock.__name__}' where name in ('epex_da', 'kpx_da');" + # ) + # op.execute( + # """update market set knowledge_horizon_par = '{"x": 1, "y": 12, "z": "Europe/Paris"}' where name='epex_da';""" + # ) # gate closure at 12:00 on the preceding day, with expected price publication at 12.42 and 12.55 (from EPEX Spot Day-Ahead Multi-Regional Coupling, https://www.epexspot.com/en/downloads#rules-fees-processes ) + # op.execute( + # """update market set knowledge_horizon_par = '{"x": 1, "y": 10, "z": "Asia/Seoul"}' where name='kpx_da';""" + # ) # gate closure at 10.00 on the preceding day, with expected price publication at 15.00 (from KPX Power Market Operation, https://www.slideshare.net/sjchung0/power-market-operation ) + # todo: add statement to update the horizon for prices on these markets, in accordance with their new knowledge horizon function + + +def downgrade(): + # Drop mixed in columns + op.drop_column("asset", "timezone") + op.drop_column("asset", "knowledge_horizon_par") + op.drop_column("asset", "knowledge_horizon_fnc") + op.drop_column("market", "timezone") + op.drop_column("market", "knowledge_horizon_par") + op.drop_column("market", "knowledge_horizon_fnc") + op.drop_column("weather_sensor", "timezone") + op.drop_column("weather_sensor", "knowledge_horizon_par") + op.drop_column("weather_sensor", "knowledge_horizon_fnc") + op.execute( + "update price set horizon = ((datetime + market.event_resolution) - '2014-12-31 00:00:00+00:00') from market where market_id = market.id and name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');" + ) # rolling horizon before end of event (i.e. at publication date) diff --git a/requirements/app.in b/requirements/app.in index 01a6c03fa..c2c24d39a 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -27,7 +27,7 @@ pyomo>=5.6 forecastiopy pysolar timetomodel>=0.6.8 -timely-beliefs>=1.0.0 +timely-beliefs>=1.2.0 python-dotenv Flask-SSLify Flask_JSON