Mix in timely beliefs Sensor

flexmeasures/conftest.py

@@ -107,6 +107,8 @@ def setup_markets(db):
         market_type=day_ahead,
         event_resolution=timedelta(hours=1),
         unit="EUR/MWh",
+        knowledge_horizon_fnc="determine_ex_ante_knowledge_horizon_for_x_days_ago_at_y_oclock",
+        knowledge_horizon_par={"x": 1, "y": 12, "z": "Europe/Paris"},
     )
     db.session.add(epex_da)
 

flexmeasures/data/models/assets.py

@@ -1,8 +1,7 @@
 from typing import Dict, List, Tuple, Union
-from datetime import timedelta
 
 import isodate
-
+import timely_beliefs as tb
 from sqlalchemy.orm import Query
 
 from flexmeasures.data.config import db
@@ -68,10 +67,9 @@ def __repr__(self):
         return "<AssetType %r>" % self.name
 
 
-class Asset(db.Model):
+class Asset(db.Model, tb.SensorDBMixin):
     """Each asset is an energy- consuming or producing hardware. """
 
-    id = db.Column(db.Integer, primary_key=True)
     # The name
     name = db.Column(db.String(80), default="", unique=True)
     # The name we want to see (don't unnecessarily capitalize, so it can be used in a sentence)
@@ -80,13 +78,6 @@ class Asset(db.Model):
     asset_type_name = db.Column(
         db.String(80), db.ForeignKey("asset_type.name"), nullable=False
     )
-    unit = db.Column(db.String(80), default="", nullable=False)
-    # Expected resolution of time series for this sensor.
-    # Defaults to zero, as it can't be None (used for calculations during
-    # query building). You should set this to a realistic value!
-    event_resolution = db.Column(
-        db.Interval(), nullable=False, default=timedelta(minutes=0)
-    )
     # How many MW at peak usage
     capacity_in_mw = db.Column(db.Float, nullable=False)
     # State of charge in MWh and its datetime and udi event

flexmeasures/data/models/markets.py

@@ -1,6 +1,7 @@
 from typing import Dict
-from datetime import timedelta
 
+import timely_beliefs as tb
+from timely_beliefs.sensors.func_store import knowledge_horizons
 from sqlalchemy.orm import Query
 
 from flexmeasures.data.config import db
@@ -40,21 +41,23 @@ def __repr__(self):
         return "<MarketType %r>" % self.name
 
 
-class Market(db.Model):
+class Market(db.Model, tb.SensorDBMixin):
     """Each market is a pricing service."""
 
-    id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(80), unique=True)
     display_name = db.Column(db.String(80), default="", unique=True)
     market_type_name = db.Column(
         db.String(80), db.ForeignKey("market_type.name"), nullable=False
     )
-    unit = db.Column(db.String(80), default="", nullable=False)
-    event_resolution = db.Column(
-        db.Interval(), nullable=False, default=timedelta(minutes=0)
-    )
 
     def __init__(self, **kwargs):
+        # Set default knowledge horizon function for an economic sensor
+        if "knowledge_horizon_fnc" not in kwargs:
+            kwargs["knowledge_horizon_fnc"] = knowledge_horizons.ex_ante.__name__
+        if "knowledge_horizon_par" not in kwargs:
+            kwargs["knowledge_horizon_par"] = {
+                knowledge_horizons.ex_ante.__code__.co_varnames[1]: "PT0H"
+            }
         super(Market, self).__init__(**kwargs)
         self.name = self.name.replace(" ", "_").lower()
         if "display_name" not in kwargs:

flexmeasures/data/models/weather.py

@@ -1,7 +1,7 @@
 from typing import Dict, Tuple
-from datetime import timedelta
 import math
 
+import timely_beliefs as tb
 from sqlalchemy.orm import Query
 from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
 from sqlalchemy.sql.expression import func
@@ -35,20 +35,15 @@ def __repr__(self):
         return "<WeatherSensorType %r>" % self.name
 
 
-class WeatherSensor(db.Model):
+class WeatherSensor(db.Model, tb.SensorDBMixin):
     """A weather sensor has a location on Earth and measures weather values of a certain weather sensor type, such as
     temperature, wind speed and radiation."""
 
-    id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String(80), unique=True)
     display_name = db.Column(db.String(80), default="", unique=False)
     weather_sensor_type_name = db.Column(
         db.String(80), db.ForeignKey("weather_sensor_type.name"), nullable=False
     )
-    unit = db.Column(db.String(80), default="", nullable=False)
-    event_resolution = db.Column(
-        db.Interval(), nullable=False, default=timedelta(minutes=0)
-    )
     # latitude is the North/South coordinate
     latitude = db.Column(db.Float, nullable=False)
     # longitude is the East/West coordinate

flexmeasures/data/services/forecasting.py

@@ -72,9 +72,9 @@ def create_forecasting_jobs(
         2) forecast each quarter-hour from 9pm to 11pm, i.e. the 6h forecast
         3) forecast each quarter-hour from 3pm to 5pm the next day, i.e. the 1d forecast
 
-    If not given, relevant horizons are deduced from the resolution of the posted data.
+    If not given, relevant horizons are derived from the resolution of the posted data.
 
-    The job needs a model configurator, for which you can supply a model search term. If ommited, the
+    The job needs a model configurator, for which you can supply a model search term. If omitted, the
     current default model configuration will be used.
 
     It's possible to customize model parameters, but this feature is (currently) meant to only

flexmeasures/data/services/time_series.py

@@ -181,11 +181,7 @@ def query_time_series_data(
         if current_app.config.get("FLEXMEASURES_MODE", "") == "demo":
             df.index = df.index.map(lambda t: t.replace(year=datetime.now().year))
 
-        flexmeasures_sensor = find_sensor_by_name(name=generic_asset_name)
-        sensor = tb.Sensor(
-            name=generic_asset_name,
-            event_resolution=flexmeasures_sensor.event_resolution,
-        )
+        sensor = find_sensor_by_name(name=generic_asset_name)
         bdf = tb.BeliefsDataFrame(df.reset_index(), sensor=sensor)
 
         # re-sample data to the resolution we need to serve

migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py

@@ -0,0 +1,165 @@
+"""mix in timely beliefs sensor with asset, market and weather sensor; introduce knowledge horizons
+
+Revision ID: 22ce09690d23
+Revises: 564e8df4e3a9
+Create Date: 2021-01-31 14:31:16.370110
+
+"""
+from alembic import op
+import json
+import sqlalchemy as sa
+from timely_beliefs.sensors.func_store import knowledge_horizons
+
+
+# revision identifiers, used by Alembic.
+revision = "22ce09690d23"
+down_revision = "564e8df4e3a9"
+branch_labels = None
+depends_on = None
+
+# set default parameters for the two default knowledge horizon functions
+ex_ante_default_par = {knowledge_horizons.ex_ante.__code__.co_varnames[0]: "PT0H"}
+ex_post_default_par = {knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"}
+
+
+def upgrade():
+
+    # Mix in timely_beliefs.Sensor with flexmeasures.Asset
+    op.add_column(
+        "asset",
+        sa.Column(
+            "knowledge_horizon_fnc",
+            sa.String(length=80),
+            nullable=True,
+            default=knowledge_horizons.ex_post.__name__,
+        ),
+    )
+    op.execute(
+        f"update asset set knowledge_horizon_fnc = '{knowledge_horizons.ex_post.__name__}';"
+    )  # default assumption that power measurements are known right after the fact
+    op.alter_column("asset", "knowledge_horizon_fnc", nullable=False)
+
+    op.add_column(
+        "asset",
+        sa.Column(
+            "knowledge_horizon_par",
+            sa.JSON(),
+            nullable=True,
+            default={knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"},
+        ),
+    )
+    op.execute(
+        f"""update asset set knowledge_horizon_par = '{json.dumps(ex_post_default_par)}';"""
+    )
+    op.alter_column("asset", "knowledge_horizon_par", nullable=False)
+
+    op.add_column("asset", sa.Column("timezone", sa.String(length=80), nullable=True))
+    op.execute("update asset set timezone = 'Asia/Seoul';")
+    op.alter_column("asset", "timezone", nullable=False)
+
+    # Mix in timely_beliefs.Sensor with flexmeasures.Market
+    op.add_column(
+        "market",
+        sa.Column(
+            "knowledge_horizon_fnc",
+            sa.String(length=80),
+            nullable=True,
+            default=knowledge_horizons.ex_ante.__name__,
+        ),
+    )
+    op.execute(
+        f"update market set knowledge_horizon_fnc = '{knowledge_horizons.ex_ante.__name__}';"
+    )  # default assumption that prices are known before a transaction
+    op.execute(
+        f"update market set knowledge_horizon_fnc = '{knowledge_horizons.at_date.__name__}' where name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');"
+    )
+    op.alter_column("market", "knowledge_horizon_fnc", nullable=False)
+
+    op.add_column(
+        "market",
+        sa.Column(
+            "knowledge_horizon_par",
+            sa.JSON(),
+            nullable=True,
+            default=ex_ante_default_par,
+        ),
+    )
+    op.execute(
+        f"""update market set knowledge_horizon_par = '{json.dumps(ex_ante_default_par)}';"""
+    )
+    op.execute(
+        """update market set knowledge_horizon_par = '{"knowledge_time": "2014-12-31 00:00:00+00:00"}' where name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');"""
+    )  # tariff publication date (unofficial)
+    op.alter_column("market", "knowledge_horizon_par", nullable=False)
+    op.execute(
+        "update price set horizon = interval '0 hours' from market where market_id = market.id and market.name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');"
+    )  # 0 hours after fixed knowledge time (i.e. at publication date)
+
+    op.add_column("market", sa.Column("timezone", sa.String(length=80), nullable=True))
+    op.execute("update market set timezone = 'UTC';")
+    op.execute("update market set timezone = 'Europe/Paris' where name='epex_da';")
+    op.execute("update market set timezone = 'Asia/Seoul' where unit='KRW/kWh';")
+    op.alter_column("market", "timezone", nullable=False)
+
+    # Mix in timely_beliefs.Sensor with flexmeasures.WeatherSensor
+    op.add_column(
+        "weather_sensor",
+        sa.Column(
+            "knowledge_horizon_fnc",
+            sa.String(length=80),
+            nullable=True,
+            default=knowledge_horizons.ex_post.__name__,
+        ),
+    )
+    op.execute(
+        f"update weather_sensor set knowledge_horizon_fnc = '{knowledge_horizons.ex_post.__name__}';"
+    )  # default assumption that weather measurements are known right after the fact
+    op.alter_column("weather_sensor", "knowledge_horizon_fnc", nullable=False)
+
+    op.add_column(
+        "weather_sensor",
+        sa.Column(
+            "knowledge_horizon_par",
+            sa.JSON(),
+            nullable=True,
+            default={knowledge_horizons.ex_post.__code__.co_varnames[1]: "PT0H"},
+        ),
+    )
+    op.execute(
+        f"""update weather_sensor set knowledge_horizon_par = '{json.dumps(ex_post_default_par)}';"""
+    )
+    op.alter_column("weather_sensor", "knowledge_horizon_par", nullable=False)
+
+    op.add_column(
+        "weather_sensor", sa.Column("timezone", sa.String(length=80), nullable=True)
+    )
+    op.execute("update weather_sensor set timezone = 'Asia/Seoul';")
+    op.alter_column("weather_sensor", "timezone", nullable=False)
+
+    # todo: execute after adding relevant tests and updating our strategy to create forecasting jobs when new information arrives
+    # op.execute(
+    #     f"update market set knowledge_horizon_fnc = '{knowledge_horizons.x_days_ago_at_y_oclock.__name__}' where name in ('epex_da', 'kpx_da');"
+    # )
+    # op.execute(
+    #     """update market set knowledge_horizon_par = '{"x": 1, "y": 12, "z": "Europe/Paris"}' where name='epex_da';"""
+    # )  # gate closure at 12:00 on the preceding day, with expected price publication at 12.42 and 12.55 (from EPEX Spot Day-Ahead Multi-Regional Coupling, https://www.epexspot.com/en/downloads#rules-fees-processes )
+    # op.execute(
+    #     """update market set knowledge_horizon_par = '{"x": 1, "y": 10, "z": "Asia/Seoul"}' where name='kpx_da';"""
+    # )  # gate closure at 10.00 on the preceding day, with expected price publication at 15.00 (from KPX Power Market Operation, https://www.slideshare.net/sjchung0/power-market-operation )
+    # todo: add statement to update the horizon for prices on these markets, in accordance with their new knowledge horizon function
+
+
+def downgrade():
+    # Drop mixed in columns
+    op.drop_column("asset", "timezone")
+    op.drop_column("asset", "knowledge_horizon_par")
+    op.drop_column("asset", "knowledge_horizon_fnc")
+    op.drop_column("market", "timezone")
+    op.drop_column("market", "knowledge_horizon_par")
+    op.drop_column("market", "knowledge_horizon_fnc")
+    op.drop_column("weather_sensor", "timezone")
+    op.drop_column("weather_sensor", "knowledge_horizon_par")
+    op.drop_column("weather_sensor", "knowledge_horizon_fnc")
+    op.execute(
+        "update price set horizon = ((datetime + market.event_resolution) - '2014-12-31 00:00:00+00:00') from market where market_id = market.id and name in ('kepco_cs_fast', 'kepco_cs_slow', 'kepco_cs_smart');"
+    )  # rolling horizon before end of event (i.e. at publication date)

requirements/app.in

@@ -27,7 +27,7 @@ pyomo>=5.6
 forecastiopy
 pysolar
 timetomodel>=0.6.8
-timely-beliefs>=1.0.0
+timely-beliefs>=1.2.0
 python-dotenv
 Flask-SSLify
 Flask_JSON