From c0554f46d69d7b94199583170800a755c345858b Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Wed, 20 Oct 2021 13:04:22 +0200 Subject: [PATCH] Add optional model and version to DataSource (#215) Allow data sources to be distinguished by what model (and version) they ran. * Add optional model and version to DataSource * Add unique constraint * Rewrite data source description to become a property * Use new description property in DataSource object representation * Use new description property to simplify util function * Changelog entry --- documentation/changelog.rst | 1 + ...ataSource_columns_for_model_and_version.py | 38 +++++++++++++++++++ flexmeasures/data/models/data_sources.py | 31 +++++++++++++-- flexmeasures/data/utils.py | 23 ++++++++--- 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index bc2336832..2fa396eee 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ New features ----------- * Set a logo for the top left corner with the new FLEXMEASURES_MENU_LOGO_PATH setting [see `PR #184 `_] * Add an extra style-sheet which applies to all pages with the new FLEXMEASURES_EXTRA_CSS_PATH setting [see `PR #185 `_] +* Data sources can be further distinguished by what model (and version) they ran [see `PR #215 `_] Bugfixes ----------- diff --git a/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py new file mode 100644 index 000000000..eac278c35 --- /dev/null +++ b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py @@ -0,0 +1,38 @@ +"""add optional DataSource columns for model and version + +Revision ID: 30fe2267e7d5 +Revises: 96f2db5bed30 +Create Date: 2021-10-11 10:54:24.348371 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "30fe2267e7d5" +down_revision = "96f2db5bed30" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "data_source", sa.Column("model", sa.String(length=80), nullable=True) + ) + op.add_column( + "data_source", sa.Column("version", sa.String(length=17), nullable=True) + ) + op.create_unique_constraint( + "_data_source_name_user_id_model_version_key", + "data_source", + ["name", "user_id", "model", "version"], + ) + + +def downgrade(): + op.drop_constraint( + "_data_source_name_user_id_model_version_key", "data_source", type_="unique" + ) + op.drop_column("data_source", "version") + op.drop_column("data_source", "model") diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 66696f37d..e9ee20bd9 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -11,16 +11,24 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): """Each data source is a data-providing entity.""" __tablename__ = "data_source" + __table_args__ = (db.UniqueConstraint("name", "user_id", "model", "version"),) # The type of data source (e.g. user, forecasting script or scheduling script) type = db.Column(db.String(80), default="") - # The id of the source (can link e.g. to fm_user table) + + # The id of the user source (can link e.g. to fm_user table) user_id = db.Column( db.Integer, db.ForeignKey("fm_user.id"), nullable=True, unique=True ) - user = db.relationship("User", backref=db.backref("data_source", lazy=True)) + # The model and version of a script source + model = db.Column(db.String(80), nullable=True) + version = db.Column( + db.String(17), # length supports up to version 999.999.999dev999 + nullable=True, + ) + def __init__( self, name: Optional[str] = None, @@ -54,8 +62,25 @@ def label(self): else: return f"data from {self.name}" + @property + def description(self): + """Extended description + + For example: + + >>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description + <<< "Seita's naive model v1.2.0" + + """ + descr = self.name + if self.model: + descr += f"'s {self.model} model" + if self.version: + descr += f" v{self.version}" + return descr + def __repr__(self): - return "" % (self.id, self.label) + return "" % (self.id, self.description) def get_or_create_source( diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 8efa09529..fe78d8a81 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import click @@ -16,20 +16,31 @@ def save_to_session(objects: List[db.Model], overwrite: bool = False): def get_data_source( - data_source_name: str, data_source_type: str = "script" + data_source_name: str, + data_source_model: Optional[str] = None, + data_source_version: Optional[str] = None, + data_source_type: str = "script", ) -> DataSource: """Make sure we have a data source. Create one if it doesn't exist, and add to session. Meant for scripts that may run for the first time. - It should probably not be used in the middle of a transaction, because we commit to the session.""" + """ data_source = DataSource.query.filter_by( - name=data_source_name, type=data_source_type + name=data_source_name, + model=data_source_model, + version=data_source_version, + type=data_source_type, ).one_or_none() if data_source is None: - data_source = DataSource(name=data_source_name, type=data_source_type) + data_source = DataSource( + name=data_source_name, + model=data_source_model, + version=data_source_version, + type=data_source_type, + ) db.session.add(data_source) db.session.flush() # populate the primary key attributes (like id) without committing the transaction click.echo( - f'Session updated with new {data_source_type} data source named "{data_source_name}".' + f'Session updated with new {data_source_type} data source "{data_source.__repr__()}".' ) return data_source