From cbf76b1214374637256d748b2a0ccabf782980ca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 14 Oct 2021 09:37:36 +0200 Subject: [PATCH 1/6] Add optional model and version to DataSource --- ...ataSource_columns_for_model_and_version.py | 34 +++++++++++++++++++ flexmeasures/data/models/data_sources.py | 31 +++++++++++++++-- flexmeasures/data/utils.py | 32 ++++++++++++----- 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py diff --git a/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py new file mode 100644 index 000000000..24a9485b5 --- /dev/null +++ b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py @@ -0,0 +1,34 @@ +"""add optional DataSource columns for model and version + +Revision ID: 30fe2267e7d5 +Revises: 96f2db5bed30 +Create Date: 2021-10-11 10:54:24.348371 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "30fe2267e7d5" +down_revision = "96f2db5bed30" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "data_source", sa.Column("model", sa.String(length=80), nullable=True) + ) + op.add_column( + "data_source", sa.Column("version", sa.String(length=17), nullable=True) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("data_source", "version") + op.drop_column("data_source", "model") + # ### end Alembic commands ### diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 66696f37d..890352f2a 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -2,6 +2,7 @@ import timely_beliefs as tb from flask import current_app +from sqlalchemy.ext.hybrid import hybrid_method from flexmeasures.data.config import db from flexmeasures.data.models.user import User, is_user @@ -14,13 +15,20 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): # The type of data source (e.g. user, forecasting script or scheduling script) type = db.Column(db.String(80), default="") - # The id of the source (can link e.g. to fm_user table) + + # The id of the user source (can link e.g. to fm_user table) user_id = db.Column( db.Integer, db.ForeignKey("fm_user.id"), nullable=True, unique=True ) - user = db.relationship("User", backref=db.backref("data_source", lazy=True)) + # The model and version of a script source + model = db.Column(db.String(80), nullable=True) + version = db.Column( + db.String(17), # length supports up to version 999.999.999dev999 + nullable=True, + ) + def __init__( self, name: Optional[str] = None, @@ -54,6 +62,25 @@ def label(self): else: return f"data from {self.name}" + @hybrid_method + def description(self, include_model: bool = True, include_version: bool = True): + """Extended description + + For example: + + >>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description() + <<< "forecast by Seita (naive model v1.2)" + + """ + descr = self.label + if include_model and self.model: + descr += f" ({self.model} model" + if include_version and self.version: + descr += f" v{self.version})" + else: + descr += ")" + return descr + def __repr__(self): return "" % (self.id, self.label) diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 8efa09529..e20b24d2f 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import click @@ -16,20 +16,36 @@ def save_to_session(objects: List[db.Model], overwrite: bool = False): def get_data_source( - data_source_name: str, data_source_type: str = "script" + data_source_name: str, + data_source_model: Optional[str] = None, + data_source_version: Optional[str] = None, + data_source_type: str = "script", ) -> DataSource: """Make sure we have a data source. Create one if it doesn't exist, and add to session. Meant for scripts that may run for the first time. - It should probably not be used in the middle of a transaction, because we commit to the session.""" + """ data_source = DataSource.query.filter_by( - name=data_source_name, type=data_source_type + name=data_source_name, + model=data_source_model, + version=data_source_version, + type=data_source_type, ).one_or_none() if data_source is None: - data_source = DataSource(name=data_source_name, type=data_source_type) + data_source = DataSource( + name=data_source_name, + model=data_source_model, + version=data_source_version, + type=data_source_type, + ) db.session.add(data_source) db.session.flush() # populate the primary key attributes (like id) without committing the transaction - click.echo( - f'Session updated with new {data_source_type} data source named "{data_source_name}".' - ) + if data_source_model is None: + click.echo( + f'Session updated with new {data_source_type} data source named "{data_source_name}".' + ) + else: + click.echo( + f'Session updated with new {data_source_type} data source named "{data_source_name}" ({data_source_model} model v{data_source_version}).' + ) return data_source From 87562bbbff82f4d14f698963ce7f4ef181c2cf58 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 18 Oct 2021 14:12:26 +0200 Subject: [PATCH 2/6] Add unique constraint --- ...ional_DataSource_columns_for_model_and_version.py | 12 ++++++++---- flexmeasures/data/models/data_sources.py | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py index 24a9485b5..eac278c35 100644 --- a/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py +++ b/flexmeasures/data/migrations/versions/30fe2267e7d5_add_optional_DataSource_columns_for_model_and_version.py @@ -17,18 +17,22 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.add_column( "data_source", sa.Column("model", sa.String(length=80), nullable=True) ) op.add_column( "data_source", sa.Column("version", sa.String(length=17), nullable=True) ) - # ### end Alembic commands ### + op.create_unique_constraint( + "_data_source_name_user_id_model_version_key", + "data_source", + ["name", "user_id", "model", "version"], + ) def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "_data_source_name_user_id_model_version_key", "data_source", type_="unique" + ) op.drop_column("data_source", "version") op.drop_column("data_source", "model") - # ### end Alembic commands ### diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 890352f2a..0f91fab3d 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -12,6 +12,7 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): """Each data source is a data-providing entity.""" __tablename__ = "data_source" + __table_args__ = (db.UniqueConstraint("name", "user_id", "model", "version"),) # The type of data source (e.g. user, forecasting script or scheduling script) type = db.Column(db.String(80), default="") From 64655304ac9a93fd9b7629c275add2eeb8924fa1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 18 Oct 2021 14:13:11 +0200 Subject: [PATCH 3/6] Rewrite data source description to become a property --- flexmeasures/data/models/data_sources.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 0f91fab3d..fdc29a1db 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -2,7 +2,6 @@ import timely_beliefs as tb from flask import current_app -from sqlalchemy.ext.hybrid import hybrid_method from flexmeasures.data.config import db from flexmeasures.data.models.user import User, is_user @@ -63,23 +62,21 @@ def label(self): else: return f"data from {self.name}" - @hybrid_method - def description(self, include_model: bool = True, include_version: bool = True): + @property + def description(self): """Extended description For example: - >>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description() - <<< "forecast by Seita (naive model v1.2)" + >>> DataSource("Seita", type="forecasting script", model="naive", version="1.2").description + <<< "Seita's naive model v1.2.0" """ - descr = self.label - if include_model and self.model: - descr += f" ({self.model} model" - if include_version and self.version: - descr += f" v{self.version})" - else: - descr += ")" + descr = self.name + if self.model: + descr += f"'s {self.model} model" + if self.version: + descr += f" v{self.version}" return descr def __repr__(self): From 79aae044668d920d7827679dc75f484dffb46162 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 Oct 2021 10:06:27 +0200 Subject: [PATCH 4/6] Use new description property in DataSource object representation --- flexmeasures/data/models/data_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index fdc29a1db..e9ee20bd9 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -80,7 +80,7 @@ def description(self): return descr def __repr__(self): - return "" % (self.id, self.label) + return "" % (self.id, self.description) def get_or_create_source( From 0837a49fa0c196b451f69eaea4cb0d83fbe0e2a6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 Oct 2021 11:04:49 +0200 Subject: [PATCH 5/6] Use new description property to simplify util function --- flexmeasures/data/utils.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index e20b24d2f..fe78d8a81 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -40,12 +40,7 @@ def get_data_source( ) db.session.add(data_source) db.session.flush() # populate the primary key attributes (like id) without committing the transaction - if data_source_model is None: - click.echo( - f'Session updated with new {data_source_type} data source named "{data_source_name}".' - ) - else: - click.echo( - f'Session updated with new {data_source_type} data source named "{data_source_name}" ({data_source_model} model v{data_source_version}).' - ) + click.echo( + f'Session updated with new {data_source_type} data source "{data_source.__repr__()}".' + ) return data_source From b9ada7f19c70843d488a6f1aa8c28fad3f17a287 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 Oct 2021 12:44:00 +0200 Subject: [PATCH 6/6] Changelog entry --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index d972e5621..19a040c52 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ New features ----------- * Set a logo for the top left corner with the new FLEXMEASURES_MENU_LOGO_PATH setting [see `PR #184 `_] * Add an extra style-sheet which applies to all pages with the new FLEXMEASURES_EXTRA_CSS_PATH setting [see `PR #185 `_] +* Data sources can be further distinguished by what model (and version) they ran [see `PR #215 `_] Bugfixes -----------