Update dependencies (#82)

Update dependencies, including a new major release of flask-security-too and a minor release of sqlalchemy.



* Add developer requirements

* New pip-tools formatting since 5.5.0

* Update flake8 and add hints in our yaml on where to find new version tags

* make upgrade-deps

* Set explicit minimum security dependency

* Fix explicit minimum security dependency

* Fix tests with deprecation of sqlalchemy RowProxy in 1.4

* One more upgrade

* Make test after upgrading dependencies

* Changelog entry

* Set minimum sqlalchemy dependency

* Two more recent upgrades

Co-authored-by: F.N. Claessen <felix@seita.nl>

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0  # New version tags can be found here: https://gitlab.com/pycqa/flake8/-/tags
     hooks:
     - id: flake8
       name: flake8 (code linting)
       language_version: python3.8
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 20.8b1  # New version tags can be found here: https://github.com/psf/black/tags
     hooks:
     - id: black
       name: black (code formatting)

Makefile

@@ -53,6 +53,7 @@ upgrade-deps:
 	pip-compile --upgrade -o requirements/app.txt requirements/app.in
 	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in
+	make test
 
 
 # ---- Data ----

documentation/changelog.rst

@@ -3,6 +3,16 @@ FlexMeasures Changelog
 **********************
 
 
+v0.2.5 | April XX, 2021
+===========================
+
+Infrastructure / Support
+----------------------
+* Updated dependencies, including Flask-Security-Too [see `PR #82 <http://www.github.com/SeitaBV/flexmeasures/pull/82>`_]
+
+
+
+
 v0.2.4 | April 2, 2021
 ===========================
 

flexmeasures/data/queries/utils.py

@@ -5,7 +5,6 @@
 import timely_beliefs as tb
 
 from sqlalchemy.orm import Query, Session
-from sqlalchemy.engine.result import RowProxy
 
 from flexmeasures.data.config import db
 from flexmeasures.data.models.data_sources import DataSource
@@ -158,32 +157,6 @@ def add_belief_timing_filter(
     return query
 
 
-def parse_sqlalchemy_results(results: List[RowProxy]) -> List[dict]:
-    """
-    Returns a list of dicts, whose keys are column names. E.g.:
-
-    data = session.execute("Select latitude from asset;").fetchall()
-    for row in parse_sqlalchemy_results(data):
-        print("------------")
-        for key, val in row:
-            print f"{key}: {val}"
-
-    """
-    parsed_results: List[dict] = []
-
-    if len(results) == 0:
-        return parsed_results
-
-    # results from SQLAlchemy are returned as a list of tuples;
-    # this procedure converts it into a list of dicts
-    for row_number, row in enumerate(results):
-        parsed_results.append({})
-        for column_number, value in enumerate(row):
-            parsed_results[row_number][row.keys()[column_number]] = value
-
-    return parsed_results
-
-
 def simplify_index(
     bdf: tb.BeliefsDataFrame, index_levels_to_columns: Optional[List[str]] = None
 ) -> pd.DataFrame:

flexmeasures/data/scripts/data_gen.py

@@ -19,13 +19,13 @@
 import inflect
 
 from flexmeasures.data.models.markets import MarketType, Market, Price
+from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.models.assets import AssetType, Asset, Power
 from flexmeasures.data.models.data_sources import DataSource
 from flexmeasures.data.models.weather import WeatherSensorType, WeatherSensor, Weather
 from flexmeasures.data.models.user import User, Role, RolesUsers
 from flexmeasures.data.models.forecasting import lookup_model_specs_configurator
 from flexmeasures.data.models.forecasting.exceptions import NotEnoughDataException
-from flexmeasures.data.queries.utils import parse_sqlalchemy_results
 from flexmeasures.utils.time_utils import ensure_local_timezone
 from flexmeasures.data.transactional import as_transaction
 
@@ -641,7 +641,7 @@ def load_tables(
         affected_classes = get_affected_classes(structure, data)
         statement = "SELECT sequence_name from information_schema.sequences;"
         data = db.session.execute(statement).fetchall()
-        sequence_names = [s["sequence_name"] for s in parse_sqlalchemy_results(data)]
+        sequence_names = [s.sequence_name for s in data]
         for c in affected_classes:
             file_path = "%s/%s/%s.obj" % (backup_path, backup_name, c.__tablename__)
             sequence_name = "%s_id_seq" % c.__tablename__
@@ -682,6 +682,7 @@ def get_affected_classes(structure: bool = True, data: bool = False) -> List:
             Role,
             User,
             RolesUsers,
+            Sensor,
             MarketType,
             Market,
             AssetType,

flexmeasures/data/services/resources.py

@@ -13,6 +13,7 @@
 import inflect
 import pandas as pd
 from sqlalchemy.orm import Query, Session
+from sqlalchemy.engine import Row
 import timely_beliefs as tb
 
 from flexmeasures.data.models.assets import (
@@ -24,7 +25,7 @@
 from flexmeasures.data.models.markets import Market, Price
 from flexmeasures.data.models.weather import Weather, WeatherSensor, WeatherSensorType
 from flexmeasures.data.models.user import User
-from flexmeasures.data.queries.utils import simplify_index, parse_sqlalchemy_results
+from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.services.time_series import aggregate_values
 from flexmeasures.utils.geo_utils import parse_lat_lng
 from flexmeasures.utils import coding_utils, time_utils
@@ -207,16 +208,14 @@ def get_center_location(db: Session, user: Optional[User]) -> Tuple[float, float
     )
     if user and not user.has_role("admin"):
         query += f" where owner_id = {user.id}"
-    locations: List[dict] = parse_sqlalchemy_results(
-        db.session.execute(query + ";").fetchall()
-    )
+    locations: List[Row] = db.session.execute(query + ";").fetchall()
     if (
         len(locations) == 0
-        or locations[0]["latitude"] is None
-        or locations[0]["longitude"] is None
+        or locations[0].latitude is None
+        or locations[0].longitude is None
     ):
-        return (52.38, 4.88)  # Amsterdam, NL
-    return locations[0]["latitude"], locations[0]["longitude"]
+        return 52.366, 4.904  # Amsterdam, NL
+    return locations[0].latitude, locations[0].longitude
 
 
 def check_cache(attribute):

requirements/app.in

@@ -36,14 +36,15 @@ timely-beliefs>=1.2.1
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata
+sqlalchemy>=1.4.0
 Flask-SSLify
 Flask_JSON
 Flask-SQLAlchemy>=2.4.3
 Flask-Migrate
 Flask-WTF
 Flask-Login
 Flask-Mail
-Flask-Security-Too
+Flask-Security-Too>=4.0
 Flask-Classful
 Flask-Marshmallow
 Flask-Cors