Merge remote-tracking branch 'origin/main' into support-blackout-tests

# Conflicts:
#	documentation/changelog.rst

Makefile

@@ -1,6 +1,6 @@
 # Note: use tabs
 # actions which are virtual, i.e. not a script
-.PHONY: install install-for-dev install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model
+.PHONY: install install-for-dev install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
 
 
 # ---- Development ---
@@ -87,3 +87,6 @@ show-data-model:
 	# With --deprecated, you'll see the legacy models, and not their replacements.
 	# Use --help to learn more. 
 	./flexmeasures/data/scripts/visualize_data_model.py --uml
+
+clean-db:
+	./flexmeasures/data/scripts/clean_database.sh ${db_name}

documentation/changelog.rst

@@ -27,14 +27,15 @@ Bugfixes
 -----------
 
 * Fix copy button on tutorials and other documentation, so that only commands are copied and no output or comments [see `PR #636 <https://www.github.com/FlexMeasures/flexmeasures/pull/636>`_]
+* GET /api/v3_0/assets/public should ask for token authentication and not forward to login page [see `PR #649 <https://www.github.com/FlexMeasures/flexmeasures/pull/649>`_]
 
 Infrastructure / Support
 ----------------------
 * Support blackout tests for sunset API versions [see `PR #651 <https://www.github.com/FlexMeasures/flexmeasures/pull/651>`_]
 * Sunset API versions 1.0, 1.1, 1.2, 1.3 and 2.0 [see `PR #650 <https://www.github.com/FlexMeasures/flexmeasures/pull/650>`_]
 * Sunset several API fields for `/sensors/<id>/schedules/trigger` (POST) that have moved into the ``flex-model`` or ``flex-context`` fields [see `PR #580 <https://www.github.com/FlexMeasures/flexmeasures/pull/580>`_]
 * Fix broken `make show-data-model` command [see `PR #638 <https://www.github.com/FlexMeasures/flexmeasures/pull/638>`_]
-
+* Bash script for a clean database to run toy-tutorial by using `make clean-db db_name=database_name` command [see `PR #640 <https://github.com/FlexMeasures/flexmeasures/pull/640>`_]
 
 v0.12.3 | February 28, 2023
 ============================

documentation/host/data.rst

@@ -145,7 +145,7 @@ Write:
    SQLALCHEMY_DATABASE_URI = "postgresql://flexmeasures:<password>@127.0.0.1/flexmeasures"
 
 
-into the config file you are using, e.g. ~/flexmeasures.cfg
+into the config file you are using, e.g. ~/.flexmeasures.cfg
 
 
 Get structure (and some data) into place

documentation/tut/toy-example-from-scratch.rst

@@ -47,7 +47,7 @@ Install Flexmeasures and the database
             $ docker pull lfenergy/flexmeasures:latest
             $ docker pull postgres
             $ docker network create flexmeasures_network
-            $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=fm-db-passwd -e POSTGRES_DB=flexmeasures-db -d --network=flexmeasures_network postgres:latest 
+            $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=fm-db-passwd -e POSTGRES_DB=flexmeasures-db -d --network=flexmeasures_network postgres:latest
             $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:fm-db-passwd@flexmeasures-tutorial-db:5432/flexmeasures-db --env SECRET_KEY=notsecret --env FLASK_ENV=development --env LOGGING_LEVEL=INFO -d --network=flexmeasures_network -p 5000:5000 lfenergy/flexmeasures
             $ docker exec flexmeasures-tutorial-fm bash -c "flexmeasures db upgrade"
 
@@ -62,16 +62,16 @@ Install Flexmeasures and the database
         To leave the container session, hold CTRL-D or type "exit".
 
         To stop the containers, you can type
-        
+
         .. code-block:: bash
-        
+
             $ docker stop flexmeasures-tutorial-db
             $ docker stop flexmeasures-tutorial-fm
 
         To start the containers again, do this (note that re-running the `docker run` commands above *deletes and re-creates* all data!):
-        
+
         .. code-block:: bash
-        
+
             $ docker start flexmeasures-tutorial-db
             $ docker start flexmeasures-tutorial-fm
 
@@ -80,7 +80,7 @@ Install Flexmeasures and the database
         .. note:: Got docker-compose? You could run this tutorial with 5 containers :) ― Go to :ref:`docker-compose-tutorial`.
 
   .. tab:: On your PC
-        
+
         This example is from scratch, so we'll assume you have nothing prepared but a (Unix) computer with Python (3.8+) and two well-known developer tools, `pip <https://pip.pypa.io>`_ and `postgres <https://www.postgresql.org/download/>`_.
 
         We'll create a database for FlexMeasures:
@@ -98,10 +98,16 @@ Install Flexmeasures and the database
 
             $ pip install flexmeasures
             $ export SQLALCHEMY_DATABASE_URI="postgresql://flexmeasures-user:fm-db-passwd@localhost:5432/flexmeasures-db" SECRET_KEY=notsecret LOGGING_LEVEL="INFO" DEBUG=0
-            $ flexmeasures db upgrade 
+            $ export FLASK_ENV="development"
+            $ flexmeasures db upgrade
 
         .. note:: When installing with ``pip``, on some platforms problems might come up (e.g. macOS, Windows). One reason is that FlexMeasures requires some libraries with lots of C code support (e.g. Numpy). One way out is to use Docker, which uses a prepared Linux image, so it'll definitely work.
 
+        In case you want to re-run the tutorial, then it's recommended to delete the old database and create a fresh one. Run the following command to create a clean database.
+
+        .. code-block:: bash
+
+            $ make clean-db db_name=flexmeasures-db
 
 Add some structural data
 ---------------------------------------
@@ -121,35 +127,35 @@ FlexMeasures offers a command to create a toy account with a battery:
     The sensor recording day-ahead prices is <Sensor 2: day-ahead prices, unit: EUR/MWh res.: 1:00:00>.
     The sensor recording solar forecasts is <Sensor 3: production, unit: MW res.: 0:15:00>.
 
-And with that, we're done with the structural data for this tutorial! 
+And with that, we're done with the structural data for this tutorial!
 
 If you want, you can inspect what you created:
 
 .. code-block:: bash
 
-    $ flexmeasures show account --id 1                       
-    
+    $ flexmeasures show account --id 1
+
     ===========================
     Account Toy Account (ID: 1)
     ===========================
 
     Account has no roles.
 
     All users:
-    
+
       Id  Name      Email                     Last Login    Roles
     ----  --------  ------------------------  ------------  -------------
        1  toy-user  toy-user@flexmeasures.io                account-admin
 
     All assets:
-    
+
       ID  Name          Type      Location
     ----  ------------  --------  -----------------
        1  toy-battery   battery   (52.374, 4.88969)
        3  toy-solar     solar     (52.374, 4.88969)
 
     $ flexmeasures show asset --id 1
-    
+
     =========================
     Asset toy-battery (ID: 1)
     =========================
@@ -162,7 +168,7 @@ If you want, you can inspect what you created:
                                 sensors_to_show: [2, [3, 1]]
 
     All sensors in asset:
-    
+
       ID  Name         Unit    Resolution    Timezone          Attributes
     ----  -----------  ------  ------------  ----------------  ------------
        1  discharging  MW      15 minutes    Europe/Amsterdam
@@ -191,7 +197,7 @@ Now to add price data. First, we'll create the csv file with prices (EUR/MWh, se
 .. code-block:: bash
 
     $ TOMORROW=$(date --date="next day" '+%Y-%m-%d')
-    $ echo "Hour,Price                                      
+    $ echo "Hour,Price
     $ ${TOMORROW}T00:00:00,10
     $ ${TOMORROW}T01:00:00,11
     $ ${TOMORROW}T02:00:00,12
@@ -314,7 +320,7 @@ Great. Let's see what we made:
                             ██ discharging
 
 
-Here, negative values denote output from the grid, so that's when the battery gets charged. 
+Here, negative values denote output from the grid, so that's when the battery gets charged.
 
 We can also look at the charging schedule in the `FlexMeasures UI <http://localhost:5000/sensors/1/>`_ (reachable via the asset page for the battery):
 

flexmeasures/api/v3_0/assets.py

@@ -2,7 +2,7 @@
 
 from flask import current_app
 from flask_classful import FlaskView, route
-from flask_security import login_required
+from flask_security import auth_token_required
 from flask_json import as_json
 from marshmallow import fields
 from webargs.flaskparser import use_kwargs, use_args
@@ -81,7 +81,7 @@ def index(self, account: Account):
         return assets_schema.dump(account.generic_assets), 200
 
     @route("/public", methods=["GET"])
-    @login_required
+    @auth_token_required
     @as_json
     def public(self):
         """Return all public assets.

flexmeasures/api/v3_0/tests/test_assets_api.py

@@ -102,13 +102,22 @@ def test_get_assets(
         assert turbine["account_id"] == setup_accounts["Supplier"].id
 
 
+def test_get_public_assets_noauth(client, setup_api_test_data, setup_accounts):
+    get_assets_response = client.get(
+        url_for("AssetAPI:public"), headers={"content-type": "application/json"}
+    )
+    print("Server responded with:\n%s" % get_assets_response.json)
+    assert get_assets_response.status_code == 401
+
+
 def test_get_public_assets(client, setup_api_test_data, setup_accounts):
     auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest")
     get_assets_response = client.get(
         url_for("AssetAPI:public"),
         headers={"content-type": "application/json", "Authorization": auth_token},
     )
     print("Server responded with:\n%s" % get_assets_response.json)
+    assert get_assets_response.status_code == 200
     assert len(get_assets_response.json) == 1
     assert get_assets_response.json[0]["name"] == "troposphere"
 

flexmeasures/data/scripts/clean_database.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# save the current directory
+MAIN_DIR=$(pwd)
+
+# function for checking database existence
+function is_database() {
+ cd /tmp
+ sudo -u postgres psql -lqt | cut -d \| -f 1 | grep -wq $1
+ cd ${MAIN_DIR}
+}
+
+# function for creating a new database
+function create_database() {
+ echo "Creating a new database ..."
+ sudo -i -u postgres createdb -U postgres $1
+ echo "Creating cube extension in $1 ..."
+ sudo -i -u postgres psql -c "\c $1" -c "CREATE EXTENSION cube;"
+ echo "Creating earthdistance extension in $1 ..."
+ sudo -i -u postgres psql -c "\c $1" -c "CREATE EXTENSION earthdistance;"
+ echo "Updating database structure ..."
+ flexmeasures db upgrade
+}
+
+# function for deleting the old database
+function delete_database() {
+ echo "Deleting database ..."
+ sudo -i -u postgres dropdb -U postgres $1
+}
+
+# Check if the database name is provided
+if [ -z "$1" ]; then
+  echo "Error: db_name is required. Please provide a value for db_name, e.g., make clean_db db_name=flexmeasures-db"
+  exit 1
+fi
+
+# Check if the database exists
+if is_database $1
+then
+  echo "$1 exists"
+  read -r -p "Make a backup first? [y/N] " response
+  response=${response,,}    # make lowercase
+  if [[ "$response" =~ ^(yes|y)$ ]]; then
+    echo "Making db dump ..."
+    flexmeasures db-ops dump
+  fi
+
+  read -r -p "This will drop your database and re-create a clean one. Continue?[y/N] " response
+  response=${response,,} # make lowercase
+  if [[ "$response" =~ ^(yes|y)$ ]]; then
+     delete_database $1
+     create_database $1
+  fi
+
+# otherwise, create a fresh database
+else
+  echo "$1 does not exist"
+  create_database $1
+fi

requirements/app.txt

@@ -351,4 +351,4 @@ zipp==3.15.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+# setuptools

requirements/dev.txt

@@ -75,4 +75,4 @@ watchdog==3.0.0
     # via -r requirements/dev.in
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+# setuptools