Cli csv upload

flexmeasures/data/models/data_sources.py

@@ -46,7 +46,7 @@ def label(self):
             return f"schedule by {self.name}"
         elif self.type == "crawling script":
             return f"data retrieved from {self.name}"
-        elif self.type == "demo script":
+        elif self.type in ("demo script", "CLI script"):
             return f"demo data entered by {self.name}"
         else:
             return f"data from {self.name}"

flexmeasures/data/scripts/cli_tasks/data_add.py

@@ -1,7 +1,7 @@
 """CLI Tasks for (de)populating the database - most useful in development"""
 
 from datetime import timedelta
-from typing import List
+from typing import List, Optional
 
 import pandas as pd
 import pytz
@@ -10,13 +10,17 @@
 from flask_security.utils import hash_password
 import click
 import getpass
+import timely_beliefs as tb
 
+from flexmeasures.data import db
 from flexmeasures.data.services.forecasting import create_forecasting_jobs
 from flexmeasures.data.services.users import create_user
-from flexmeasures.data.models.time_series import Sensor, SensorSchema
+from flexmeasures.data.models.time_series import Sensor, SensorSchema, TimedBelief
 from flexmeasures.data.models.assets import Asset, AssetSchema
 from flexmeasures.data.models.markets import Market
 from flexmeasures.data.models.weather import WeatherSensor, WeatherSensorSchema
+from flexmeasures.data.models.data_sources import DataSource
+from flexmeasures.utils.time_utils import server_now
 
 
 @click.group("add")
@@ -201,6 +205,78 @@ def add_initial_structure():
     populate_structure(app.db)
 
 
+@fm_add_data.command("beliefs")
+@with_appcontext
+@click.argument("file", type=click.Path(exists=True))
+@click.option(
+    "--sensor-id",
+    required=True,
+    type=click.IntRange(min=1),
+    help="Sensor to which the beliefs pertain.",
+)
+@click.option(
+    "--horizon",
+    required=False,
+    type=click.IntRange(),
+    help="Belief horizon in minutes (use positive horizon for ex-ante beliefs or negative horizon for ex-post beliefs).",
+)
+@click.option(
+    "--cp",
+    required=False,
+    type=click.FloatRange(0, 1),
+    help="Cumulative probability in the range [0, 1].",
+)
+def add_beliefs(
+    file: str, sensor_id: int, horizon: Optional[int] = None, cp: Optional[float] = None
+):
+    """Add sensor data from a csv file.
+
+    Structure your csv file as follows:
+
+        - One header line (will be ignored!)
+        - UTC datetimes in 1st column
+        - values in 2nd column
+
+    For example:
+
+        Date,Inflow (cubic meter)
+        2020-12-03 14:00,212
+        2020-12-03 14:10,215.6
+        2020-12-03 14:20,203.8
+
+    In case no --horizon is specified, the moment of executing this CLI command is taken
+    as the time at which the beliefs were recorded.
+    """
+    sensor = Sensor.query.filter(Sensor.id == sensor_id).one_or_none()
+    source = (
+        DataSource.query.filter(DataSource.name == "Seita")
+        .filter(DataSource.type == "CLI script")
+        .one_or_none()
+    )
+    if not source:
+        print("SETTING UP CLI SCRIPT AS NEW DATA SOURCE...")
+        source = DataSource(name="Seita", type="CLI script")
+        db.session.add(source)
+    bdf = tb.read_csv(
+        file,
+        sensor,
+        source=source,
+        cumulative_probability=cp,
+        parse_dates=True,
+        infer_datetime_format=True,
+        **(
+            dict(belief_horizon=timedelta(minutes=horizon))
+            if horizon is not None
+            else dict(
+                belief_time=server_now().astimezone(pytz.timezone(sensor.timezone))
+            )
+        ),
+    )
+    TimedBelief.add(bdf, commit_transaction=False)
+    db.session.commit()
+    print(f"Successfully created beliefs\n{bdf}")
+
+
 @fm_add_data.command("forecasts")
 @with_appcontext
 @click.option(

requirements/app.in

@@ -32,7 +32,7 @@ netCDF4
 siphon
 tables
 timetomodel>=0.6.8
-timely-beliefs>=1.3.0
+timely-beliefs>=1.3.1
 python-dotenv
 # a backport, not needed in Python3.8
 importlib_metadata