DarkSky API replacement

documentation/changelog.rst

@@ -6,6 +6,8 @@ FlexMeasures Changelog
 v0.5.0 | May XX, 2021
 ===========================
 
+.. warning:: If you retrieve weather forecasts through FlexMeasures: we had to switch to OpenWeatherMap, as Dark Sky is closing. This requires an update to config variables ― the new setting is called ``OPENWEATHERMAP_API_KEY``.
+
 New features
 -----------
 * Allow plugins to overwrite UI routes and customise the teaser on the login form [see `PR #106 <http://www.github.com/SeitaBV/flexmeasures/pull/106>`_]
@@ -18,6 +20,7 @@ Bugfixes
 Infrastructure / Support
 ----------------------
 * Make assets use MW as their default unit and enforce that in CLI, as well (API already did) [see `PR #108 <http://www.github.com/SeitaBV/flexmeasures/pull/108>`_]
+* For weather forecasts, switch from Dark Sky (closed from Aug 1, 2021) to OpenWeatherMap API [see `PR #113 <http://www.github.com/SeitaBV/flexmeasures/pull/113>`_]
 * Re-use the database between automated tests, if possible. This shaves 2/3rd off of the time it takes for the FlexMeasures test suite to run [see `PR #115 <http://www.github.com/SeitaBV/flexmeasures/pull/115>`_]
 * Let CLI package and plugins use Marshmallow Field definitions [see `PR #125 <http://www.github.com/SeitaBV/flexmeasures/pull/125>`_]
 
@@ -35,6 +38,8 @@ Bugfixes
 v0.4.0 | April 29, 2021
 ===========================
 
+.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
+
 New features
 -----------
 * Configure the UI menu with ``FLEXMEASURES_LISTED_VIEWS`` [see `PR #91 <https://github.com/SeitaBV/flexmeasures/pull/91>`_]

documentation/configuration.rst

@@ -136,15 +136,10 @@ Default: ``timedelta(hours=2 * 24)``
 Tokens
 ------
 
-DARK_SKY_API_KEY
+OPENWEATHERMAP_API_KEY
 ^^^^^^^^^^^^^^^^
 
-Token for accessing the DarkSky weather forecasting service.
-
-.. note:: DarkSky will soon become non-public (Aug 1, 2021), so they are not giving out new tokens.
-          We'll use another service soon (`see this issue <https://github.com/SeitaBV/flexmeasures/issues/3>`_).
-          This is unfortunate.
-          In the meantime, if you can't find anybody lending their token, consider posting weather forecasts to the FlexMeasures database yourself.
+Token for accessing the OPenWeatherMap weather forecasting service.
 
 Default: ``None``
 

documentation/dev/data.rst

@@ -175,7 +175,7 @@ Then we import the data dump we made earlier:
 
 .. code-block:: bash
 
-   flask db-ops restore <DATABASE DUMP FILENAME>
+   flexmeasures db-ops restore <DATABASE DUMP FILENAME>
 
 
 A potential ``alembic_version`` error should not prevent other data tables from being restored.

flexmeasures/api/v2_0/routes.py

@@ -477,7 +477,7 @@ def reset_user_password(id: int):
     .. :quickref: User; Password reset
 
     Reset the user's password, and send them instructions on how to reset the password.
-    This endoint is useful from a security standpoint, in case of worries the password might be compromised.
+    This endpoint is useful from a security standpoint, in case of worries the password might be compromised.
     It sets the current password to something random, invalidates cookies and auth tokens,
     and also sends an email for resetting the password to the user.
 

flexmeasures/data/models/assets.py

@@ -232,7 +232,7 @@ def __init__(self, **kwargs):
         super(Power, self).__init__(**kwargs)
 
     def __repr__(self):
-        return "<Power %.2f on Asset %s at %s by DataSource %s, horizon %s>" % (
+        return "<Power %.5f on Asset %s at %s by DataSource %s, horizon %s>" % (
             self.value,
             self.asset_id,
             self.datetime,

flexmeasures/data/scripts/cli_tasks/data_add.py

@@ -467,11 +467,12 @@ def create_forecasts(
 
 
 @fm_add_data.command("external-weather-forecasts")
+@with_appcontext
 @click.option(
     "--region",
     type=str,
     default="",
-    help="Name of the region (will create sub-folder, should later tag the forecast in the DB, probably).",
+    help="Name of the region (will create sub-folder if you store json files, should later probably tag the forecast in the DB).",
 )
 @click.option(
     "--location",
@@ -486,7 +487,7 @@ def create_forecasts(
     "--num_cells",
     type=int,
     default=1,
-    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter.",
+    help="Number of cells on the grid. Only used if a region of interest has been mapped in the location parameter. Defaults to 1.",
 )
 @click.option(
     "--method",
@@ -497,13 +498,13 @@ def create_forecasts(
 @click.option(
     "--store-in-db/--store-as-json-files",
     default=False,
-    help="Store forecasts in the database, or simply save as json files.",
+    help="Store forecasts in the database, or simply save as json files. (defaults to json files)",
 )
 def collect_weather_data(region, location, num_cells, method, store_in_db):
     """
     Collect weather forecasts from the DarkSky API
 
-    This function can get weather data for one location or for several location within
+    This function can get weather data for one location or for several locations within
     a geometrical grid (See the --location parameter).
     """
     from flexmeasures.data.scripts.grid_weather import get_weather_forecasts

flexmeasures/data/scripts/grid_weather.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python
 
 import os
-from typing import Tuple, List
+from typing import Tuple, List, Dict
 import json
 from datetime import datetime
 
 import click
 from flask import Flask, current_app
-from forecastiopy import ForecastIO
+import requests
+import pytz
 
 from flexmeasures.utils.time_utils import as_server_time, get_timezone
 from flexmeasures.utils.geo_utils import compute_irradiance
@@ -18,7 +19,7 @@
 from flexmeasures.data.models.data_sources import DataSource
 
 FILE_PATH_LOCATION = "/../raw_data/weather-forecasts"
-DATA_SOURCE_NAME = "DarkSky"
+DATA_SOURCE_NAME = "OpenWeatherMap"
 
 
 class LatLngGrid(object):
@@ -217,12 +218,12 @@ def locations_hex(self) -> List[Tuple[float, float]]:
                 sw = (
                     lat + self.cell_size_lat / 2,
                     lng - self.cell_size_lat / 3 ** (1 / 2) / 2,
-                )  # South west coord.
+                )  # South west coordinates
                 locations.append(sw)
                 se = (
                     lat + self.cell_size_lat / 2,
                     lng + self.cell_size_lng / 3 ** (1 / 2) / 2,
-                )  # South east coord.
+                )  # South east coordinates
                 locations.append(se)
         return locations
 
@@ -317,22 +318,30 @@ def get_data_source() -> DataSource:
     return data_source
 
 
-def call_darksky(api_key: str, location: Tuple[float, float]) -> dict:
-    """Make a single call to the Dark Sky API and return the result parsed as dict"""
-    return ForecastIO.ForecastIO(
-        api_key,
-        units=ForecastIO.ForecastIO.UNITS_SI,
-        lang=ForecastIO.ForecastIO.LANG_ENGLISH,
-        latitude=location[0],
-        longitude=location[1],
-        extend="hourly",
-    ).forecast
+def call_openweatherapi(
+    api_key: str, location: Tuple[float, float]
+) -> Tuple[int, List[Dict]]:
+    """
+    Make a single "one-call" to the Open Weather API and return the API timestamp as well as the 48 hourly forecasts.
+    See https://openweathermap.org/api/one-call-api for docs.
+    Note that the first forecast is about the current hour.
+    """
+    query_str = f"lat={location[0]}&lon={location[1]}&units=metric&exclude=minutely,daily,alerts&appid={api_key}"
+    res = requests.get(f"http://api.openweathermap.org/data/2.5/onecall?{query_str}")
+    assert (
+        res.status_code == 200
+    ), f"OpenWeatherMap returned status code {res.status_code}: {res.text}"
+    data = res.json()
+    return data["current"]["dt"], data["hourly"]
 
 
 def save_forecasts_in_db(
-    api_key: str, locations: List[Tuple[float, float]], data_source: DataSource
+    api_key: str,
+    locations: List[Tuple[float, float]],
+    data_source: DataSource,
+    max_degree_difference_for_nearest_weather_sensor: int = 2,
 ):
-    """Process the response from DarkSky into Weather timed values.
+    """Process the response from OpenWeatherMap API into Weather timed values.
     Collects all forecasts for all locations and all sensors at all locations, then bulk-saves them.
     """
     click.echo("[FLEXMEASURES] Getting weather forecasts:")
@@ -344,22 +353,24 @@ def save_forecasts_in_db(
     for location in locations:
         click.echo("[FLEXMEASURES] %s, %s" % location)
 
-        forecasts = call_darksky(api_key, location)
+        api_timestamp, forecasts = call_openweatherapi(api_key, location)
         time_of_api_call = as_server_time(
-            datetime.fromtimestamp(forecasts["currently"]["time"], get_timezone())
+            datetime.fromtimestamp(api_timestamp, tz=get_timezone())
         ).replace(second=0, microsecond=0)
         click.echo(
-            "[FLEXMEASURES] Called Dark Sky API successfully at %s." % time_of_api_call
+            "[FLEXMEASURES] Called OpenWeatherMap API successfully at %s."
+            % time_of_api_call
         )
 
-        # map sensor name in our db to sensor name/label in dark sky response
+        # map sensor name in our db to sensor name/label in OWM response
         sensor_name_mapping = dict(
-            temperature="temperature", wind_speed="windSpeed", radiation="cloudCover"
+            temperature="temp", wind_speed="wind_speed", radiation="clouds"
         )
 
-        for fc in forecasts["hourly"]["data"]:
+        # loop through forecasts, including the one of current hour (horizon 0)
+        for fc in forecasts:
             fc_datetime = as_server_time(
-                datetime.fromtimestamp(fc["time"], get_timezone())
+                datetime.fromtimestamp(fc["dt"], get_timezone())
             ).replace(second=0, microsecond=0)
             fc_horizon = fc_datetime - time_of_api_call
             click.echo(
@@ -375,6 +386,16 @@ def save_forecasts_in_db(
                             flexmeasures_sensor_type, lat=location[0], lng=location[1]
                         )
                         if weather_sensor is not None:
+                            # Complain if the nearest weather sensor is further away than 2 degrees
+                            if abs(
+                                location[0] - weather_sensor.latitude
+                            ) > max_degree_difference_for_nearest_weather_sensor or abs(
+                                location[1] - weather_sensor.longitude
+                                > max_degree_difference_for_nearest_weather_sensor
+                            ):
+                                raise Exception(
+                                    f"No sufficiently close weather sensor found (within 2 degrees distance) for type {flexmeasures_sensor_type}! We're looking for: {location}, closest available: ({weather_sensor.latitude}, {weather_sensor.longitude})"
+                                )
                             weather_sensors[flexmeasures_sensor_type] = weather_sensor
                         else:
                             raise Exception(
@@ -383,13 +404,14 @@ def save_forecasts_in_db(
                             )
 
                     fc_value = fc[needed_response_label]
-                    # the radiation is not available in dark sky -> we compute it ourselves
+                    # the radiation is not available in OWM -> we compute it ourselves
                     if flexmeasures_sensor_type == "radiation":
                         fc_value = compute_irradiance(
                             location[0],
                             location[1],
                             fc_datetime,
-                            fc[needed_response_label],
+                            # OWM sends cloud coverage in percent, we need a ratio
+                            fc[needed_response_label] / 100.0,
                         )
 
                     db_forecasts.append(
@@ -424,15 +446,19 @@ def save_forecasts_as_json(
     click.echo("[FLEXMEASURES] Getting weather forecasts:")
     click.echo("[FLEXMEASURES]  Latitude, Longitude")
     click.echo("[FLEXMEASURES]  ----------------------")
-    # UTC timestamp to remember when data was fetched.
-    now_str = datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S")
-    os.mkdir("%s/%s" % (data_path, now_str))
     for location in locations:
         click.echo("[FLEXMEASURES] %s, %s" % location)
-        forecasts = call_darksky(api_key, location)
-        forecasts_file = "%s/%s/forecast_lat_%s_lng_%s.json" % (
-            data_path,
-            now_str,
+        api_timestamp, forecasts = call_openweatherapi(api_key, location)
+        time_of_api_call = as_server_time(
+            datetime.fromtimestamp(api_timestamp, tz=pytz.utc)
+        ).replace(second=0, microsecond=0)
+        now_str = time_of_api_call.strftime("%Y-%m-%dT%H-%M-%S")
+        path_to_files = os.path.join(data_path, now_str)
+        if not os.path.exists(path_to_files):
+            click.echo(f"Making directory: {path_to_files} ...")
+            os.mkdir(path_to_files)
+        forecasts_file = "%s/forecast_lat_%s_lng_%s.json" % (
+            path_to_files,
             str(location[0]),
             str(location[1]),
         )
@@ -451,11 +477,11 @@ def get_weather_forecasts(
 ):
     """
     Get current weather forecasts for a latitude/longitude grid and store them in individual json files.
-    Note that 1000 free calls per day can be made to the Dark Sky API,
-    so we can make a call every 15 minutes for up to 10 assets or every hour for up to 40 assets.
+    Note that 1000 free calls per day can be made to the OpenWeatherMap API,
+    so we can make a call every 15 minutes for up to 10 assets or every hour for up to 40 assets (or get a paid account).
     """
-    if app.config.get("DARK_SKY_API_KEY") is None:
-        raise Exception("No DarkSky API key available.")
+    if app.config.get("OPENWEATHERMAP_API_KEY") is None:
+        raise Exception("Setting OPENWEATHERMAP_API_KEY not available.")
 
     if (
         location.count(",") == 0
@@ -504,7 +530,7 @@ def get_weather_forecasts(
     else:
         raise Exception("location parameter '%s' has too many locations." % location)
 
-    api_key = app.config.get("DARK_SKY_API_KEY")
+    api_key = app.config.get("OPENWEATHERMAP_API_KEY")
 
     # Save the results
     if store_in_db:

flexmeasures/utils/config_defaults.py

@@ -65,7 +65,7 @@ class Config(object):
     CORS_RESOURCES: Union[dict, list, str] = [r"/api/*"]
     CORS_SUPPORTS_CREDENTIALS: bool = True
 
-    DARK_SKY_API_KEY: Optional[str] = None
+    OPENWEATHERMAP_API_KEY: Optional[str] = None
 
     MAPBOX_ACCESS_TOKEN: Optional[str] = None
 

requirements/app.in

@@ -26,7 +26,6 @@ rq-win; os_name == 'nt' or os_name == 'win'
 redis; os_name == 'nt' or os_name == 'win'
 tldextract
 pyomo>=5.6
-forecastiopy
 pvlib
 # the following three are optional in pvlib, but we use them
 netCDF4

requirements/app.txt

@@ -99,8 +99,6 @@ flask==1.1.2
     #   flask-sslify
     #   flask-wtf
     #   rq-dashboard
-forecastiopy==0.22
-    # via -r requirements/app.in
 greenlet==1.0.0
     # via sqlalchemy
 humanize==3.3.0
@@ -260,7 +258,6 @@ requests-file==1.5.1
     # via tldextract
 requests==2.25.1
     # via
-    #   forecastiopy
     #   pvlib
     #   requests-file
     #   siphon