From 18c53a0f330fa22d4c2b1153eff0e3e0fdd0a9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 24 Mar 2021 10:53:13 +0100 Subject: [PATCH] Pip-installable (#54) Make it possible to install FlexMeasures from pip and then configure and run it. In the process, - use setuptools_scm to generate our versions (and simplify to_pypi.sh) - let `flexmeasures` be the name of our CLI (instead of flask) - improve installation advice, including adding a complete configuration settings overview - add better support on startup for setting a secret key and for warnings about configuration settings which are not necessary to start the platform but are needed for some features. - remove custom theme code --- .gitignore | 3 + Configuration.md | 277 ++++++++++++++++++ Developing.md | 32 +- Installation.md | 159 +++++++--- Makefile | 2 +- NOTICE | 6 +- documentation/changelog.rst | 11 +- documentation/conf.py | 13 +- flexmeasures/__init__.py | 13 +- flexmeasures/api/__init__.py | 2 + flexmeasures/api/common/schemas/__init__.py | 0 flexmeasures/app.py | 23 +- flexmeasures/data/Readme.md | 44 +-- flexmeasures/data/__init__.py | 4 +- .../data/migrations}/README | 0 .../data/migrations}/alembic.ini | 0 .../data/migrations}/env.py | 0 .../data/migrations}/script.py.mako | 0 .../versions/01fe99da5716_initial.py | 0 .../02ddbbff29a7_naming_conventions.py | 0 ..._power_table_and_drop_measurement_table.py | 0 .../1a4f0e5c4b86_unique_userids_in_ds.py | 0 .../1b64acf01809_forecasting_job_table.py | 0 .../versions/1bcccdf0c3e1_unique_usernames.py | 0 .../1e8d27922f56_create_price_table.py | 0 ...or_with_asset_market_and_weather_sensor.py | 0 ...owner_deletion_deletes_assets_and_power.py | 10 +- ...lumns_in_power_price_and_weather_tables.py | 0 .../versions/31f251554682_merge.py | 0 ...5_drop_login_columns_in_bvp_users_table.py | 0 ...subclass_of_timely_beliefs_beliefsource.py | 0 .../3e43d3274d16_Asset_soc_udi_event_id.py | 0 .../45d937300b0f_create_measurement_table.py | 0 ...e_and_weather_sensor_and_weather_tables.py | 0 ...et_tables_and_generic_asset_type_tables.py | 0 ...default_resolution_for_existing_sensors.py | 0 ...e8df4e3a9_stop_using_bvp_in_table_names.py | 0 .../5d39829d91af_create_data_sources_table.py | 0 ...c6e45c4d_add_soc_columns_in_asset_table.py | 0 ...s_table_and_make_display_names_nullable.py | 0 ...7987667dbd43_add_asset_type_hover_label.py | 8 +- ...a204_add_owner_id_column_in_asset_table.py | 0 ...ake_data_source_id_columns_primary_keys.py | 0 .../versions/919dc9f1dc1f_merge.py | 0 ...lumns_to_power_price_and_weather_tables.py | 0 ...c2_create_market_type_and_market_tables.py | 0 ...ocation_columns_to_weather_sensor_table.py | 0 ..._add_timezone_column_in_bvp_users_table.py | 0 .../a5b970eadb3b_time_series_indexes.py | 0 ...c74_add_market_id_column_to_asset_table.py | 0 ...87ce8b529f_create_latest_task_run_table.py | 0 ...c40_weathersensors_unique_type_location.py | 0 ...add_user_fs_uniquifier_for_faster_auth_.py | 0 ...a3_add_event_resolution_field_to_asset_.py | 0 ...nd_bvp_users_and_bvp_roles_users_tables.py | 0 ...82c_add_horizon_columns_as_primary_keys.py | 0 ...nd_display_name_columns_to_market_table.py | 0 ..._source_id_column_in_data_sources_table.py | 0 .../forecasting/model_specs/__init__.py | 0 .../scripts/cli_tasks/background_workers.py | 4 +- flexmeasures/data/scripts/cli_tasks/db_pop.py | 20 +- flexmeasures/data/services/time_series.py | 19 +- flexmeasures/data/transactional.py | 5 +- flexmeasures/ui/__init__.py | 2 +- .../ui/templates/admin/login_user.html | 6 +- flexmeasures/ui/templates/base.html | 24 +- flexmeasures/ui/utils/__init__.py | 0 flexmeasures/ui/utils/view_utils.py | 44 +-- flexmeasures/ui/views/analytics.py | 113 +++---- flexmeasures/utils/app_utils.py | 64 +++- flexmeasures/utils/config_defaults.py | 88 +++--- flexmeasures/utils/config_utils.py | 96 ++++-- requirements/app.in | 2 + requirements/app.txt | 2 + requirements/dev.in | 3 +- requirements/dev.txt | 1 + flexmeasures/run-local.py => run-local.py | 0 setup.cfg | 7 +- setup.py | 32 +- to_pypi.sh | 38 +++ 80 files changed, 888 insertions(+), 289 deletions(-) create mode 100644 Configuration.md create mode 100644 flexmeasures/api/common/schemas/__init__.py rename {migrations => flexmeasures/data/migrations}/README (100%) rename {migrations => flexmeasures/data/migrations}/alembic.ini (100%) rename {migrations => flexmeasures/data/migrations}/env.py (100%) rename {migrations => flexmeasures/data/migrations}/script.py.mako (100%) rename {migrations => flexmeasures/data/migrations}/versions/01fe99da5716_initial.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/02ddbbff29a7_naming_conventions.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/11b735abebe7_create_power_table_and_drop_measurement_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/1a4f0e5c4b86_unique_userids_in_ds.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/1b64acf01809_forecasting_job_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/1bcccdf0c3e1_unique_usernames.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/1e8d27922f56_create_price_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py (72%) rename {migrations => flexmeasures/data/migrations}/versions/2c9a32614784_rename_data_source_columns_in_power_price_and_weather_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/31f251554682_merge.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/3d56402cde15_drop_login_columns_in_bvp_users_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/3db3e71d101d_make_datasource_a_subclass_of_timely_beliefs_beliefsource.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/3e43d3274d16_Asset_soc_udi_event_id.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/45d937300b0f_create_measurement_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/4b6cebbdf473_create_weather_sensor_type_and_weather_sensor_and_weather_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/50cf294e007d_complete_adding_units_to_all_generic_asset_tables_and_display_names_to_all_generic_asset_tables_and_generic_asset_type_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/550a9020f1bf_default_resolution_for_existing_sensors.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/564e8df4e3a9_stop_using_bvp_in_table_names.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/5d39829d91af_create_data_sources_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/61bfc6e45c4d_add_soc_columns_in_asset_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/7113b0f00678_drop_forecasting_jobs_table_and_make_display_names_nullable.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/7987667dbd43_add_asset_type_hover_label.py (92%) rename {migrations => flexmeasures/data/migrations}/versions/8abe32ffa204_add_owner_id_column_in_asset_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/8fcc5fec67bc_make_data_source_id_columns_primary_keys.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/919dc9f1dc1f_merge.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/91a938bfa5a8_add_horizon_columns_to_power_price_and_weather_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/9254559dcac2_create_market_type_and_market_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/9c7fc8e46f1e_add_location_columns_to_weather_sensor_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/a328412b4623_add_timezone_column_in_bvp_users_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/a5b970eadb3b_time_series_indexes.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/ac2613fffc74_add_market_id_column_to_asset_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/b087ce8b529f_create_latest_task_run_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/b2b43f0eec40_weathersensors_unique_type_location.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/b797328ac32d_add_user_fs_uniquifier_for_faster_auth_.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/bddc5e9f72a3_add_event_resolution_field_to_asset_.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/d3440de27ab9_create_bvp_roles_and_bvp_users_and_bvp_roles_users_tables.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/db00b66be82c_add_horizon_columns_as_primary_keys.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/db1f67336324_add_price_unit_and_display_name_columns_to_market_table.py (100%) rename {migrations => flexmeasures/data/migrations}/versions/e0c2f9aff251_rename_source_id_column_in_data_sources_table.py (100%) create mode 100644 flexmeasures/data/models/forecasting/model_specs/__init__.py create mode 100644 flexmeasures/ui/utils/__init__.py rename flexmeasures/run-local.py => run-local.py (100%) create mode 100755 to_pypi.sh diff --git a/.gitignore b/.gitignore index 858ac55f3..c0775eda1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +build +dist + raw_data instance venv diff --git a/Configuration.md b/Configuration.md new file mode 100644 index 000000000..cf9858fac --- /dev/null +++ b/Configuration.md @@ -0,0 +1,277 @@ +# Configuration + +The following configurations are used by FlexMeasures. + +Required settings (e.g. postgres db) are marked with a double star (\*\*). +To enable easier quickstart tutorials, these settings can be set by env vars. +Recommended settings (e.g. mail, redis) are marked by one star (\*). + +Note: FlexMeasures is best configured via a config file. The config file for FlexMeasures can be placed in one of two locations: + +* in the user's home directory (e.g. `~/.flexmeasures.cfg` on Unix). In this case, note the dot at the beginning of the filename! +* in the apps's instance directory (e.g. `/path/to/your/flexmeasures/code/instance/flexmeasures.cfg`). The path to that instance directory is shown to you by running flexmeasures (e.g. `flexmeasures run`) with required settings missing or otherwise by running `flexmeasures shell`. + + +## Basic functionality + +### LOGGING_LEVEL +Level above which log messages are added to the log file. See the `logging` package in the Python standard library. + +Default: `logging.WARNING` + +### FLEXMEASURES_MODE +The mode in which FlexMeasures is being run, e.g. "demo" or "play". +This is used to turn on certain extra behaviours. + +Default: `""` + +### FLEXMEASURES_LP_SOLVER +The command to run the scheduling solver. + +Default: `"cbc"` + +### FLEXMEASURES_HOSTS_AND_AUTH_START +Configuration used for entity addressing. This contains the domain on which FlexMeasures runs +and the first month when the domain was under the current owner's administration. + +Default: `{"flexmeasures.io": "2021-01"}` + +### FLEXMEASURES_DB_BACKUP_PATH +Relative path to the folder where database backups are stored if that feature is being used. + +Default: `"migrations/dumps"` + +### FLEXMEASURES_PROFILE_REQUESTS +Whether to turn on a feature which times requests made through FlexMeasures. Interesting for developers. + +Default: `False` + + +## UI + +### FLEXMEASURES_PLATFORM_NAME +Name being used in headings + +Default: `"FlexMeasures"` + +### FLEXMEASURES_HIDE_NAN_IN_UI +Whether to hide the word "nan" if any value in metrics tables is `NaN`. + +Default: `False` + +### RQ_DASHBOARD_POLL_INTERVAL + +Interval in which viewing the queues dashboard refreshes itself, in miliseconds. + +Default: `3000` (3 seconds) + + +## Timing + +### FLEXMEASURES_TIMEZONE +Timezone in which the platform operates. This is useful when datetimes are being localized. + +Default: `"Asia/Seoul"` + +### FLEXMEASURES_PLANNING_TTL +Time to live for UDI event ids of successful scheduling jobs. Set a negative timedelta to persist forever. + +Default: `timedelta(days=7)` + +### FLEXMEASURES_PLANNING_HORIZON +The horizon to use when making schedules. + +Default: `timedelta(hours=2 * 24)` + +## Tokens + +### DARK_SKY_API_KEY +Token for accessing the DarkSky weather forecasting service. + +Note: DarkSky will be soon (Aug 1, 2021) become non-public, so thay are not giving out new tokens. We'll use another service soon, [see this issue](https://github.com/SeitaBV/flexmeasures/issues/3). + +This is unfortunate. In the meantime, if you can't find anybody lending their token, you can add weather forecasts to the FlexMeasures db yourself. + +Default: `None` + +### MAPBOX_ACCESS_TOKEN +Token for accessing the mapbox API (for displaying maps on the dashboard and asset pages). You can learn how to obtain one [here](https://docs.mapbox.com/help/glossary/access-token/) + +Default: `None` + + +### FLEXMEASURES_TASK_CHECK_AUTH_TOKEN +Token which external services can use to check on the status of recurring tasks within FlexMeasures. + +Default: `None` + + +## SQLAlchemy + +This is only a selection of the most important settings. +See [the Flask-SQLAlchemy Docs](https://flask-sqlalchemy.palletsprojects.com/en/master/config) for all possibilities. + +### SQLALCHEMY_DATABASE_URI (**) +Connection string to the postgres database, format: `postgresql://:@[:]/` + +Default: `None` + +### SQLALCHEMY_ENGINE_OPTIONS +Configuration of the SQLAlchemy engine. + +Default: +``` + { + "pool_recycle": 299, + "pool_pre_ping": True, + "connect_args": {"options": "-c timezone=utc"}, + } +``` + +## Security + +This is only a selection of the most important settings. +See [the Flask-Security Docs](https://flask-security-too.readthedocs.io/en/stable/configuration.html) as well as the [Flask-CORS docs](https://flask-cors.readthedocs.io/en/latest/configuration.html) for all possibilities. + +### SECRET_KEY (**) +Used to sign user sessions and also as extra salt (a.k.a. pepper) for password salting if `SECURITY_PASSWORD_SALT` is not set. +This is actually part of Flask - but is also used by Flask-Security to sign all tokens. + +It is critical this is set to a strong value. For python3 consider using: `secrets.token_urlsafe()` + +You can also set this in a file (which some Flask tutorials advised). Leave this setting set to `None` to get more instructions. + +Default: `None` + +### SECURITY_PASSWORD_SALT + +Extra password salt (a.k.a. pepper) + +Default: `None` (falls back to `SECRET_KEY`) + + +### SECURITY_TOKEN_AUTHENTICATION_HEADER +Name of the header which carries the auth bearer token in API requests. + +Default: `Authorization` + +### SECURITY_TOKEN_MAX_AGE +Maximal age of security tokens in seconds. + +Default: `60 * 60 * 6` (six hours) + +### SECURITY_TRACKABLE +Wether to track user statistics. Turning this on requires certain user fields. +We do not use this feature, but we do track number of logins. + +Default: `False` + +### CORS_ORIGINS +Allowed cross-origins. Set to "*" to allow all. For development (e.g. javascript on localhost) you might use "null" in this list. + +Default: `[]` + +### CORS_RESOURCES: +FlexMeasures resources which get cors protection. This can be a regex, a list of them or dict with all possible options. + +Default: `[r"/api/*"]` + +### CORS_SUPPORTS_CREDENTIALS +Allows users to make authenticated requests. If true, injects the Access-Control-Allow-Credentials header in responses. This allows cookies and credentials to be submitted across domains. + +Note: This option cannot be used in conjunction with a “*” origin + +Default: `True` + + +## Mail +For FlexMeasures to be able to send email to users (e.g. for resetting passwords), you need an email account which can do that (e.g. GMail). + +This is only a selection of the most important settings. +See [the Flask-Mail Docs](https://flask-mail.readthedocs.io/en/latest/#configuring-flask-mail) for others. + +### MAIL_SERVER (*) + +Email name server domain. + +Default: `"localhost"` + +### MAIL_PORT (*) +SMTP port of the mail server. + +Default: `25` + +### MAIL_USE_TLS +Whether to use TLS. + +Default: `False` + +### MAIL_USE_SSL +Whether to use SSL. + +Default: `False` + +### MAIL_USERNAME (*) + +Login name of the mail system user. + +Default: `None` + +### MAIL_DEFAULT_SENDER (*) +Tuple of shown name of sender and their email address. + +Default: +``` +( + "FlexMeasures", + "no-reply@example.com", +) +``` + +## MAIL_PASSWORD: +Password of mail system user. + +Default: `None` + + +## Redis +FlexMeasures uses the Redis database to support our forecasting and scheduling job queues. + +### FLEXMEASURES_REDIS_URL (*) +URL of redis server. + +Default: `"localhost"` + +### FLEXMEASURES_REDIS_PORT (*) +Port of redis server. + +Default: `6379` + +### FLEXMEASURES_REDIS_DB_NR (*) +Number of the redis database to use (Redis per default has 16 databases, nubered 0-15) + +Default: `0` + +### FLEXMEASURES_REDIS_PASSWORD (*) +Password of the redis server. + +Default: `None` + + +## Demonstrations + +### FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS = False +When `FLEXMEASURES_MODE=demo`, this can hold login credentials (demo user email and password, e.g. `("demo at seita.nl", "flexdemo")`), so anyone can log in and try out the platform. + +Default: `None` + +### FLEXMEASURES_DEMO_YEAR = 2015 +When `FLEXMEASURES_MODE=demo`, this setting can be used to make the FlexMeasures platform select data from a specific year (e.g. 2015), +so that old imported data can be demoed as if it were current + +Default: `None` + +### FLEXMEASURES_SHOW_CONTROL_UI +The control page is still mocked, so this setting controls if it is to be shown. + +Default: `False` diff --git a/Developing.md b/Developing.md index 5a3f165c0..3f834a509 100644 --- a/Developing.md +++ b/Developing.md @@ -21,11 +21,28 @@ Install all dependencies including the ones needed for development: make install-for-dev +## Configuration + +Follow the confguration Quickstart advice in `Installation.md`. + + +## Loading data + +If you have a SQL Dump file, you can load that: + + psql -U {user_name} -h {host_name} -d {database_name} -f {file_path} + + + ## Run locally Now, to start the web application, you can run: - python flexmeasures/run-local.py + flexmeasures run + +Or: + + python run-local.py And access the server at http://localhost:5000 @@ -47,3 +64,16 @@ You can add --cov-report=html after which a htmlcov/index.html is generated. It's also possible to use: python setup.py test + + +## Versioning + +We use [setuptool_scm](https://github.com/pypa/setuptools_scm/) for versioning, which bases the FlexMeasures version on the latest git tag and the commits since then. + +So as a developer, it's crucial to use git tags for versions only. + +We use semantic versioning, and we always include the patch version, not only max and min, so that setuptools_scm makes the correct guess about the next minor version. Thus, we should use `2.0.0` instead of `2.0`. + +See `to_pypi.sh` for more commentary on the development versions. + +Our API has its own version, which moves much slower. This is important to explicitly support outside apps who were coded against older versions. \ No newline at end of file diff --git a/Installation.md b/Installation.md index 89a6e8ad8..bb672e22f 100644 --- a/Installation.md +++ b/Installation.md @@ -1,44 +1,131 @@ -# Building & Running FlexMeasures +# Installing & Running FlexMeasures -## Dependencies +## Quickstart + +This section walks you through getting FlexMeasures to run with the least effort. We'll cover making a secret key, connecting a database and creating one user & one asset. + +### Install FlexMeasures Install dependencies and the `flexmeasures` platform itself: - make install + pip install flexmeasures + + +### Make a secret key for sessions and password salts + +Set a secret key which is used to sign user sessions and re-salt their passwords. The quickest way is with an environment variable, like this: + + `export SECRET_KEY=something-secret` + +(on Windows, use `set` instead of `export`) + +This suffices for a quick start. + +If you want to consistently use FlexMeasures, we recommend you add this setting to your config file at `~/.flexmeasures.cfg` and use a truly random string. Here is a Pythonic way to generate a good secret key: + + `python -c "import secrets; print(secrets.token_urlsafe())"` + + +### Configure environment + +Set an environment variable to indicate in which environment you are operating (one out of development|testing|staging|production). We'll go with `development` here: + + `export FLASK_ENV=development` + +(on Windows, use `set` instead of `export`) + +or: + + `echo "FLASK_ENV=development" >> .env` + +Note: The default is `production`, which will not work well on localhost due to SSL issues. + + + +### Preparing the time series database + +* Make sure you have a Postgres (Version 9+) database for FlexMeasures to use. See `data/Readme.md` (section "Getting ready to use") for instructions on this. +* Tell `flexmeasures` about it: + + `export SQLALCHEMY_DATABASE_URI="postgresql://:@[:]/"` + + If you install this on localhost, `host-address` is `127.0.0.1` and the port can be left out. + (on Windows, use `set` instead of `export`) +* Create the Postgres DB structure for FlexMeasures: + + `flexmeasures db upgrade` + +This suffices for a quick start. -## Configure environment +Note that for a more permanent configuration, you can create your FlexMeasures configuration file at `~/.flexmeasures.cfg` and add this: -* Set an env variable to indicate in which environment you are operating (one out of development|testing|staging|production), e.g.: + `SQLALCHEMY_DATABASE_URI="postgresql://:@[:]/"` - `echo "FLASK_ENV=development" >> .env` - `export FLASK_ENV=production` -* If you need to customise settings, create `flexmeasures/_config.py` and add required settings. - If you're unsure what you need, just continue for now and the app will tell you what it misses. +### Add a user -## Make a secret key for sessions +FlexMeasures is a web-based platform, so we need a user account: - mkdir -p /path/to/flexmeasures/instance - head -c 24 /dev/urandom > /path/to/flexmeasures/instance/secret_key +`flexmeasures new-user --username --email --roles=admin` -## Preparing the time series database +* This will ask you to set a password for the user. +* Giving the first user the `admin` role is probably what you want. -* Make sure you have a Postgres (Version 9+) database. See `data/Readme.md` for instructions on this. -* Tell `flexmeasures` about it. Either you are using the default for the environment you're in (see `flexmeasures/utils/config_defaults`), - or you can configure your own connection string: In `flexmeasures/_conf.py`, - set the variable `SQLALCHEMY_DATABASE_URI = 'postgresql://:@[:]/'` -* Run `flask db upgrade` to create the Postgres DB structure. -## Preparing the job queue database +### Add structure -To let FlexMeasures queue forecasting and scheduling jobs, install a Redis server and configure access to it within FlexMeasures' config file (see above). You can find the default settings in `flexmeasures/utils/config_defaults.py`. +Populate the database with some standard energy asset types: -TODO: more detail + `flexmeasures db-populate --structure` -## Install an LP solver -For planning balancing actions, the flexmeasures platform uses a linear program solver. Currently that is the Cbc solver. See the `FLEXMEASURES_LP_SOLVER` config setting if you want to change to a different solver. +### Run FlexMeasures + +It's finally time to start running FlexMeasures: + +`flexmeasures run` + +(This might print some warnings, see the next section where we go into more detail) + +Note: In a production context, you shouldn't run a script - hand the `app` object to a WSGI process, as your platform of choice describes. +Often, that requires a WSGI script. We provide an example WSGI script in [the CI Readme](ci/Readme.md). + +You can visit `http://localhost:5000` now to see if the app's UI works. +When you see the dashboard, the map will not work. For that, you'll need to get your MAPBOX_ACCESS_TOKEN and add it to your config file (see Configuration.md for details). + + +### Add your first asset + +Head over to `http://localhost:5000/assets` and add a new asset there. + +TODO: [issue 57](https://github.com/SeitaBV/flexmeasures/issues/57) should create a CLI function for this. + +Note: You can also use the [`POST /api/v2_0/assets`](https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html#post--api-v2_0-assets) endpoint in the FlexMeasures API to create an asset. + +### Add data + +You can use the [`POST /api/v2_0/postMeterData`](https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html#post--api-v2_0-postMeterData) endpoint in the FlexMeasures API to send meter data. + +TODO: [issue 56](https://github.com/SeitaBV/flexmeasures/issues/56) should create a CLI function for adding a lot of data at once, from a CSV dataset. + +Also, you can add forecasts for your meter data with the `db_populate` command, here is an example: + + `flexmeasures db-populate --forecasts --from-date 2020-03-08 --to-date 2020-04-08 --asset-type Asset --asset my-solar-panel ` + +Note: You can also use the API to send forecast data. + + +## Other settings, for full functionality + +### Set mail settings + +For FlexMeasures to be able to send email to users (e.g. for resetting passwords), you need an email account which can do that (e.g. GMail). Set the MAIL_* settings in your configuration, see [the Configuration documentation](Configuration.md). + + +### Install an LP solver + +For planning balancing actions, the FlexMeasures platform uses a linear program solver. Currently that is the Cbc solver. See the `FLEXMEASURES_LP_SOLVER` config setting if you want to change to a different solver. Installing Cbc can be done on Unix via: @@ -48,29 +135,31 @@ Installing Cbc can be done on Unix via: We provide a script for installing from source (without requiring `sudo` rights) in [the CI Readme](ci/Readme.md). -More information (e.g. for installing on Windows) on [the website](https://projects.coin-or.org/Cbc). +More information (e.g. for installing on Windows) on [the Cbc website](https://projects.coin-or.org/Cbc). -## Run +### Start collecting weather data -Now, to start the web application, you can run: +To collect weather measurements and forecasts, there is a task you could run periodically, probably once per hour. Here is an example: - python flexmeasures/run-local.py + flexmeasures collect-weather-data--location 33.4366,126.5269 --store-in-db -But in a production context, you shouldn't run a script - hand the `app` object to a WSGI process, as your platform of choice describes. -Often, that requires a WSGI script. We provide an example WSGI script in [the CI Readme](ci/Readme.md). +### Preparing the job queue database and start workers +To let FlexMeasures queue forecasting and scheduling jobs, install a [Redis](https://redis.io/) server and configure access to it within FlexMeasures' config file (see above). You can find the necessary settings in [the Configuration documentation](Configuration.md). -## Loading data +Then run one worker for each kind of job (in a separate terminal): -If you have a SQL Dump file, you can load that: + flexmeasures run-worker --queue forecasting + flexmeasures run-worker --queue scheduling - psql -U {user_name} -h {host_name} -d {database_name} -f {file_path} +You can also clear the job queues: -Else, you can populate some standard data, most of which comes from files: + flexmeasures clear-queue --queue forecasting + flexmeasures clear-queue --queue scheduling -* Finally, run `flask db_populate --structure --data --small` to load this data into the database. - The `--small` parameter will only load four assets and four days, so use this first to try things out. TODO: check which command is possible at the moment. Also add a TODO saying where we want to go with this (support for loading data). +When the main FlexMeasures process runs (e.g. by `flexmeasures run`), the queues of forecasting and scheduling jobs can be visited at `http://localhost:5000/tasks/forecasting` and `http://localhost:5000/tasks/schedules`, respectively (by admins). +When forecasts and schedules have been generated, they should be visible at `http://localhost:5000/analytics`. You can also access forecasts via the FlexMeasures API at [GET /api/v2_0/getPrognosis](https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html#get--api-v2_0-getPrognosis), and schedules via [GET /api/v2_0/getDeviceMessage](https://flexmeasures.readthedocs.io/en/latest/api/v2_0.html#get--api-v2_0-getDeviceMessage). diff --git a/Makefile b/Makefile index 05faf545c..5b0d11aae 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # ---- Development --- run-local: - python flexmeasures/run-local.py + python run-local.py test: make install-for-dev diff --git a/NOTICE b/NOTICE index 05899a61c..dbaf65b0c 100644 --- a/NOTICE +++ b/NOTICE @@ -1,7 +1,3 @@ FlexMeasures -Copyright 2018 Seita Energy Flexibility +Copyright 2018-2021 Seita Energy Flexibility Apache 2.0 license - -Sphinx RTD Theme (see documentation/_themes/sphinx_rtd_flexmeasures_theme) -Copyright 2010-2021 Dave Snider, Read the Docs, Inc., FontAwesome, Łukasz Dziedzic, The Modernizr Team, Google Inc., Seita Energy Flexibility -MIT license, BSD license, SIL Open Font license 1.1, Apache2.0 license \ No newline at end of file diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 0a833676e..626c5dedb 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -6,15 +6,22 @@ FlexMeasures Changelog v0.2.4 | March XX, 2021 =========================== +New features +----------- +* FlexMeasures can be installed with `pip` and its CLI commands can be run with `flexmeasures` [see `PR #54 `_] + Bugfixes -------- * Show screenshots in documentation and add some missing content [see `PR #60 `_] * Documentation listed 2.0 API endpoints twice [see `PR #59 `_] * User page did not list number of assets correctly [see `PR #64 `_] -Infrastructure/Support +Infrastructure / Support ---------------------- * Dump and restore postgres database as CLI commands [see `PR #68 `_] +* Improved installation tutorial as part of [`PR #54 `_] + + v0.2.3 | February 27, 2021 =========================== @@ -34,7 +41,7 @@ Bugfixes * Password reset link on account page was broken [see `PR #23 `_] -Infrastructure/Support +Infrastructure / Support ---------------------- * CI via Github Actions [see `PR #1 `_] * Integration with `timely beliefs `__ lib: Sensors [see `PR #13 `_] diff --git a/documentation/conf.py b/documentation/conf.py index af3ebad60..4a3c812a9 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -6,9 +6,8 @@ # full list see the documentation: # http://www.sphinx-doc.org/en/stable/config -import os -import sys from datetime import datetime +from pkg_resources import get_distribution # If extensions (or modules to document with autodoc) are in another directory, @@ -16,10 +15,6 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # -# Insert FlexMeasures' path into the system. -sys.path.insert(0, os.path.abspath("..")) -from flexmeasures import __version__ # noqa: E402 - # -- Project information ----------------------------------------------------- @@ -27,10 +22,10 @@ copyright = f"{datetime.now().year}, Seita Energy Flexibility, developed in partnership with A1 Engineering, South Korea" author = "Seita B.V." -# The short X.Y version -version = __version__ # The full version, including alpha/beta/rc tags -release = __version__ +release = get_distribution("flexmeasures").version +# The short X.Y.Z version +version = ".".join(release.split(".")[:3]) rst_prolog = """ """ diff --git a/flexmeasures/__init__.py b/flexmeasures/__init__.py index f9aa3e110..52dea79f6 100644 --- a/flexmeasures/__init__.py +++ b/flexmeasures/__init__.py @@ -1 +1,12 @@ -__version__ = "0.3.2" +from importlib_metadata import version, PackageNotFoundError + + +__version__ = "Unknown" + +# This uses importlib.metadata behaviour added in Python 3.8 +# and relies on setuptools_scm. +try: + __version__ = version("flexmeasures") +except PackageNotFoundError: + # package is not installed + pass diff --git a/flexmeasures/api/__init__.py b/flexmeasures/api/__init__.py index 6aec891b5..ae38b31ce 100644 --- a/flexmeasures/api/__init__.py +++ b/flexmeasures/api/__init__.py @@ -4,6 +4,7 @@ from flask_json import as_json from flask_login import current_user +from flexmeasures import __version__ as flexmeasures_version from flexmeasures.data.models.user import User from flexmeasures.api.common.utils.args_parsing import ( FMValidationError, @@ -75,6 +76,7 @@ def get_versions() -> dict: "/api/v1/getService and /api/v1_1/getService. An authentication token can be requested at: " "/api/requestAuthToken", "versions": ["v1", "v1_1", "v1_2", "v1_3", "v2_0"], + "flexmeasures_version": flexmeasures_version, } return response diff --git a/flexmeasures/api/common/schemas/__init__.py b/flexmeasures/api/common/schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 86b38ed0f..e152b0517 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -1,6 +1,8 @@ # flake8: noqa: E402 import os import time +from typing import Optional + from flask import Flask, g, request from flask.cli import load_dotenv from flask_mail import Mail @@ -12,15 +14,19 @@ from rq import Queue -def create(env=None) -> Flask: +def create(env: Optional[str] = None, path_to_config: Optional[str] = None) -> Flask: """ Create a Flask app and configure it. + Set the environment by setting FLASK_ENV as environment variable (also possible in .env). Or, overwrite any FLASK_ENV setting by passing an env in directly (useful for testing for instance). + + A path to a config file can be passed in (otherwise a config file will be searched in the home or instance directories) """ + from flexmeasures.utils import config_defaults from flexmeasures.utils.config_utils import read_config, configure_logging - from flexmeasures.utils.app_utils import install_secret_key + from flexmeasures.utils.app_utils import set_secret_key from flexmeasures.utils.error_utils import add_basic_error_handlers # Create app @@ -34,12 +40,12 @@ def create(env=None) -> Flask: app.env = env if env == "testing": app.testing = True + if env == "development": + app.debug = config_defaults.DevelopmentConfig.DEBUG # App configuration - read_config(app) - if app.debug and not app.testing and not app.cli: - print(app.config) + read_config(app, path_to_config=path_to_config) add_basic_error_handlers(app) app.mail = Mail(app) @@ -74,10 +80,13 @@ def create(env=None) -> Flask: # Some basic security measures if not app.env == "documentation": - install_secret_key(app) + set_secret_key(app) + if app.config.get("SECURITY_PASSWORD_SALT", None) is None: + app.config["SECURITY_PASSWORD_SALT"] = app.config["SECRET_KEY"] + if not app.env in ("documentation", "development"): SSLify(app) - # Register database and models, including user auth security measures + # Register database and models, including user auth security handlers from flexmeasures.data import register_at as register_db_at diff --git a/flexmeasures/data/Readme.md b/flexmeasures/data/Readme.md index 496c46259..36195bebc 100644 --- a/flexmeasures/data/Readme.md +++ b/flexmeasures/data/Readme.md @@ -58,7 +58,7 @@ This may in fact not be needed: From the terminal: Open a console (use your Windows key and type `cmd`). -Proceed to create a database as the postgres superuser (using your postgres user password):: +Proceed to create a database as the postgres superuser (using your postgres user password): sudo -i -u postgres createdb -U postgres flexmeasures @@ -97,7 +97,7 @@ Write: SQLALCHEMY_DATABASE_URI = "postgresql://flexmeasures:@127.0.0.1/flexmeasures" -into the config file you are using, e.g. flexmeasures/development_config.py +into the config file you are using, e.g. ~/flexmeasures.cfg ## Get structure (and some data) into place @@ -113,7 +113,7 @@ On the to-be-exported database: Note that we only dump the data here. Locally, we create a fresh database with the structure being based on the data model given by the local codebase: - flask db-reset + flexmeasures db-reset Then we import the data dump we made earlier: @@ -126,19 +126,19 @@ Note: To make sure passwords will be decrypted correctly when you authenticate, ### Create data manually -First of all, you can get the database structure with +First, you can get the database structure with: - flask db upgrade + flexmeasures db upgrade Note: If you develop code (and might want to make changes to the data model), you should also check out the maintenance section about database migrations. You can create users with the `new-user` command. Check it out: - flask new-user --help + flexmeasures new-user --help You can create some pre-determined asset types and data sources with this command: - flask db-populate --structure + flexmeasures db-populate --structure TODO: We should instead offer CLI commands to be able to create asset types as needed. @@ -148,13 +148,13 @@ TODO: We still need a decent way to load in metering data, e.g. from CSV - often You can create forecasts for your existing metered data with this command: - flask db-populate --forecasts + flexmeasures db-populate --forecasts Check out it's `--help` content to learn more. You can set which assets and which time window you want to forecast. At the time of writing, the forecasts horizons are fixed to 1, 6, 24 and 48 hours. Of course, making forecasts takes a while for a larger dataset. Just to note: There is also a command to get rid of data: - flask db_depopulate --structure --data --forecasts + flexmeasures db-depopulate --structure --data --forecasts ## Visualize the data model @@ -175,15 +175,15 @@ e.g. dev, staging and production can be kept in sync. Run these commands from the repository root directory (read below comments first): - flask db init - flask db migrate - flask db upgrade + flexmeasures db init + flexmeasures db migrate + flexmeasures db upgrade -The first command (`flask db init`) is only needed here once, it initialises the alembic migration tool. +The first command (`flexmeasures db init`) is only needed here once, it initialises the alembic migration tool. The second command generates the SQL for your current db model and the third actually gives you the db structure. With every migration, you get a new migration step in `migrations/versions`. Be sure to add that to `git`, -as future calls to `flask db upgrade` will need those steps, and they might happen on another computer. +as future calls to `flexmeasures db upgrade` will need those steps, and they might happen on another computer. Hint: You can edit these migrations steps, if you want. @@ -191,14 +191,14 @@ Hint: You can edit these migrations steps, if you want. Just to be clear that the `db init` command is needed only at the beginning - you usually do, if your model changed: - flask db migrate --message "Please explain what you did, it helps for later" - flask db upgrade + flexmeasures db migrate --message "Please explain what you did, it helps for later" + flexmeasures db upgrade ## Get database structure updated The goal is that on any other computer, you can always execute - flask db upgrade + flexmeasures db upgrade to have the database structure up-to-date with all migrations. @@ -206,13 +206,13 @@ to have the database structure up-to-date with all migrations. The history of migrations is at your fingertips: - flask db current - flask db history + flexmeasures db current + flexmeasures db history You can move back and forth through the history: - flask db downgrade - flask db upgrade + flexmeasures db downgrade + flexmeasures db upgrade Both of these accept a specific revision id parameter, as well. @@ -245,7 +245,7 @@ It relies on a Redis server, which is has to be installed locally, or used on a Forecasting jobs are usually created (and enqueued) when new data comes in via the API. To asynchronously work on these forecasting jobs, run this in a console: - flask run_forecasting_worker + flexmeasures run_worker --queue forecasting You should be able to run multiple workers in parallel, if necessary. You can add the `--name` argument to keep them a bit more organized. diff --git a/flexmeasures/data/__init__.py b/flexmeasures/data/__init__.py index 7994e44ca..f42f4554b 100644 --- a/flexmeasures/data/__init__.py +++ b/flexmeasures/data/__init__.py @@ -1,3 +1,5 @@ +import os + from flask import Flask from flask_migrate import Migrate @@ -9,7 +11,7 @@ def register_at(app: Flask): # First configure the central db object and Alembic's migration tool configure_db_for(app) - Migrate(app, db) + Migrate(app, db, directory=os.path.join(app.root_path, "data", "migrations")) configure_auth(app, db) diff --git a/migrations/README b/flexmeasures/data/migrations/README similarity index 100% rename from migrations/README rename to flexmeasures/data/migrations/README diff --git a/migrations/alembic.ini b/flexmeasures/data/migrations/alembic.ini similarity index 100% rename from migrations/alembic.ini rename to flexmeasures/data/migrations/alembic.ini diff --git a/migrations/env.py b/flexmeasures/data/migrations/env.py similarity index 100% rename from migrations/env.py rename to flexmeasures/data/migrations/env.py diff --git a/migrations/script.py.mako b/flexmeasures/data/migrations/script.py.mako similarity index 100% rename from migrations/script.py.mako rename to flexmeasures/data/migrations/script.py.mako diff --git a/migrations/versions/01fe99da5716_initial.py b/flexmeasures/data/migrations/versions/01fe99da5716_initial.py similarity index 100% rename from migrations/versions/01fe99da5716_initial.py rename to flexmeasures/data/migrations/versions/01fe99da5716_initial.py diff --git a/migrations/versions/02ddbbff29a7_naming_conventions.py b/flexmeasures/data/migrations/versions/02ddbbff29a7_naming_conventions.py similarity index 100% rename from migrations/versions/02ddbbff29a7_naming_conventions.py rename to flexmeasures/data/migrations/versions/02ddbbff29a7_naming_conventions.py diff --git a/migrations/versions/11b735abebe7_create_power_table_and_drop_measurement_table.py b/flexmeasures/data/migrations/versions/11b735abebe7_create_power_table_and_drop_measurement_table.py similarity index 100% rename from migrations/versions/11b735abebe7_create_power_table_and_drop_measurement_table.py rename to flexmeasures/data/migrations/versions/11b735abebe7_create_power_table_and_drop_measurement_table.py diff --git a/migrations/versions/1a4f0e5c4b86_unique_userids_in_ds.py b/flexmeasures/data/migrations/versions/1a4f0e5c4b86_unique_userids_in_ds.py similarity index 100% rename from migrations/versions/1a4f0e5c4b86_unique_userids_in_ds.py rename to flexmeasures/data/migrations/versions/1a4f0e5c4b86_unique_userids_in_ds.py diff --git a/migrations/versions/1b64acf01809_forecasting_job_table.py b/flexmeasures/data/migrations/versions/1b64acf01809_forecasting_job_table.py similarity index 100% rename from migrations/versions/1b64acf01809_forecasting_job_table.py rename to flexmeasures/data/migrations/versions/1b64acf01809_forecasting_job_table.py diff --git a/migrations/versions/1bcccdf0c3e1_unique_usernames.py b/flexmeasures/data/migrations/versions/1bcccdf0c3e1_unique_usernames.py similarity index 100% rename from migrations/versions/1bcccdf0c3e1_unique_usernames.py rename to flexmeasures/data/migrations/versions/1bcccdf0c3e1_unique_usernames.py diff --git a/migrations/versions/1e8d27922f56_create_price_table.py b/flexmeasures/data/migrations/versions/1e8d27922f56_create_price_table.py similarity index 100% rename from migrations/versions/1e8d27922f56_create_price_table.py rename to flexmeasures/data/migrations/versions/1e8d27922f56_create_price_table.py diff --git a/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py b/flexmeasures/data/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py similarity index 100% rename from migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py rename to flexmeasures/data/migrations/versions/22ce09690d23_mix_in_timely_beliefs_sensor_with_asset_market_and_weather_sensor.py diff --git a/migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py b/flexmeasures/data/migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py similarity index 72% rename from migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py rename to flexmeasures/data/migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py index ecad0ab1a..ba6d130b2 100644 --- a/migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py +++ b/flexmeasures/data/migrations/versions/26373d8266db_owner_deletion_deletes_assets_and_power.py @@ -17,11 +17,11 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint("asset_owner_id_fkey", "asset", type_="foreignkey") + op.drop_constraint("asset_owner_id_bvp_users_fkey", "asset", type_="foreignkey") op.create_foreign_key( None, "asset", "bvp_users", ["owner_id"], ["id"], ondelete="CASCADE" ) - op.drop_constraint("power_asset_id_fkey", "power", type_="foreignkey") + op.drop_constraint("power_asset_id_asset_fkey", "power", type_="foreignkey") op.create_foreign_key( None, "power", "asset", ["asset_id"], ["id"], ondelete="CASCADE" ) @@ -31,9 +31,11 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_constraint(None, "power", type_="foreignkey") - op.create_foreign_key("power_asset_id_fkey", "power", "asset", ["asset_id"], ["id"]) + op.create_foreign_key( + "power_asset_id_asset_fkey", "power", "asset", ["asset_id"], ["id"] + ) op.drop_constraint(None, "asset", type_="foreignkey") op.create_foreign_key( - "asset_owner_id_fkey", "asset", "bvp_users", ["owner_id"], ["id"] + "asset_owner_id_bvp_users_fkey", "asset", "bvp_users", ["owner_id"], ["id"] ) # ### end Alembic commands ### diff --git a/migrations/versions/2c9a32614784_rename_data_source_columns_in_power_price_and_weather_tables.py b/flexmeasures/data/migrations/versions/2c9a32614784_rename_data_source_columns_in_power_price_and_weather_tables.py similarity index 100% rename from migrations/versions/2c9a32614784_rename_data_source_columns_in_power_price_and_weather_tables.py rename to flexmeasures/data/migrations/versions/2c9a32614784_rename_data_source_columns_in_power_price_and_weather_tables.py diff --git a/migrations/versions/31f251554682_merge.py b/flexmeasures/data/migrations/versions/31f251554682_merge.py similarity index 100% rename from migrations/versions/31f251554682_merge.py rename to flexmeasures/data/migrations/versions/31f251554682_merge.py diff --git a/migrations/versions/3d56402cde15_drop_login_columns_in_bvp_users_table.py b/flexmeasures/data/migrations/versions/3d56402cde15_drop_login_columns_in_bvp_users_table.py similarity index 100% rename from migrations/versions/3d56402cde15_drop_login_columns_in_bvp_users_table.py rename to flexmeasures/data/migrations/versions/3d56402cde15_drop_login_columns_in_bvp_users_table.py diff --git a/migrations/versions/3db3e71d101d_make_datasource_a_subclass_of_timely_beliefs_beliefsource.py b/flexmeasures/data/migrations/versions/3db3e71d101d_make_datasource_a_subclass_of_timely_beliefs_beliefsource.py similarity index 100% rename from migrations/versions/3db3e71d101d_make_datasource_a_subclass_of_timely_beliefs_beliefsource.py rename to flexmeasures/data/migrations/versions/3db3e71d101d_make_datasource_a_subclass_of_timely_beliefs_beliefsource.py diff --git a/migrations/versions/3e43d3274d16_Asset_soc_udi_event_id.py b/flexmeasures/data/migrations/versions/3e43d3274d16_Asset_soc_udi_event_id.py similarity index 100% rename from migrations/versions/3e43d3274d16_Asset_soc_udi_event_id.py rename to flexmeasures/data/migrations/versions/3e43d3274d16_Asset_soc_udi_event_id.py diff --git a/migrations/versions/45d937300b0f_create_measurement_table.py b/flexmeasures/data/migrations/versions/45d937300b0f_create_measurement_table.py similarity index 100% rename from migrations/versions/45d937300b0f_create_measurement_table.py rename to flexmeasures/data/migrations/versions/45d937300b0f_create_measurement_table.py diff --git a/migrations/versions/4b6cebbdf473_create_weather_sensor_type_and_weather_sensor_and_weather_tables.py b/flexmeasures/data/migrations/versions/4b6cebbdf473_create_weather_sensor_type_and_weather_sensor_and_weather_tables.py similarity index 100% rename from migrations/versions/4b6cebbdf473_create_weather_sensor_type_and_weather_sensor_and_weather_tables.py rename to flexmeasures/data/migrations/versions/4b6cebbdf473_create_weather_sensor_type_and_weather_sensor_and_weather_tables.py diff --git a/migrations/versions/50cf294e007d_complete_adding_units_to_all_generic_asset_tables_and_display_names_to_all_generic_asset_tables_and_generic_asset_type_tables.py b/flexmeasures/data/migrations/versions/50cf294e007d_complete_adding_units_to_all_generic_asset_tables_and_display_names_to_all_generic_asset_tables_and_generic_asset_type_tables.py similarity index 100% rename from migrations/versions/50cf294e007d_complete_adding_units_to_all_generic_asset_tables_and_display_names_to_all_generic_asset_tables_and_generic_asset_type_tables.py rename to flexmeasures/data/migrations/versions/50cf294e007d_complete_adding_units_to_all_generic_asset_tables_and_display_names_to_all_generic_asset_tables_and_generic_asset_type_tables.py diff --git a/migrations/versions/550a9020f1bf_default_resolution_for_existing_sensors.py b/flexmeasures/data/migrations/versions/550a9020f1bf_default_resolution_for_existing_sensors.py similarity index 100% rename from migrations/versions/550a9020f1bf_default_resolution_for_existing_sensors.py rename to flexmeasures/data/migrations/versions/550a9020f1bf_default_resolution_for_existing_sensors.py diff --git a/migrations/versions/564e8df4e3a9_stop_using_bvp_in_table_names.py b/flexmeasures/data/migrations/versions/564e8df4e3a9_stop_using_bvp_in_table_names.py similarity index 100% rename from migrations/versions/564e8df4e3a9_stop_using_bvp_in_table_names.py rename to flexmeasures/data/migrations/versions/564e8df4e3a9_stop_using_bvp_in_table_names.py diff --git a/migrations/versions/5d39829d91af_create_data_sources_table.py b/flexmeasures/data/migrations/versions/5d39829d91af_create_data_sources_table.py similarity index 100% rename from migrations/versions/5d39829d91af_create_data_sources_table.py rename to flexmeasures/data/migrations/versions/5d39829d91af_create_data_sources_table.py diff --git a/migrations/versions/61bfc6e45c4d_add_soc_columns_in_asset_table.py b/flexmeasures/data/migrations/versions/61bfc6e45c4d_add_soc_columns_in_asset_table.py similarity index 100% rename from migrations/versions/61bfc6e45c4d_add_soc_columns_in_asset_table.py rename to flexmeasures/data/migrations/versions/61bfc6e45c4d_add_soc_columns_in_asset_table.py diff --git a/migrations/versions/7113b0f00678_drop_forecasting_jobs_table_and_make_display_names_nullable.py b/flexmeasures/data/migrations/versions/7113b0f00678_drop_forecasting_jobs_table_and_make_display_names_nullable.py similarity index 100% rename from migrations/versions/7113b0f00678_drop_forecasting_jobs_table_and_make_display_names_nullable.py rename to flexmeasures/data/migrations/versions/7113b0f00678_drop_forecasting_jobs_table_and_make_display_names_nullable.py diff --git a/migrations/versions/7987667dbd43_add_asset_type_hover_label.py b/flexmeasures/data/migrations/versions/7987667dbd43_add_asset_type_hover_label.py similarity index 92% rename from migrations/versions/7987667dbd43_add_asset_type_hover_label.py rename to flexmeasures/data/migrations/versions/7987667dbd43_add_asset_type_hover_label.py index 01a7faca5..a4a48eb26 100644 --- a/migrations/versions/7987667dbd43_add_asset_type_hover_label.py +++ b/flexmeasures/data/migrations/versions/7987667dbd43_add_asset_type_hover_label.py @@ -23,7 +23,7 @@ def upgrade(): ) # ### end Alembic commands ### op.drop_constraint( - constraint_name="asset_asset_type_name_fkey", + constraint_name="asset_asset_type_name_asset_type_fkey", table_name="asset", type_="foreignkey", ) @@ -46,7 +46,7 @@ def upgrade(): "UPDATE asset SET asset_type_name = 'two-way_evse' where asset_type_name = 'bidirectional_charging_station'" ) op.create_foreign_key( - constraint_name="asset_asset_type_name_fkey", + constraint_name="asset_asset_type_name_asset_type_fkey", source_table="asset", referent_table="asset_type", local_cols=["asset_type_name"], @@ -65,7 +65,7 @@ def downgrade(): op.drop_column("asset_type", "hover_label") # ### end Alembic commands ### op.drop_constraint( - constraint_name="asset_asset_type_name_fkey", + constraint_name="asset_asset_type_name_asset_type_fkey", table_name="asset", type_="foreignkey", ) @@ -88,7 +88,7 @@ def downgrade(): "UPDATE asset SET asset_type_name = 'bidirectional_charging_station' where asset_type_name = 'two-way_evse'" ) op.create_foreign_key( - constraint_name="asset_asset_type_name_fkey", + constraint_name="asset_asset_type_name_asset_type_fkey", source_table="asset", referent_table="asset_type", local_cols=["asset_type_name"], diff --git a/migrations/versions/8abe32ffa204_add_owner_id_column_in_asset_table.py b/flexmeasures/data/migrations/versions/8abe32ffa204_add_owner_id_column_in_asset_table.py similarity index 100% rename from migrations/versions/8abe32ffa204_add_owner_id_column_in_asset_table.py rename to flexmeasures/data/migrations/versions/8abe32ffa204_add_owner_id_column_in_asset_table.py diff --git a/migrations/versions/8fcc5fec67bc_make_data_source_id_columns_primary_keys.py b/flexmeasures/data/migrations/versions/8fcc5fec67bc_make_data_source_id_columns_primary_keys.py similarity index 100% rename from migrations/versions/8fcc5fec67bc_make_data_source_id_columns_primary_keys.py rename to flexmeasures/data/migrations/versions/8fcc5fec67bc_make_data_source_id_columns_primary_keys.py diff --git a/migrations/versions/919dc9f1dc1f_merge.py b/flexmeasures/data/migrations/versions/919dc9f1dc1f_merge.py similarity index 100% rename from migrations/versions/919dc9f1dc1f_merge.py rename to flexmeasures/data/migrations/versions/919dc9f1dc1f_merge.py diff --git a/migrations/versions/91a938bfa5a8_add_horizon_columns_to_power_price_and_weather_tables.py b/flexmeasures/data/migrations/versions/91a938bfa5a8_add_horizon_columns_to_power_price_and_weather_tables.py similarity index 100% rename from migrations/versions/91a938bfa5a8_add_horizon_columns_to_power_price_and_weather_tables.py rename to flexmeasures/data/migrations/versions/91a938bfa5a8_add_horizon_columns_to_power_price_and_weather_tables.py diff --git a/migrations/versions/9254559dcac2_create_market_type_and_market_tables.py b/flexmeasures/data/migrations/versions/9254559dcac2_create_market_type_and_market_tables.py similarity index 100% rename from migrations/versions/9254559dcac2_create_market_type_and_market_tables.py rename to flexmeasures/data/migrations/versions/9254559dcac2_create_market_type_and_market_tables.py diff --git a/migrations/versions/9c7fc8e46f1e_add_location_columns_to_weather_sensor_table.py b/flexmeasures/data/migrations/versions/9c7fc8e46f1e_add_location_columns_to_weather_sensor_table.py similarity index 100% rename from migrations/versions/9c7fc8e46f1e_add_location_columns_to_weather_sensor_table.py rename to flexmeasures/data/migrations/versions/9c7fc8e46f1e_add_location_columns_to_weather_sensor_table.py diff --git a/migrations/versions/a328412b4623_add_timezone_column_in_bvp_users_table.py b/flexmeasures/data/migrations/versions/a328412b4623_add_timezone_column_in_bvp_users_table.py similarity index 100% rename from migrations/versions/a328412b4623_add_timezone_column_in_bvp_users_table.py rename to flexmeasures/data/migrations/versions/a328412b4623_add_timezone_column_in_bvp_users_table.py diff --git a/migrations/versions/a5b970eadb3b_time_series_indexes.py b/flexmeasures/data/migrations/versions/a5b970eadb3b_time_series_indexes.py similarity index 100% rename from migrations/versions/a5b970eadb3b_time_series_indexes.py rename to flexmeasures/data/migrations/versions/a5b970eadb3b_time_series_indexes.py diff --git a/migrations/versions/ac2613fffc74_add_market_id_column_to_asset_table.py b/flexmeasures/data/migrations/versions/ac2613fffc74_add_market_id_column_to_asset_table.py similarity index 100% rename from migrations/versions/ac2613fffc74_add_market_id_column_to_asset_table.py rename to flexmeasures/data/migrations/versions/ac2613fffc74_add_market_id_column_to_asset_table.py diff --git a/migrations/versions/b087ce8b529f_create_latest_task_run_table.py b/flexmeasures/data/migrations/versions/b087ce8b529f_create_latest_task_run_table.py similarity index 100% rename from migrations/versions/b087ce8b529f_create_latest_task_run_table.py rename to flexmeasures/data/migrations/versions/b087ce8b529f_create_latest_task_run_table.py diff --git a/migrations/versions/b2b43f0eec40_weathersensors_unique_type_location.py b/flexmeasures/data/migrations/versions/b2b43f0eec40_weathersensors_unique_type_location.py similarity index 100% rename from migrations/versions/b2b43f0eec40_weathersensors_unique_type_location.py rename to flexmeasures/data/migrations/versions/b2b43f0eec40_weathersensors_unique_type_location.py diff --git a/migrations/versions/b797328ac32d_add_user_fs_uniquifier_for_faster_auth_.py b/flexmeasures/data/migrations/versions/b797328ac32d_add_user_fs_uniquifier_for_faster_auth_.py similarity index 100% rename from migrations/versions/b797328ac32d_add_user_fs_uniquifier_for_faster_auth_.py rename to flexmeasures/data/migrations/versions/b797328ac32d_add_user_fs_uniquifier_for_faster_auth_.py diff --git a/migrations/versions/bddc5e9f72a3_add_event_resolution_field_to_asset_.py b/flexmeasures/data/migrations/versions/bddc5e9f72a3_add_event_resolution_field_to_asset_.py similarity index 100% rename from migrations/versions/bddc5e9f72a3_add_event_resolution_field_to_asset_.py rename to flexmeasures/data/migrations/versions/bddc5e9f72a3_add_event_resolution_field_to_asset_.py diff --git a/migrations/versions/d3440de27ab9_create_bvp_roles_and_bvp_users_and_bvp_roles_users_tables.py b/flexmeasures/data/migrations/versions/d3440de27ab9_create_bvp_roles_and_bvp_users_and_bvp_roles_users_tables.py similarity index 100% rename from migrations/versions/d3440de27ab9_create_bvp_roles_and_bvp_users_and_bvp_roles_users_tables.py rename to flexmeasures/data/migrations/versions/d3440de27ab9_create_bvp_roles_and_bvp_users_and_bvp_roles_users_tables.py diff --git a/migrations/versions/db00b66be82c_add_horizon_columns_as_primary_keys.py b/flexmeasures/data/migrations/versions/db00b66be82c_add_horizon_columns_as_primary_keys.py similarity index 100% rename from migrations/versions/db00b66be82c_add_horizon_columns_as_primary_keys.py rename to flexmeasures/data/migrations/versions/db00b66be82c_add_horizon_columns_as_primary_keys.py diff --git a/migrations/versions/db1f67336324_add_price_unit_and_display_name_columns_to_market_table.py b/flexmeasures/data/migrations/versions/db1f67336324_add_price_unit_and_display_name_columns_to_market_table.py similarity index 100% rename from migrations/versions/db1f67336324_add_price_unit_and_display_name_columns_to_market_table.py rename to flexmeasures/data/migrations/versions/db1f67336324_add_price_unit_and_display_name_columns_to_market_table.py diff --git a/migrations/versions/e0c2f9aff251_rename_source_id_column_in_data_sources_table.py b/flexmeasures/data/migrations/versions/e0c2f9aff251_rename_source_id_column_in_data_sources_table.py similarity index 100% rename from migrations/versions/e0c2f9aff251_rename_source_id_column_in_data_sources_table.py rename to flexmeasures/data/migrations/versions/e0c2f9aff251_rename_source_id_column_in_data_sources_table.py diff --git a/flexmeasures/data/models/forecasting/model_specs/__init__.py b/flexmeasures/data/models/forecasting/model_specs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/data/scripts/cli_tasks/background_workers.py b/flexmeasures/data/scripts/cli_tasks/background_workers.py index caa500475..a71718910 100644 --- a/flexmeasures/data/scripts/cli_tasks/background_workers.py +++ b/flexmeasures/data/scripts/cli_tasks/background_workers.py @@ -8,7 +8,7 @@ from flexmeasures.data.services.forecasting import handle_forecasting_exception -@app.cli.command("run_worker") +@app.cli.command("run-worker") @click.option( "--name", default=None, @@ -51,7 +51,7 @@ def run_worker(name: str, queue: str): worker.work() -@app.cli.command("clear_queue") +@app.cli.command("clear-queue") @click.option( "--queue", default=None, diff --git a/flexmeasures/data/scripts/cli_tasks/db_pop.py b/flexmeasures/data/scripts/cli_tasks/db_pop.py index 3633b2eff..8e118d04f 100644 --- a/flexmeasures/data/scripts/cli_tasks/db_pop.py +++ b/flexmeasures/data/scripts/cli_tasks/db_pop.py @@ -25,21 +25,27 @@ @app.cli.command() -@click.option("--username") -@click.option("--email") +@click.option("--username", required=True) +@click.option("--email", required=True) @click.option("--roles", help="e.g. anonymous,Prosumer,CPO") -@click.option("--timezone", help="timezone as string, e.g. 'UTC' or 'Europe/Amsterdam'") +@click.option( + "--timezone", + default="Europe/Amsterdam", + help="timezone as string, e.g. 'UTC' or 'Europe/Amsterdam'", +) def new_user( username: str, email: str, roles: List[str], timezone: str = "Europe/Amsterdam" ): """ + Create a FlexMeasures user. + The `users create` task from Flask Security Too is too simple for us. Use this to add email, timezone and roles. """ try: pytz.timezone(timezone) except pytz.UnknownTimeZoneError: - print("Timezone %s is unkown!" % timezone) + print("Timezone %s is unknown!" % timezone) return pwd1 = getpass.getpass(prompt="Please enter the password:") pwd2 = getpass.getpass(prompt="Please repeat the password:") @@ -111,7 +117,9 @@ def db_populate( to_date: str = "2015-12-31", asset: str = None, ): - """Initialize the database with static values.""" + """Initialize the database with static values. + TODO: split into a function for structural data and one for forecasts. + """ if structure: from flexmeasures.data.scripts.data_gen import populate_structure @@ -241,7 +249,7 @@ def db_reset( @click.option( "--data/--no-data", default=False, - help="Save (time series) data. Only do this for small data sets!", + help="Save (time series) data to a backup. Only do this for small data sets!", ) def db_save( name: str, dir: str = BACKUP_PATH, structure: bool = True, data: bool = False diff --git a/flexmeasures/data/services/time_series.py b/flexmeasures/data/services/time_series.py index 299e06476..134e37f64 100644 --- a/flexmeasures/data/services/time_series.py +++ b/flexmeasures/data/services/time_series.py @@ -238,28 +238,23 @@ def drop_non_unique_ids( def convert_query_window_for_demo( query_window: Tuple[datetime, datetime] ) -> Tuple[datetime, datetime]: + demo_year = current_app.config.get("FLEXMEASURES_DEMO_YEAR", None) + if demo_year is None: + return query_window try: - start = query_window[0].replace( - year=current_app.config.get("FLEXMEASURES_DEMO_YEAR") - ) + start = query_window[0].replace(year=demo_year) except ValueError as e: # Expand the query_window in case a leap day was selected if "day is out of range for month" in str(e): - start = (query_window[0] - timedelta(days=1)).replace( - year=current_app.config.get("FLEXMEASURES_DEMO_YEAR") - ) + start = (query_window[0] - timedelta(days=1)).replace(year=demo_year) else: start = query_window[0] try: - end = query_window[-1].replace( - year=current_app.config.get("FLEXMEASURES_DEMO_YEAR") - ) + end = query_window[-1].replace(year=demo_year) except ValueError as e: # Expand the query_window in case a leap day was selected if "day is out of range for month" in str(e): - end = (query_window[-1] + timedelta(days=1)).replace( - year=current_app.config.get("FLEXMEASURES_DEMO_YEAR") - ) + end = (query_window[-1] + timedelta(days=1)).replace(year=demo_year) else: end = query_window[-1] return start, end diff --git a/flexmeasures/data/transactional.py b/flexmeasures/data/transactional.py index f553e72d2..920da7397 100644 --- a/flexmeasures/data/transactional.py +++ b/flexmeasures/data/transactional.py @@ -34,10 +34,7 @@ def wrap(*args, **kwargs): the_db = db # run actual function, handle any exceptions and re-raise try: - if db_obj_passed: - db_function(*args[1:], **kwargs) - else: - db_function(*args, **kwargs) + db_function(*args, **kwargs) the_db.session.commit() except Exception as e: current_app.logger.error( diff --git a/flexmeasures/ui/__init__.py b/flexmeasures/ui/__init__.py index ecf09204d..c3df28dff 100644 --- a/flexmeasures/ui/__init__.py +++ b/flexmeasures/ui/__init__.py @@ -139,7 +139,7 @@ def add_jinja_variables(app): "FLEXMEASURES_MODE", "FLEXMEASURES_PLATFORM_NAME", "FLEXMEASURES_SHOW_CONTROL_UI", - "FLEXMEASURES_PUBLIC_DEMO", + "FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS", ): app.jinja_env.globals[v] = app.config.get(v, "") app.jinja_env.globals["documentation_exists"] = ( diff --git a/flexmeasures/ui/templates/admin/login_user.html b/flexmeasures/ui/templates/admin/login_user.html index dd67ffdcb..a8e2a74d1 100644 --- a/flexmeasures/ui/templates/admin/login_user.html +++ b/flexmeasures/ui/templates/admin/login_user.html @@ -26,14 +26,14 @@

{{ _('Login') }}

- {% if FLEXMEASURES_PUBLIC_DEMO %} + {% if FLEXMEASURES_MODE == "demo" and FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS %}

Interested in a demo?

Simply log in to our demo account.

    -
  • Email: demo at seita.nl
    • -
  • Password: flexdemo
    • +
  • Email: {{ FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS[0] }}
    • +
  • Password: {{ FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS[1] }}

diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index a539aa206..666643572 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -275,20 +275,20 @@

Icons from Flaticon - {% if app_running_since %} - {% if current_user.has_role('anonymous') %} - + {% if app_running_since %} + This app is running since {{ app_running_since }} {% endif %} + {% if not current_user.has_role('anonymous') %} + {% if flexmeasures_version %} + on version {{ flexmeasures_version }} + {% else %} + {% if git_version != "Unknown" %} + on version {{ git_version }}+{{ git_commits_since }}. + {% else %} + on revision {{ git_hash }}. + {% endif %} + {% endif %} {% endif %}
diff --git a/flexmeasures/ui/utils/__init__.py b/flexmeasures/ui/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flexmeasures/ui/utils/view_utils.py b/flexmeasures/ui/utils/view_utils.py index f32d5645d..6ee924e06 100644 --- a/flexmeasures/ui/utils/view_utils.py +++ b/flexmeasures/ui/utils/view_utils.py @@ -11,6 +11,7 @@ import iso8601 import pytz +from flexmeasures import __version__ as flexmeasures_version from flexmeasures.utils import time_utils from flexmeasures.ui import flexmeasures_ui from flexmeasures.data.models.user import User @@ -62,6 +63,8 @@ def render_flexmeasures_template(html_filename: str, **variables): session.get("forecast_horizon", "") ) + variables["flexmeasures_version"] = flexmeasures_version + ( variables["git_version"], variables["git_commits_since"], @@ -280,8 +283,12 @@ def set_individual_traces_for_session(): def get_git_description() -> Tuple[str, int, str]: - """Return the latest git version (tag) as a string, the number of commits since then as an int and the - current commit hash as string.""" + """ + Get information about the SCM (git) state if possible (if a .git directory exists). + + Returns the latest git version (tag) as a string, the number of commits since then as an int and the + current commit hash as string. + """ def _minimal_ext_cmd(cmd: list): # construct minimal environment @@ -299,25 +306,22 @@ def _minimal_ext_cmd(cmd: list): version = "Unknown" commits_since = 0 sha = "Unknown" - try: + + path_to_flexmeasures_root = os.path.join( + os.path.dirname(__file__), "..", "..", ".." + ) + if os.path.exists(os.path.join(path_to_flexmeasures_root, ".git")): commands = ["git", "describe", "--always", "--long"] - path_to_flexmeasures = os.path.join(os.path.dirname(__file__), "..", "..", "..") - if not os.path.exists(os.path.join(path_to_flexmeasures, ".git")): - # convention if we are operating in a non-git checkout, could be made configurable - commands.insert( - 1, - "--git-dir=%s" - % os.path.join(path_to_flexmeasures, "..", "flexmeasures.git"), - ) - git_output = _minimal_ext_cmd(commands) - components = git_output.strip().decode("ascii").split("-") - if not (len(components) == 1 and components[0] == ""): - sha = components.pop() - if len(components) > 0: - commits_since = int(components.pop()) - version = "-".join(components) - except OSError as ose: - current_app.logger.warning("Problem when reading git describe: %s" % ose) + try: + git_output = _minimal_ext_cmd(commands) + components = git_output.strip().decode("ascii").split("-") + if not (len(components) == 1 and components[0] == ""): + sha = components.pop() + if len(components) > 0: + commits_since = int(components.pop()) + version = "-".join(components) + except OSError as ose: + current_app.logger.warning("Problem when reading git describe: %s" % ose) return version, commits_since, sha diff --git a/flexmeasures/ui/views/analytics.py b/flexmeasures/ui/views/analytics.py index 85f27e880..f1176b839 100644 --- a/flexmeasures/ui/views/analytics.py +++ b/flexmeasures/ui/views/analytics.py @@ -62,10 +62,15 @@ def analytics_view(): group for group in asset_groups if asset_groups[group].count() > 0 ] selected_resource = set_session_resource(assets, asset_group_names) + if selected_resource is None: + raise Exception( + "No assets exist yet, so the analytics view will not work. Please add an asset!" + ) + selected_market = set_session_market(selected_resource) sensor_types = get_sensor_types(selected_resource) - selected_sensor_type = set_session_sensor_type(sensor_types) session_asset_types = selected_resource.unique_asset_types + selected_sensor_type = set_session_sensor_type(sensor_types) set_individual_traces_for_session() view_shows_individual_traces = ( session["showing_individual_traces_for"] in ("power", "schedules") @@ -84,6 +89,8 @@ def analytics_view(): # Only show production positive if all assets are producers show_consumption_as_positive = False if showing_pure_production_data else True + # ---- Get data + data, metrics, weather_type, selected_weather_sensor = get_data_and_metrics( query_window, resolution, @@ -97,59 +104,20 @@ def analytics_view(): selected_resource.assets, ) + # TODO: get rid of these hacks, which we use because we mock the current year's data from 2015 data in demo mode + # Our demo server uses 2015 data as if it's the current year's data. Here we mask future beliefs. + if current_app.config.get("FLEXMEASURES_MODE", "") == "demo": + data = filter_for_past_data(data) + data = filter_forecasts_for_limited_time_window(data) + + # ---- Making figures + # Set shared x range shared_x_range = Range1d(start=query_window[0], end=query_window[1]) shared_x_range2 = Range1d( start=query_window[0], end=query_window[1] ) # only needed if we draw two legends (if individual traces are on) - # TODO: get rid of this hack, which we use because we mock the current year's data from 2015 data in demo mode - # Our demo server uses 2015 data as if it's the current year's data. Here we mask future beliefs. - if current_app.config.get("FLEXMEASURES_MODE", "") == "demo": - - most_recent_quarter = time_utils.get_most_recent_quarter() - - # Show only past data, pretending we're in the current year - if not data["power"].empty: - data["power"] = data["power"].loc[ - data["power"].index.get_level_values("event_start") - < most_recent_quarter - ] - if not data["prices"].empty: - data["prices"] = data["prices"].loc[ - data["prices"].index < most_recent_quarter + timedelta(hours=24) - ] # keep tomorrow's prices - if not data["weather"].empty: - data["weather"] = data["weather"].loc[ - data["weather"].index < most_recent_quarter - ] - if not data["rev_cost"].empty: - data["rev_cost"] = data["rev_cost"].loc[ - data["rev_cost"].index.get_level_values("event_start") - < most_recent_quarter - ] - - # Show forecasts only up to a limited horizon - horizon_days = 10 # keep a 10 day forecast - max_forecast_datetime = most_recent_quarter + timedelta(hours=horizon_days * 24) - if not data["power_forecast"].empty: - data["power_forecast"] = data["power_forecast"].loc[ - data["power_forecast"].index < max_forecast_datetime - ] - if not data["prices_forecast"].empty: - data["prices_forecast"] = data["prices_forecast"].loc[ - data["prices_forecast"].index < max_forecast_datetime - ] - if not data["weather_forecast"].empty: - data["weather_forecast"] = data["weather_forecast"].loc[ - data["weather_forecast"].index < max_forecast_datetime - ] - if not data["rev_cost_forecast"].empty: - data["rev_cost_forecast"] = data["rev_cost_forecast"].loc[ - data["rev_cost_forecast"].index < max_forecast_datetime - ] - - # Making figures tools = ["box_zoom", "reset", "save"] power_fig = make_power_figure( selected_resource.display_name, @@ -520,6 +488,53 @@ def get_data_and_metrics( return data, metrics, weather_type, selected_sensor +def filter_for_past_data(data): + """ Make sure we only show past data, useful for demo mode """ + most_recent_quarter = time_utils.get_most_recent_quarter() + + if not data["power"].empty: + data["power"] = data["power"].loc[ + data["power"].index.get_level_values("event_start") < most_recent_quarter + ] + if not data["prices"].empty: + data["prices"] = data["prices"].loc[ + data["prices"].index < most_recent_quarter + timedelta(hours=24) + ] # keep tomorrow's prices + if not data["weather"].empty: + data["weather"] = data["weather"].loc[ + data["weather"].index < most_recent_quarter + ] + if not data["rev_cost"].empty: + data["rev_cost"] = data["rev_cost"].loc[ + data["rev_cost"].index.get_level_values("event_start") < most_recent_quarter + ] + return data + + +def filter_forecasts_for_limited_time_window(data): + """ Show forecasts only up to a limited horizon """ + most_recent_quarter = time_utils.get_most_recent_quarter() + horizon_days = 10 # keep a 10 day forecast + max_forecast_datetime = most_recent_quarter + timedelta(hours=horizon_days * 24) + if not data["power_forecast"].empty: + data["power_forecast"] = data["power_forecast"].loc[ + data["power_forecast"].index < max_forecast_datetime + ] + if not data["prices_forecast"].empty: + data["prices_forecast"] = data["prices_forecast"].loc[ + data["prices_forecast"].index < max_forecast_datetime + ] + if not data["weather_forecast"].empty: + data["weather_forecast"] = data["weather_forecast"].loc[ + data["weather_forecast"].index < max_forecast_datetime + ] + if not data["rev_cost_forecast"].empty: + data["rev_cost_forecast"] = data["rev_cost_forecast"].loc[ + data["rev_cost_forecast"].index < max_forecast_datetime + ] + return data + + def make_power_figure( resource_display_name: str, data: pd.DataFrame, @@ -540,7 +555,7 @@ def make_power_figure( data, unit="MW", legend_location="top_right", - legend_labels=("Actual", "Forecast") + legend_labels=("Actual", "Forecast", None) if schedule_data is None or schedule_data["event_value"].isnull().all() else ("Actual", "Forecast", "Schedule"), forecasts=forecast_data, diff --git a/flexmeasures/utils/app_utils.py b/flexmeasures/utils/app_utils.py index c82eca149..7d1845db6 100644 --- a/flexmeasures/utils/app_utils.py +++ b/flexmeasures/utils/app_utils.py @@ -1,21 +1,65 @@ import os import sys +import click +from flask.cli import FlaskGroup -def install_secret_key(app, filename="secret_key"): - """Configure the SECRET_KEY from a file - in the instance directory. +from flexmeasures.app import create as create_app - If the file does not exist, print instructions - to create it from a shell with a random key, - then exit. + +@click.group(cls=FlaskGroup, create_app=create_app) +def flexmeasures_cli(): + """Management scripts for the FlexMeasures platform.""" + pass + + +def set_secret_key(app, filename="secret_key"): + """Set the SECRET_KEY or exit. + + We first check if it is already in the config. + + Then we look for it in environment var SECRET_KEY. + + Finally, we look for `filename` in the app's instance directory. + + If nothing is found, we print instructions + to create the secret and then exit. """ + secret_key = app.config.get("SECRET_KEY", None) + if secret_key is not None: + return + secret_key = os.environ.get("SECRET_KEY", None) + if secret_key is not None: + app.config["SECRET_KEY"] = secret_key + return filename = os.path.join(app.instance_path, filename) try: app.config["SECRET_KEY"] = open(filename, "rb").read() except IOError: - print("Error: No secret key. Create it with:") - if not os.path.isdir(os.path.dirname(filename)): - print("mkdir -p", os.path.dirname(filename)) - print("head -c 24 /dev/urandom >", filename) + print( + """ + Error: No secret key set. + + You can add the SECRET_KEY setting to your conf file (this example works only on Unix): + + echo "SECRET_KEY=\\"`head -c 24 /dev/urandom`\\"" >> your-flexmeasures.cfg + + OR you can add an env var: + + export SECRET_KEY=xxxxxxxxxxxxxxx + (on windows, use "set" instead of "export") + + OR you can create a secret key file (this example works only on Unix): + + mkdir -p %s + head -c 24 /dev/urandom > %s + + You can also use Python to create a good secret: + + python -c "import secrets; print(secrets.token_urlsafe())" + + """ + % (os.path.dirname(filename), filename) + ) + sys.exit(2) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 6f7b487c1..c84ebe111 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -1,6 +1,6 @@ from datetime import timedelta import logging -from typing import List, Optional, Union +from typing import List, Optional, Union, Dict, Tuple """ This lays out our configuration requirements and allows to set trivial defaults, per environment adjustable. @@ -15,14 +15,14 @@ class Config(object): Otherwise, set to None, so that it can be set either by subclasses or the env-specific config script. """ - DEBUG = False - LOGGING_LEVEL = logging.WARNING - CSRF_ENABLED = True + DEBUG: bool = False + LOGGING_LEVEL: int = logging.WARNING + SECRET_KEY: Optional[str] = None SQLALCHEMY_DATABASE_URI: Optional[str] = None # https://stackoverflow.com/questions/33738467/how-do-i-know-if-i-can-disable-sqlalchemy-track-modifications - SQLALCHEMY_TRACK_MODIFICATIONS = False - SQLALCHEMY_ENGINE_OPTIONS = { + SQLALCHEMY_TRACK_MODIFICATIONS: bool = False + SQLALCHEMY_ENGINE_OPTIONS: dict = { "pool_recycle": 299, # https://www.pythonanywhere.com/forums/topic/2599/ # "pool_timeout": 20, "pool_pre_ping": True, # https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic @@ -31,10 +31,10 @@ class Config(object): }, # https://stackoverflow.com/a/59932909/13775459 } - MAIL_SERVER: Optional[str] = None - MAIL_PORT: Optional[str] = None - MAIL_USE_TLS: Optional[str] = None - MAIL_USE_SSL: Optional[str] = None + MAIL_SERVER: Optional[str] = "localhost" + MAIL_PORT: Optional[int] = 25 + MAIL_USE_TLS: Optional[bool] = False + MAIL_USE_SSL: Optional[bool] = False MAIL_USERNAME: Optional[str] = None MAIL_DEFAULT_SENDER = ( "FlexMeasures", @@ -69,49 +69,57 @@ class Config(object): MAPBOX_ACCESS_TOKEN: Optional[str] = None - JSONIFY_PRETTYPRINT_REGULAR = False + JSONIFY_PRETTYPRINT_REGULAR: bool = False RQ_DASHBOARD_POLL_INTERVAL: int = ( 3000 # Web interface poll period for updates in ms ) - FLEXMEASURES_PLATFORM_NAME = "FlexMeasures" - FLEXMEASURES_MODE = "" - FLEXMEASURES_PUBLIC_DEMO = False - FLEXMEASURES_TIMEZONE = "Asia/Seoul" - FLEXMEASURES_SHOW_CONTROL_UI = False - FLEXMEASURES_HIDE_NAN_IN_UI = False - FLEXMEASURES_DEMO_YEAR = 2015 + FLEXMEASURES_PLATFORM_NAME: str = "FlexMeasures" + FLEXMEASURES_MODE: str = "" + FLEXMEASURES_TIMEZONE: str = "Asia/Seoul" + FLEXMEASURES_SHOW_CONTROL_UI: bool = False + FLEXMEASURES_HIDE_NAN_IN_UI: bool = False + FLEXMEASURES_PUBLIC_DEMO_CREDENTIALS: Optional[Tuple] = None + FLEXMEASURES_DEMO_YEAR: Optional[int] = None # Configuration used for entity addressing: - # we list the domain on which FlexMeasures runs + # This setting contains the domain on which FlexMeasures runs # and the first month when the domain was under the current owner's administration - FLEXMEASURES_HOSTS_AND_AUTH_START = {"flexmeasures.io": "2021-01"} + FLEXMEASURES_HOSTS_AND_AUTH_START: dict = {"flexmeasures.io": "2021-01"} FLEXMEASURES_PROFILE_REQUESTS: bool = False - FLEXMEASURES_DB_BACKUP_PATH = "migrations/dumps" - FLEXMEASURES_LP_SOLVER = "cbc" + FLEXMEASURES_DB_BACKUP_PATH: str = "migrations/dumps" + FLEXMEASURES_LP_SOLVER: str = "cbc" FLEXMEASURES_PLANNING_HORIZON: timedelta = timedelta(hours=2 * 24) FLEXMEASURES_PLANNING_TTL: timedelta = timedelta( days=7 ) # Time to live for UDI event ids of successful scheduling jobs. Set a negative timedelta to persist forever. FLEXMEASURES_TASK_CHECK_AUTH_TOKEN: Optional[str] = None - FLEXMEASURES_PA_DOMAIN_NAMES: List[str] = [] - FLEXMEASURES_REDIS_URL = "localhost" - FLEXMEASURES_REDIS_PORT = 6379 - FLEXMEASURES_REDIS_DB_NR = 0 # Redis per default has 16 databases, [0-15] - FLEXMEASURES_REDIS_PASSWORD = None - - # names of settings which cannot be None - required: List[str] = [ - "SQLALCHEMY_DATABASE_URI", - "MAIL_SERVER", - "MAIL_PORT", - "MAIL_USE_TLS", - "MAIL_USE_SSL", - "MAIL_USERNAME", - "MAIL_DEFAULT_SENDER", - "MAIL_PASSWORD", - "SECURITY_PASSWORD_SALT", - ] + FLEXMEASURES_REDIS_URL: str = "localhost" + FLEXMEASURES_REDIS_PORT: int = 6379 + FLEXMEASURES_REDIS_DB_NR: int = 0 # Redis per default has 16 databases, [0-15] + FLEXMEASURES_REDIS_PASSWORD: Optional[str] = None + + +# names of settings which cannot be None +# SECRET_KEY is also required but utils.app_utils.set_secret_key takes care of this better. +required: List[str] = ["SQLALCHEMY_DATABASE_URI"] + +# settings whose absence should trigger a warning +mail_warning = "Without complete mail settings, FlexMeasures will not be able to send mails to users, e.g. for password resets!" +redis_warning = "Without complete redis connection settings, FlexMeasures will not be able to run forecasting and scheduling job queues." +warnable: Dict[str, str] = { + "MAIL_SERVER": mail_warning, + "MAIL_PORT": mail_warning, + "MAIL_USE_TLS": mail_warning, + "MAIL_USE_SSL": mail_warning, + "MAIL_USERNAME": mail_warning, + "MAIL_DEFAULT_SENDER": mail_warning, + "MAIL_PASSWORD": mail_warning, + "FLEXMEASURES_REDIS_URL": redis_warning, + "FLEXMEASURES_REDIS_PORT": redis_warning, + "FLEXMEASURES_REDIS_DB_NR": redis_warning, + "FLEXMEASURES_REDIS_PASSWORD": redis_warning, +} class ProductionConfig(Config): diff --git a/flexmeasures/utils/config_utils.py b/flexmeasures/utils/config_utils.py index 6f48cb59a..f3d911d54 100644 --- a/flexmeasures/utils/config_utils.py +++ b/flexmeasures/utils/config_utils.py @@ -1,12 +1,19 @@ import os import sys import logging +from typing import Optional, List, Tuple from datetime import datetime from logging.config import dictConfig as loggingDictConfig +from pathlib import Path +from flask import Flask from inflection import camelize -from flexmeasures.utils.config_defaults import Config as DefaultConfig +from flexmeasures.utils.config_defaults import ( + Config as DefaultConfig, + required, + warnable, +) basedir = os.path.abspath(os.path.dirname(__file__)) @@ -43,7 +50,7 @@ def configure_logging(): loggingDictConfig(flexmeasures_logging_config) -def read_config(app): +def read_config(app: Flask, path_to_config: Optional[str]): """Read configuration from various expected sources, complain if not setup correctly. """ if app.env not in ( @@ -58,50 +65,91 @@ def read_config(app): ) sys.exit(2) + # Load default config settings app.config.from_object( "flexmeasures.utils.config_defaults.%sConfig" % camelize(app.env) ) - env_config_path = "%s/%s_config.py" % (app.root_path, app.env) - + # Now read user config, if possible. If no explicit path is given, try home dir first, then instance dir + if path_to_config is not None and not os.path.exists(path_to_config): + print(f"Cannot find config file {path_to_config}!") + sys.exit(2) + path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) + path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") + if path_to_config is None: + path_to_config = path_to_config_home + if not os.path.exists(path_to_config): + path_to_config = path_to_config_instance try: - app.config.from_pyfile(env_config_path) + app.config.from_pyfile(path_to_config) except FileNotFoundError: pass + # Finally, all required varaiables can be set as env var: + for req_var in required: + app.config[req_var] = os.getenv(req_var, app.config.get(req_var, None)) - # Check for missing values. Testing might affect only specific functionality (-> dev's responsibility) + # Check for missing values. + # Testing might affect only specific functionality (-> dev's responsibility) + # Documentation runs fine without them. if not app.testing and app.env != "documentation": - missing_settings = check_config_completeness(app) - if len(missing_settings) > 0: - if not os.path.exists(env_config_path): + if not are_required_settings_complete(app): + if not os.path.exists(path_to_config): print( - 'Missing configuration settings: %s\nAs FLASK_ENV=%s, please provide the file "%s"' - " in the flexmeasures directory, and include these settings." - % (", ".join(missing_settings), app.env, env_config_path) + f"You can provide these settings ― as environment variables or in your config file (e.g. {path_to_config_home} or {path_to_config_instance})." ) else: print( - "Missing configuration settings: %s" % ", ".join(missing_settings) + f"Please provide these settings ― as environment variables or in your config file ({path_to_config})." ) sys.exit(2) + missing_fields, config_warnings = get_config_warnings(app) + if len(config_warnings) > 0: + for warning in config_warnings: + print(f"Warning: {warning}") + print(f"You might consider setting {', '.join(missing_fields)}.") # Set the desired logging level on the root logger (controlling extension logging level) # and this app's logger. - logging.getLogger().setLevel(app.config.get("LOGGING_LEVEL")) - app.logger.setLevel(app.config.get("LOGGING_LEVEL")) + logging.getLogger().setLevel(app.config.get("LOGGING_LEVEL", "INFO")) + app.logger.setLevel(app.config.get("LOGGING_LEVEL", "INFO")) # print("Logging level is %s" % logging.getLevelName(app.logger.level)) app.config["START_TIME"] = datetime.utcnow() -def check_config_completeness(app): - """Check if all settings we expect are not None. Return the ones that are None.""" - expected_settings = [] - for attr in [ +def are_required_settings_complete(app) -> bool: + """ + Check if all settings we expect are not None. Return False if they are not. + Printout helpful advice. + """ + expected_settings = [s for s in get_configuration_keys(app) if s in required] + missing_settings = [s for s in expected_settings if app.config.get(s) is None] + if len(missing_settings) > 0: + print( + f"Missing the required configuration settings: {', '.join(missing_settings)}" + ) + return False + return True + + +def get_config_warnings(app) -> Tuple[List[str], List[str]]: + """return missing settings and the warnings for them.""" + missing_settings = [] + config_warnings = [] + for setting, warning in warnable.items(): + if app.config.get(setting) is None: + missing_settings.append(setting) + config_warnings.append(warning) + config_warnings = list(set(config_warnings)) + return missing_settings, config_warnings + + +def get_configuration_keys(app) -> List[str]: + """ + Collect all members of DefaultConfig who are not in-built fields or callables. + """ + return [ a for a in DefaultConfig.__dict__ - if not a.startswith("__") and a in DefaultConfig.required - ]: - if not callable(getattr(DefaultConfig, attr)): - expected_settings.append(attr) - return [s for s in expected_settings if app.config.get(s) is None] + if not a.startswith("__") and not callable(getattr(DefaultConfig, a)) + ] diff --git a/requirements/app.in b/requirements/app.in index 7d9af9459..592ed01c5 100644 --- a/requirements/app.in +++ b/requirements/app.in @@ -34,6 +34,8 @@ tables timetomodel>=0.6.8 timely-beliefs>=1.2.1 python-dotenv +# a backport, not needed in Python3.8 +importlib_metadata Flask-SSLify Flask_JSON Flask-SQLAlchemy>=2.4.3 diff --git a/requirements/app.txt b/requirements/app.txt index 045379400..c9ed77bec 100644 --- a/requirements/app.txt +++ b/requirements/app.txt @@ -42,6 +42,7 @@ flask==1.1.2 # via -r requirements/app.in, flask-babelex, flask-cla forecastiopy==0.22 # via -r requirements/app.in humanize==2.6.0 # via -r requirements/app.in idna==2.10 # via email-validator, requests, tldextract +importlib-metadata==3.7.0 # via -r requirements/app.in inflect==4.1.0 # via -r requirements/app.in inflection==0.5.1 # via -r requirements/app.in iso8601==0.1.12 # via -r requirements/app.in @@ -111,6 +112,7 @@ webargs==7.0.1 # via -r requirements/app.in werkzeug==1.0.1 # via flask wtforms==2.3.3 # via flask-wtf xlrd==1.2.0 # via -r requirements/app.in +zipp==3.4.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/dev.in b/requirements/dev.in index eabe567c2..17dc56110 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -5,4 +5,5 @@ pre-commit black flake8 flake8-blind-except -mypy \ No newline at end of file +mypy +watchdog \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 704a2e83d..9cbaeda3d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -28,6 +28,7 @@ toml==0.10.1 # via -c requirements/test.txt, black, pre-commit typed-ast==1.4.1 # via black, mypy typing-extensions==3.7.4.3 # via black, mypy virtualenv==20.0.31 # via pre-commit +watchdog==2.0.2 # via -r requirements/dev.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/flexmeasures/run-local.py b/run-local.py similarity index 100% rename from flexmeasures/run-local.py rename to run-local.py diff --git a/setup.cfg b/setup.cfg index c8930222a..72b2bacf1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,12 +1,11 @@ [aliases] -test=pytest -flake8=flake8 +test = pytest +flake8 = flake8 [flake8] exclude = .git,__pycache__,documentation max-line-length = 160 max-complexity = 13 -# this is a whitelist since flake8 v3; B9 is flake-bugbear select = B,C,E,F,W,B9 -# ignore E501 bcs we use bugbear for line-length, W503 and E203 because black does, too ignore = E501, W503, E203 + diff --git a/setup.py b/setup.py index aaa7dd060..d915519e8 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,9 @@ -from setuptools import setup - -from flexmeasures import __version__ +from setuptools import setup, find_packages def load_requirements(use_case): reqs = [] - with open("requirements/%s.in" % use_case, "r") as f: + with open("requirements/%s.txt" % use_case, "r") as f: reqs = [ req for req in f.read().splitlines() @@ -21,16 +19,23 @@ def load_requirements(use_case): description="FlexMeasures - A free platform for real-time optimization of flexible energy.", author="Seita BV", author_email="nicolas@seita.nl", + url="https://github.com/seitabv/flexmeasures", keywords=["smart grid", "renewables", "balancing", "forecasting", "scheduling"], - version=__version__, + python_requires=">=3.7.1", # not enforced, just info install_requires=load_requirements("app"), - setup_requires=["pytest-runner"], tests_require=load_requirements("test"), - packages=["flexmeasures"], - include_package_data=True, + setup_requires=["pytest-runner", "setuptools_scm"], + use_scm_version={"local_scheme": "no-local-version"}, # handled by setuptools_scm + packages=find_packages(), + include_package_data=True, # setuptools_scm takes care of adding the files in SCM + entry_points={ + "console_scripts": [ + "flexmeasures=flexmeasures.utils.app_utils:flexmeasures_cli" + ], + }, classifiers=[ "Programming Language :: Python", - "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", @@ -38,5 +43,14 @@ def load_requirements(use_case): long_description="""\ The *FlexMeasures Platform* is a tool for scheduling flexible actions for energy assets. For this purpose, it performs monitoring, forecasting and scheduling services. + +FlexMeasures is fully usable via APIs, which are inspired by the Universal Smart Energy Framework (USEF). +Some algorithms are included, but projects will usually write their own (WIP). + +Energy Flexibility is one of the key ingredients to reducing CO2. FlexMeasures is meant +to facilitate the transition to a carbon-free energy system. By open-sourcing FlexMeasures, +we hope to speed up this transition world-wide. + +Please visit https://flexmeasures.io to learn more. """, ) diff --git a/to_pypi.sh b/to_pypi.sh new file mode 100755 index 000000000..a26723733 --- /dev/null +++ b/to_pypi.sh @@ -0,0 +1,38 @@ +#!/bin/bash + + +# Script to release FlexMeasures to PyPi. +# +# The version +# ------------- +# The version comes from setuptools_scm. See `python setup.py --version`. +# setuptools_scm works via git tags that should implement a semantic versioning scheme, e.g. v0.2.3 +# +# If there were zero commits since since tag, we have a real release and the version basicaly *is* what the tag says. +# Otherwise, the version also include a .devN identifier, where N is the number of commits since the last version tag. +# +# More information on creating a dev release +# ------------------------------------------- +# Note that the only way to create a new dev release is to add another commit on your development branch. +# It might have been convenient to not have to commit to do that (for exoerimenting with very small changes), +# but we decided against that. Let's explore why for a bit: +# +# First, setuptools_scm has the ability to add a local scheme (git commit and date/time) to the version, +# but we've disabled that, as that extra part isn't formatted in a way that Pypi accepts it. +# Another way would have been to add a local version identifier ("+M", not the plus sign), +# which is allowed in PEP 440 but explicitly disallowed by Pypi. +# Finally, if we simply add a number to .devN (-> .devNM), the ordering of dev versions would be +# disturbed after the next local commit (e.g. we add 1 to .dev4, making it .dev41, and then the next version, .dev5, +# is not the highest version chosen by PyPi). +# +# So we'll use these tools as the experts intend us to. +# If you want, you can read more about acceptable versions in PEP 440: https://www.python.org/dev/peps/pep-0440/ + + +rm -rf build/* dist/* +pip -q install twine + +python setup.py egg_info sdist +python setup.py egg_info bdist_wheel + +twine upload dist/* \ No newline at end of file