diff --git a/documentation/changelog.rst b/documentation/changelog.rst index aa1035037..0a833676e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,7 +12,9 @@ Bugfixes * Documentation listed 2.0 API endpoints twice [see `PR #59 `_] * User page did not list number of assets correctly [see `PR #64 `_] - +Infrastructure/Support +---------------------- +* Dump and restore postgres database as CLI commands [see `PR #68 `_] v0.2.3 | February 27, 2021 =========================== diff --git a/flexmeasures/data/Readme.md b/flexmeasures/data/Readme.md index e049434a6..496c46259 100644 --- a/flexmeasures/data/Readme.md +++ b/flexmeasures/data/Readme.md @@ -105,11 +105,11 @@ You need data to enjoy the benefits of FlexMeasures or to develop features for i ### Import from another database -Here is a short recipe to import data from a database (e.g. a demo database) into your local system. +Here is a short recipe to import data from a FlexMeasures database (e.g. a demo database) into your local system. On the to-be-exported database: - pg_dump --host=YOUR_URL --port=YOUR_PORT --username=YOUR_USER --no-privileges --no-owner --data-only --format=c --file=pgbackup_`date +%F-%H%M`.dump YOUR_DB_NAME + flask db-dump Note that we only dump the data here. Locally, we create a fresh database with the structure being based on the data model given by the local codebase: @@ -117,7 +117,7 @@ Note that we only dump the data here. Locally, we create a fresh database with t Then we import the data dump we made earlier: - pg_restore -U YOUR_DEV_USER --password -h 127.0.0.1 -d YOUR_DEV_DB_NAME ~/Downloads/pgbackup_YOUR_DATETIME.dump + flask db-restore A potential `alembic_version` error should not prevent other data tables from being restored. You can also choose to import a complete db dump into a freshly created database, of course. diff --git a/flexmeasures/data/scripts/cli_tasks/db_pop.py b/flexmeasures/data/scripts/cli_tasks/db_pop.py index 71a3fc0d1..3633b2eff 100644 --- a/flexmeasures/data/scripts/cli_tasks/db_pop.py +++ b/flexmeasures/data/scripts/cli_tasks/db_pop.py @@ -1,6 +1,7 @@ """CLI Tasks for (de)populating the database - most useful in development""" -from datetime import timedelta +from datetime import datetime, timedelta +import subprocess from typing import List import pandas as pd @@ -321,3 +322,54 @@ def create_power_forecasts( end_of_roll=pd.Timestamp(to_date).tz_localize(timezone) - timedelta(hours=horizon_hours), ) + + +@app.cli.command() +def db_dump(): + """Create a database dump of the database used by the app.""" + db_uri = app.config.get("SQLALCHEMY_DATABASE_URI") + db_host_and_db_name = db_uri.split("@")[-1] + click.echo(f"Backing up {db_host_and_db_name} database") + db_name = db_host_and_db_name.split("/")[-1] + time_of_saving = datetime.now().strftime("%F-%H%M") + dump_filename = f"pgbackup_{db_name}_{time_of_saving}.dump" + command_for_dumping = f"pg_dump --no-privileges --no-owner --data-only --format=c --file={dump_filename} {db_uri}" + try: + proc = subprocess.Popen(command_for_dumping, shell=True) # , env={ + # 'PGPASSWORD': DB_PASSWORD + # }) + proc.wait() + click.echo(f"db dump successful: saved to {dump_filename}") + + except Exception as e: + click.echo(f"Exception happened during dump: {e}") + click.echo("db dump unsuccessful") + + +@app.cli.command() +@click.argument("file", type=click.Path(exists=True)) +def db_restore(file: str): + """Restore the database used by the app, from a given database dump file, after you've reset the database. + + From the command line: + + % db-dump + % db-reset + % db-restore FILE + + """ + + db_uri = app.config.get("SQLALCHEMY_DATABASE_URI") + db_host_and_db_name = db_uri.split("@")[-1] + click.echo(f"Restoring {db_host_and_db_name} database from file {file}") + command_for_restoring = f"pg_restore -d {db_uri} {file}" + try: + proc = subprocess.Popen(command_for_restoring, shell=True) # , env={ + # 'PGPASSWORD': DB_PASSWORD + # }) + proc.wait() + click.echo("db restore successful") + + except Exception as e: + click.echo(f"Exception happened during restore: {e}") + click.echo("db restore unsuccessful")