From 7156f12c600da7e336aaa97c098e2f3e439b5f79 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 26 Apr 2021 13:48:44 -0600 Subject: [PATCH 1/3] removing flavor=spark from parquet writing --- buildstockbatch/postprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildstockbatch/postprocessing.py b/buildstockbatch/postprocessing.py index 5d75181a..eaf8d992 100644 --- a/buildstockbatch/postprocessing.py +++ b/buildstockbatch/postprocessing.py @@ -147,7 +147,7 @@ def read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, buildin def write_dataframe_as_parquet(df, fs, filename): tbl = pa.Table.from_pandas(df, preserve_index=False) with fs.open(filename, 'wb') as f: - parquet.write_table(tbl, f, flavor='spark') + parquet.write_table(tbl, f) def clean_up_results_df(df, cfg, keep_upgrade_id=False): From 4e39fa66dbe29b44a0f529cc9118cddf55ad1868 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 26 Apr 2021 16:14:29 -0600 Subject: [PATCH 2/3] updating docs for athena engine version --- docs/installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 0d491ffd..2f793526 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -53,9 +53,11 @@ configure your user account with your AWS credentials. This setup only needs to 1. `Install the AWS CLI`_ version 2 (select the version for your local OS; not for Docker). 2. `Configure the AWS CLI`_. (Don't type the ``$`` in the example.) +3. You may need to `change the Athena Engine version`_ for your query workgroup to v2. .. _Install the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html .. _Configure the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration +.. _change the Athena Engine version: https://docs.aws.amazon.com/athena/latest/ug/engine-versions-changing.html .. _eagle_install: From e8211a4f335d93c9f097e1937cbcd76e18cae9d3 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 26 Apr 2021 16:17:40 -0600 Subject: [PATCH 3/3] updating changelog --- docs/changelog/changelog_dev.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst index 108d3970..1730aee3 100644 --- a/docs/changelog/changelog_dev.rst +++ b/docs/changelog/changelog_dev.rst @@ -60,3 +60,11 @@ Development Changelog The glue crawler was failing when there was a trailing ``/`` character. This fixes that as well as checks to make sure files were uploaded before running the crawler. + + .. change:: + :tags: bugfix + :pullreq: 224 + :tickets: 221 + + Defaults to the newer datetime encoding in the parquet files now that + Athena can understand it.