Merge pull request #224 from NREL/athena-datetime

Writing parquet files with newer datetime format

buildstockbatch/postprocessing.py

@@ -147,7 +147,7 @@ def read_simulation_outputs(fs, reporting_measures, sim_dir, upgrade_id, buildin
 def write_dataframe_as_parquet(df, fs, filename):
     tbl = pa.Table.from_pandas(df, preserve_index=False)
     with fs.open(filename, 'wb') as f:
-        parquet.write_table(tbl, f, flavor='spark')
+        parquet.write_table(tbl, f)
 
 
 def clean_up_results_df(df, cfg, keep_upgrade_id=False):

docs/changelog/changelog_dev.rst

@@ -61,9 +61,17 @@ Development Changelog
         This fixes that as well as checks to make sure files were uploaded
         before running the crawler.
 
+    .. change::
+        :tags: bugfix
+        :pullreq: 224
+        :tickets: 221
+
+        Defaults to the newer datetime encoding in the parquet files now that
+        Athena can understand it.
+
     .. change::
         :tags: workflow
-        :pullreq:
-        :tickets:
+        :pullreq: 219
+        :tickets: 189
 
-        Adding measure arguments for reporting measures in the workflow generator.
+        Adding measure arguments for reporting measures in the workflow generator.

docs/installation.rst

@@ -66,9 +66,11 @@ configure your user account with your AWS credentials. This setup only needs to
 
 1. `Install the AWS CLI`_ version 2 (select the version for your local OS; not for Docker).
 2. `Configure the AWS CLI`_. (Don't type the ``$`` in the example.)
+3. You may need to `change the Athena Engine version`_ for your query workgroup to v2.
 
 .. _Install the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html
 .. _Configure the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration
+.. _change the Athena Engine version: https://docs.aws.amazon.com/athena/latest/ug/engine-versions-changing.html
 
 .. _eagle_install: