Merge pull request #2129 from PecanProject/release/v1.6.0

Release/v1.6.0
PecanProject · Oct 2, 2018 · 5ea78dd · 5ea78dd
2 parents 5123e52 + 069eb06
commit 5ea78dd
Show file tree

Hide file tree

Showing 765 changed files with 20,106 additions and 5,673 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -1,5 +1,6 @@
 <!--- Provide a general summary of the issue in the Title above -->
-
+<!--- Please select appropriate Priority, Status,and Type labels-->
+<!--- If you do not have permission to select labels please state which labels you would like -->
 ## Description
 <!--- What change or feature do you propose? -->
 

diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md
@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Machine (please complete the following information):**
+ - Server [e.g. BU, NCSA, VM, etc.]
+ - OS: [e.g.Linux]
+ - Browser(if applicable) [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/Feature_request.md b/.github/ISSUE_TEMPLATE/Feature_request.md
@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,4 +1,6 @@
 <!--- Provide a general summary of your changes in the Title above -->
+<!--- Please select appropriate Priority, Status,and Type labels-->
+<!--- If you do not have permission to select labels please state which labels you would like -->
 
 ## Description
 <!--- Describe your changes in detail -->

diff --git a/.gitignore b/.gitignore
@@ -83,3 +83,5 @@ shiny/BenchmarkReport/*
 .doc/
 # files generated by tests
 base/qaqc/tests/testthat/Rplots.pdf
+docker/base/build.*.log
+docker/models/build.*.log
diff --git a/.travis.yml b/.travis.yml
@@ -5,13 +5,15 @@
 # trusty. They are marked with TRUSTY for easy grepping.
 
 language: r
-
 r:
   - 3.4
 
 # TRUSTY: Change version when 16.04 image is available.
 dist: trusty
-sudo: false
+sudo: required
+
+env:
+  - PGHOST=localhost
 
 cache: 
   - directories:
@@ -22,30 +24,30 @@ cache:
   - packages
 
 addons:
-  postgresql: 9.6
   apt:
     sources:
       - sourceline: 'ppa:marutter/c2d4u'
       - sourceline: 'ppa:nschloe/hdf5-backports'
+      - sourceline: 'ppa:ubuntugis/ppa' # for GDAL 2 binaries
     packages:
-      - libnetcdf-dev
-      - liblapack-dev
-      - libcurl4-openssl-dev
-      - jags
-      - libudunits2-dev
-      - python-dev 
-      - postgresql-9.6-postgis-2.3
-      - postgresql-9.6-postgis-2.3-scripts
-      - netcdf-bin
       - bc
       - curl
-      - udunits-bin 
+      - gdal-bin
+      - jags
+      - libcurl4-openssl-dev
+      - libgdal-dev
       - libgmp-dev
+      - libhdf5-dev
+      - liblapack-dev
+      - libnetcdf-dev
       - libproj-dev
+      - libudunits2-dev
+      - netcdf-bin
       - pandoc
+      - python-dev
       - tcl
       - tcl-dev
-      - libhdf5-dev
+      - udunits-bin
       # R package binaries from c2d4u
       - r-bioc-biocinstaller
       - r-cran-ape
@@ -58,35 +60,21 @@ addons:
       - r-cran-lme4
       - r-cran-matrixstats
       - r-cran-mcmcpack
-      - r-cran-mcmcpack
       - r-cran-raster
       - r-cran-rcpp
       - r-cran-rcurl
+      - r-cran-redland
       - r-cran-rncl
       - r-cran-roxygen2
       - r-cran-shiny
       - r-cran-testthat
       - r-cran-tidyverse
       - r-cran-xml
       - r-cran-xml2
-      # need to compile documentation
-      # BROKEN MISSING FONT
-      #- texinfo
-      #- texlive-latex-base
-      #- texlive-latex-recommended
-      #- texlive-latex-extra
-      #- texlive-fonts-recommended
-      #- texlive-fonts-extra
 
 
-## notifications should go to gitter
+## notifications should go to slack
 notifications:
-  webhooks:
-    urls:
-      - secure: "gL81TwDOcK/32Hxxl2BcY7pioyrtyV4y1f+D/vGEpQz8mYL+M+55tUkhHJF53779XSUdQdML/gsr8JZVtApJdLIBysFU67GVYXm1s7x/b8J61CkMfDgsmposEWK4NFYHfeIRj32ioeajrQ+RKi1I6chjzYT7gLyl70gtPelRJ4s="
-    on_success: always
-    on_failure: always
-    on_start: always
   slack:
     # Slack token created by Chris Black, 2018-02-17
     secure: "DHHSNmiCf71SLa/FFSqx9oOnJjJt2GHYk7NsFIBb9ZY10RvQtIPfaoNxkPjqu9HLyZWJSFtg/uNKKplEHc6W80NoXyqoTvwOxTPjMaViXaCNqsmzjjR/JaCWT/oWGXyAw0VX3S8cwuIexlKQGgZwJpIzoVOZqUrDrHI/O17kZoM="
@@ -96,7 +84,7 @@ notifications:
 
 ## list of services to be running
 services:
-  - postgresql
+  - docker
 
 install:
   - pushd $HOME
@@ -107,6 +95,14 @@ install:
   - popd
 
 before_script:
+  - sudo service postgresql stop
+  - docker run --detach --rm --name postgresql --publish 5432:5432 mdillon/postgis:9.6-alpine
+  - echo -n "Waiting for Postgres to start...";
+    until psql -U postgres -c 'select 1' >/dev/null 2>&1;
+      do echo -n ".";
+      sleep 1;
+    done;
+    echo " OK"
   - psql -q -o /dev/null -U postgres -c "CREATE ROLE BETY WITH LOGIN CREATEDB SUPERUSER CREATEROLE UNENCRYPTED PASSWORD 'bety'";
   - psql -q -o /dev/null -U postgres -c "CREATE DATABASE bety OWNER bety;"
   - curl -o bety.sql http://isda.ncsa.illinois.edu/~kooper/PEcAn/data/bety.sql
@@ -118,12 +114,20 @@ before_script:
 
 script:
   - set -e
-  # - scripts/build.sh --no-git --tests --name travis
   - echo 'Installing PEcAn packages'
-  - make
+  # TODO: Would be nice to add `-Otarget` after `-j` below,
+  # but not supported in Make 3.x.
+  # When Travis updates, check for Make 4 and add -O if available.
+  - make # TODO: add -j2 here after #1976 closed
   - echo 'Testing PEcAn packages'
-  - make test
-  - make document
+  - make -j2 test
+  - make -j2 document
+  # Skip modules with known broken checks
+  - mkdir -p .check/modules # only needed for `touch` below
+  - touch .check/modules/benchmark # delete when #1978 closed
+  - touch .check/modules/data.land # delete when #1977 closed
+  - touch .check/modules/rtm # delete when #1976 closed
+  - make -j2 check
   - echo 'Testing Integration'
   - ./tests/integration.sh travis
   - if [[ `git status -s` ]]; then 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,7 +6,103 @@ section for the next release.
 For more information about this file see also [Keep a Changelog](http://keepachangelog.com/) .
 
 
-## [Unreleased]
+## [1.6.0] - 2018-09-01
+
+### Fixes
+- Fixed issue #2064 which sends one met path to write.sa.config.
+- Fixed issue #1939 which corrects output time vector for FATES output
+- Update to read.output to look for and read only PEcAn formatted .nc output based on the pecan standard filename format of YYYY.nc.  Solves issues with models such as FATES and dvm-dos-tem where the original model output is also in .nc file format and was not ignored by read.output, causing errors with output parsing and plotting with Shiny. Removed deprecated function convert.outputs
+- PEcAn.data.atmosphere: 
+    - download.Geostreams is pickier about formatting start/end datess, for fewer surprises in result timestamps
+    - Fixed swapped lat/lon in met2CF.Geostreams
+    - download.GFDL now records reference date in time units field, as required by the CF met standard
+    - Reduced download.GFDL network load by not preloading dimension data
+    - Fixed spurious `No geonamesUsername set` warning by updating geonames package to development version
+- ED:
+    - Change all history parameter files to have zero storage respiration
+- missing_port_bety    
+- dataone_download.R:
+
+    - Added functionality that spoofs our user address to prevent authentication errors with downloading files via wget. 
+- Could not specify the port for BETY in config.php. Can now use `db_bety_port` to specify port.
+
+    - Added functionality that spoofs our user address to prevent authentication errors with downloading files via wget.
+
+- Data_Ingest_App:
+    - use `updateSelectizeInput` to populate `selectizeInput` with choices from BETYdb. This instantly loads the inputfields where other methods take minutes to load. 
+
+
+### Added
+
+- PEcAn.undertainty gains one new function (input.ens.gen) and three functions moved from PEcAn.utils (see "Changed" below)
+- IC workflow now has functionality to generate ensembles.
+- You can now generate ensembles for parameters and met separatly and using different methods. 
+- Soil process is now capable of reading in soil data from gSSURGO databse.
+- In modules/rtm new function foursail()  to interface with the 4SAIL Fortran code. To enable the use of 4SAIL with any version of PROSPECT (i.e. 4, 5, 5b, D) and custom soil/background reflectance inputs
+- Shiny/Dependency explorer 
+- Explore the interdependencies between pecan packages/functions.
+- From history you can now select an old run and show the curl command to re-execute this run. This only works with runs submitted through web interface right now.
+- Experimental support for docker (#1028)
+
+- dataone_download.R:
+  - Added progress messages to indicate that the function is working during longer downloads via PEcAn logger. 
+  - Store path to downloaded data as newdir_D1 so that the download app can call this path. 
+
+- shiny/Data-Ingest 
+  - Download data from DataONE to a temporary directory, display the contents of that directory, and then move files from the temporary directory to dbfiles
+  - Upload files from local machines (via drag and drop on some browsers), display files, and then move these files to a directory in dbfiles.
+  - Spinner displays when class "shiny-busy" is invoked during the dataONE download process. In this way, users can be sure that the app is live and is processing their request. 
+  - Users can now input the name of the destination directory that they wish to create within dbfiles. 
+  - Updated Travis.yml to include librdf0-dev so that it can download redland, datapack, and dataone. 
+  - Added Data-Ingest UI (inputs, dbfiles, and formats record UI and some basic server side functionality are online)
+  - Modularized input record, format record, and dbfiles record into shiny modules. This allows the app to be greatly simplified to two, single-page workflows. These functions can also be used "plug-and-play" style elsewhere in PEcAn shiny apps to load in data. 
+  - Replaced modularized input, format and dbfiles records with static "Ingest Workflow" page. On this page, the user can select either importing from dataONE or Uploading from local files. If creating a new format is necessary, the user can click "Create New Format" and a dropdown menu will walk them through this process. 
+  - Selected files now autofill name value in input record workflow
+  - Store inputs and formats in the global environment
+  - "Test BETY" button allows users create a record in BETY with `dbfile.input.insert`
+  - Added `input.format.vars` to query the BETYdb
+  - New File: `helper.R`
+  - New Function: `auto.name.directory` This function uses the format_name and the site_id for a given input to create a directory name in the style of other dbfiles names. 
+  - `Next Step` buttons progress workflow programmatically
+  - New formats-variables UI allows user to create a table of formats-variable records before completing the ingest process
+  - Two separate complete Ingest buttons are rendered at the end of the workflow to trigger actions specific to local upload or dataONE download workflows. These buttons are rendered programmatically depending on the state of the selectInputMethod radio button.
+  - Converted time inputs to properly merge startDate and startTime with EndDate and EndTime so they can be inserted into the start_date and end_date columns in BETYdb.
+  - Error handling introduced using `shinytoastr` package
+  - DESCRIPTION: `Depends`: PEcAn.visualization, shinytoastr, shinyWidgets, shinyjs
+
+- pecan/base/db
+  - New File: `input.format.vars.R`. This function registers the format and the (optional) formats_variables record using `db_merge_into`. 
+
+- `data.atmosphere`
+	- `check_met_input_file` -- Check that target met file conforms to PEcAn meteorology data standard.
+	- `get_cf_variables_table` -- Retrieve CF variables table as a `data.frame` 
+
+
+- docker:
+  - Added updated docker container builds
+    - Use docker.sh to create docker images
+    - Use release.sh to push released images to push to docker registry (hub.docker.com by default)
+  - Create pecan/depends docker image that holds all PEcAn dependencies
+    - Needs to build seperatly, not part of the docker.sh build process to speed things up
+    - Build using `(cd docker ; docker build -t pecan/depends:latest -f Dockerfile.depends .)`
+  - docker-compose.yml file to bring up full PEcAn stack using docker
+    - First time to start requires to install BETY database (see documentation)
+  - SIPNET docker image which works with PEcAn docker stack
+  - Data container that will download and install demo data in /data folder
+
+
+### Removed
+  - pecan.worldmap function no longer used, dropped from visualization package
+  - shiny/Data-Ingest/DESCRIPTION no longer `DEPENDS` on `shinyFiles` or `shinycssloaders`
+
+### Changed
+
+- Fixed Git instructions and remote execution instructions.
+- Five functions from PEcAn.utils functions have been moved to other packages. The versions in PEcAn.utils are deprecated, will not be updated with any new features, and will be removed in a future release.
+  - run.write.configs and runModule.run.write.configs have been moved to PEcAn.workflow 
+  - read.ensemble.output, get.ensemble.samples and write.ensemble.configs have been moved to PEcAn.uncertainty
+- Change the way packages are checked for and called in SHINY apps. DESCRIPTION files in SHINY apps are not the place to declare pacakge dpendencies.    
+
 
 ## [1.5.3] - 2018-05-15
 
@@ -30,10 +126,16 @@ For more information about this file see also [Keep a Changelog](http://keepacha
 - PEcAn now supports PFTs whose members are cultivars rather than species, and will automatically restrict the meta-analysis to matching records, e.g. runs with a PFT containing only Panicum virgatum 'Cave-In-Rock' will not use observations from Panicum virgatum 'Alamo', but a PFT containing the whole species will use observations from both. However, there is not yet any BETYdb interface to *create* cultivar-PFTs other than manual SQL.
 - New base package `PEcAn.workflow`, for functions used to perform the each major step of the analysis. These were previously scattered in other base packages.
 - Added PR review time estimate to PR template 
+- New set of `PEcAn.logger` functions similar to `stopifnot` to facilitate assertive programming: `severeifnot`, `errorifnot`, `warnifnot`, `infoifnot`, `debugifnot`
 - PEcAnRTM:
     - Exposed PROSPECT absorption coefficients and `gpm()` function ("generalized plate model"), facilitating experimentation with different absorption coefficients
     - Added `spectra` S3 class and methods for subsetting (e.g. `myspec[[400:700]]`), plotting (`plot()` and `matplot()`), and combining spectra by wavelength.
     - Added `resample` functions for quickly resampling spectra (and, more generally, vectors and functions) to different dimensions. 
+    - `EDR` API has been revised. Setup has been refactored from EDR via new `setup_edr` function, which relies on the ED utilities (see `PEcAn.ED2` below), and the `EDR` function now focuses only on execution. Also, added new `params2edr` function to make it easy to convert complex EDR parameters list to flat parameter vector required by `invert_bt` (or other optimization functions).
+- PEcAn.ED2:
+    - New set of utilities for working with ED meteorology and vegetation inputs, and the ED2IN file. Existing PEcAn code has been revised to use these utilities.
+- PEcAn.data.atmosphere:
+    - New utilities for efficiently downloading NARR time series using THREDDS/OpenDAP
 
 ### Removed
 - Removed deprecated copies of PEcAn.utils::SafeList, PEcAn.utils::listToXml (both moved to PEcAn.settings in v 1.5.2), and PEcAn.utils::fqdn (moved to PEcAn.remote in 1.5.2). This fixes the masses of deprecation warnings in otherwise normal run logs (#1719).
@@ -51,6 +153,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha
 - Replaced `rhdf5` library with `hdf5r`, a more modern alternative that is available on CRAN.
 - PEcAn.DB function `runModule.get.trait.data` has been moved to the new PEcAn.workflow package to avoid a circular package dependency between PEcAn.DB and PEcAn.settings.
 - Major documentation refactoring. The documentation names are now directly tied to the order in which they are rendered, and all `Rmd` files in all subdirectories of the documentation source are rendered by default. The overall structure of the documentation has been revised for clarity and cohesiveness.
+- Edited met2model.ED2 to not enforce leap years. 
 - Integrate demo 1 into basic user guide
 
 ## [1.5.2] - 2017-12-07

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -12,7 +12,7 @@ New functionality is typically directed toward modules to provide a slimmer PEcA
 Generally, new model should be added to the models folder and new modules should be added to the modules folder. 
 Exceptions include code that is reused in many models or modules and wrapper functions that call specific implementations in models; these can be placed in the core packages.
 
-If you are unsure of whether your contribution should be implemented as a model, module or part of PEcAn Core, you may visit [Gitter](https://gitter.im/PecanProject/pecan) or ask on the pecan-develop mailing list for advice.
+If you are unsure of whether your contribution should be implemented as a model, module or part of PEcAn Core, you may visit [Chat Room](https://join.slack.com/t/pecanproject/shared_invite/enQtMzkyODUyMjQyNTgzLTYyZTZiZWQ4NGE1YWU3YWIyMTVmZjEyYzA3OWJhYTZmOWQwMDkwZGU0Mjc4Nzk0NGYwYTIyM2RiZmMyNjg5MTE) or ask on the pecan-develop mailing list for advice.
 
 ## Creating Issues
 

diff --git a/Dockerfile b/Dockerfile