Merge branch 'release-1.7.0'

.gitignore

@@ -47,7 +47,6 @@ supervisord.conf
 codekit-config.json
 .vagrant/
 /deployment/puppet/.librarian/
-/deployment/puppet/Puppetfile.lock
 /refinery/celerybeat-schedule*
 /refinery/config/config.json
 /refinery/ui/production/

.travis.yml

@@ -1,20 +1,21 @@
 sudo: required
 
 language: python
+python:
+  - "2.7"
+
 cache:
   directories:
     - /home/travis/.cache/pip
     - /home/travis/.cypress/Cypress
 
-python:
-  - 2.7
-
 addons:
   postgresql: "10"
   apt:
     packages:
       - postgresql-10
       - postgresql-client-10
+      - libgconf-2-4  # for Cypress
 
 install:
   - node --version
@@ -54,7 +55,7 @@ before_script:
 # See http://www.stuartellis.eu/articles/erb/#running-erb-from-the-command-line
   - erb config/config.json.erb > config/config.json
   - python manage.py migrate --noinput --fake-initial
-  - npm install -g grunt-cli@0.1.13 bower@1.8.2 --progress false --quiet || ( cat npm-debug.log && false )
+  - npm install -g grunt-cli@0.1.13 bower@1.8.8 --progress false --quiet || ( cat npm-debug.log && false )
   - cd ui
   - npm install --progress false --quiet || ( cat npm-debug.log && false )
   - bower install --config.interactive=false --quiet

Vagrantfile

@@ -10,8 +10,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # please see the online documentation at vagrantup.com.
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "ubuntu/trusty64"
-  config.vm.box_version = "20181203.0.1"
+  config.vm.box = "ubuntu/xenial64"
+  config.vm.box_version = "20190514.0.1"
   config.vm.hostname = "refinery"
   # nic_type set to virtio to increase guest network performance (https://superuser.com/a/850389)
   config.vm.network "private_network", ip: "192.168.50.50", nic_type: "virtio"
@@ -37,7 +37,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision :puppet do |puppet|
     puppet.manifests_path = "deployment/puppet/manifests"
     puppet.manifest_file  = "site.pp"
-    puppet.hiera_config_path = "deployment/puppet/hiera.yaml"  # to avoid missing file warning
   end
 
   # workaround for services that start on boot before /vagrant is available

deployment/bootstrap.sh

@@ -8,15 +8,18 @@ export DEBIAN_FRONTEND=noninteractive
 # print commands and their expanded arguments
 set -x
 
-/usr/bin/apt-get clean
+/usr/bin/wget -q -P /var/cache/apt/archives https://apt.puppetlabs.com/puppet5-release-xenial.deb
+/usr/bin/dpkg -i /var/cache/apt/archives/puppet5-release-xenial.deb
 /usr/bin/apt-get -qq update
+/usr/bin/apt-get -q -y install git htop nmon puppet-agent ruby-dev tree
 /usr/bin/apt-get -y autoremove
+/usr/bin/apt-get clean
 
-/usr/bin/apt-get -q -y install git htop nmon ruby-dev tree
+PATH=/opt/puppetlabs/bin:$PATH
 
-/usr/bin/gem install librarian-puppet -v 2.2.3 --no-rdoc --no-ri
+/usr/bin/gem install librarian-puppet -v 3.0.0 --no-rdoc --no-ri
 
 cd /vagrant/deployment/puppet
-librarian-puppet config path /usr/share/puppet/modules --local
+librarian-puppet config path /opt/puppetlabs/puppet/modules --local
 librarian-puppet config tmp /tmp --local
 librarian-puppet install

deployment/puppet/Puppetfile

@@ -1,17 +1,13 @@
 forge "https://forgeapi.puppetlabs.com"
 
 mod 'puppet/nodejs', '2.1.0'
-mod 'puppetlabs-apache', '1.11.0'
-mod 'puppetlabs/apt', '2.3.0'
-mod 'puppetlabs/docker_platform', '2.2.1'
-mod 'puppetlabs/limits', '0.1.0'
-mod 'puppetlabs/lvm', '0.7.0'
-mod 'puppetlabs/postgresql', '4.9.0'
-mod 'puppetlabs/rabbitmq', '5.6.0'
-mod 'puppetlabs/stdlib', '4.13.1'
-mod 'puppet-archive', '1.3.0'
-mod 'saz/timezone', '3.2.0'
-mod 'stankevich/python', '1.9.8'
-mod 'thias/sysctl', '1.0.2'
+mod 'puppet/python', '3.0.1'
+mod 'puppet/rabbitmq', '9.0.0'
+mod 'puppetlabs-apache', '5.0.0'
+mod 'puppetlabs/docker', '3.5.0'
+mod 'puppetlabs/lvm', '1.3.0'
+mod 'puppetlabs/postgresql', '5.12.1'
+mod 'saz/timezone', '5.1.1'
+mod 'tpdownes/sysctl', '2.1.0'
 
 mod 'refinery/refinery', :path => "refinery"

deployment/puppet/Puppetfile.lock

@@ -0,0 +1,73 @@
+FORGE
+  remote: https://forgeapi.puppetlabs.com
+  specs:
+    camptocamp-systemd (2.6.0)
+      puppetlabs-inifile (< 4.0.0, >= 1.6.0)
+      puppetlabs-stdlib (< 7.0.0, >= 4.13.1)
+    puppet-archive (3.2.1)
+      puppetlabs-stdlib (< 6.0.0, >= 4.13.1)
+    puppet-nodejs (2.1.0)
+      puppetlabs-stdlib (< 5.0.0, >= 4.1.0)
+      treydock-gpg_key (< 1.0.0, >= 0.0.3)
+    puppet-python (3.0.1)
+      puppetlabs-stdlib (< 7.0.0, >= 4.19.0)
+      stahnma-epel (< 2.0.0, >= 1.2.2)
+    puppet-rabbitmq (9.0.0)
+      camptocamp-systemd (< 3.0.0, >= 2.1.0)
+      puppet-archive (< 4.0.0, >= 2.0.0)
+      puppetlabs-stdlib (< 6.0.0, >= 4.13.1)
+    puppetlabs-apache (5.0.0)
+      puppetlabs-concat (< 7.0.0, >= 2.2.1)
+      puppetlabs-stdlib (< 7.0.0, >= 4.13.1)
+    puppetlabs-apt (6.3.0)
+      puppetlabs-stdlib (< 6.0.0, >= 4.16.0)
+      puppetlabs-translate (< 2.0.0, >= 1.0.0)
+    puppetlabs-concat (5.3.0)
+      puppetlabs-stdlib (< 6.0.0, >= 4.13.1)
+      puppetlabs-translate (< 2.0.0, >= 1.0.0)
+    puppetlabs-docker (3.5.0)
+      puppetlabs-apt (< 7.0.0, >= 4.4.1)
+      puppetlabs-powershell (< 3.0.0, >= 2.1.4)
+      puppetlabs-reboot (< 3.0.0, >= 2.0.0)
+      puppetlabs-stdlib (< 6.0.0, >= 4.24.0)
+      puppetlabs-translate (< 2.0.0, >= 0.0.1)
+    puppetlabs-inifile (3.0.0)
+      puppetlabs-translate (< 2.0.0, >= 1.0.0)
+    puppetlabs-lvm (1.3.0)
+      puppetlabs-mount_core (< 2.0.0, >= 1.0.0)
+      puppetlabs-stdlib (< 7.0.0, >= 4.13.1)
+    puppetlabs-mount_core (1.0.3)
+    puppetlabs-postgresql (5.12.1)
+      puppetlabs-apt (< 7.0.0, >= 2.0.0)
+      puppetlabs-concat (< 6.0.0, >= 1.1.0)
+      puppetlabs-stdlib (< 6.0.0, >= 4.13.1)
+    puppetlabs-powershell (2.3.0)
+    puppetlabs-reboot (2.1.2)
+    puppetlabs-stdlib (4.25.1)
+    puppetlabs-translate (1.2.0)
+    saz-timezone (5.1.1)
+      puppetlabs-stdlib (< 6.0.0, >= 2.6.0)
+      stm-debconf (< 3.0.0, >= 2.0.0)
+    stahnma-epel (1.3.1)
+      puppetlabs-stdlib (>= 3.0.0)
+    stm-debconf (2.3.0)
+    tpdownes-sysctl (2.1.0)
+    treydock-gpg_key (0.1.0)
+
+PATH
+  remote: refinery
+  specs:
+    refinery-refinery (0.0.1)
+
+DEPENDENCIES
+  puppet-nodejs (= 2.1.0)
+  puppet-python (= 3.0.1)
+  puppet-rabbitmq (= 9.0.0)
+  puppetlabs-apache (= 5.0.0)
+  puppetlabs-docker (= 3.5.0)
+  puppetlabs-lvm (= 1.3.0)
+  puppetlabs-postgresql (= 5.12.1)
+  refinery-refinery (>= 0)
+  saz-timezone (= 5.1.1)
+  tpdownes-sysctl (= 2.1.0)
+

deployment/puppet/hiera.yaml

@@ -1 +1 @@
-# placeholder to avoid a missing file warning in Puppet
+version: 5

deployment/puppet/refinery/manifests/apache2.pp

@@ -16,7 +16,7 @@
   # processing of metadata during data set import
   $timeout = 180  # seconds
 
-  class { '::apache':
+  class { 'apache':
     default_mods           => false,  # to allow declaration of ::apache::mod::reqtimeout below
     # recommended for use with AWS ELB
     mpm_module             => 'worker',
@@ -31,28 +31,28 @@
     },
   }
 
-  class { '::apache::mod::wsgi':
+  class { 'apache::mod::wsgi':
     mod_path     => 'mod_wsgi.so',
     package_name => 'libapache2-mod-wsgi',
   }
 
   # recommended for use with AWS ELB to avoid HTTP 408 errors
-  class { '::apache::mod::reqtimeout':
+  class { 'apache::mod::reqtimeout':
     timeouts => [
       "header=${ $timeout + 5 }-${ $timeout + 25 },MinRate=500",
       "body=${ $timeout + 5 },MinRate=500",
     ],
   }
 
-  class { '::apache::mod::dir': }  # to allow ELB health checks using default vhost
+  class { 'apache::mod::dir': }  # to allow ELB health checks using default vhost
 
   # recommended for use with AWS ELB
-  ::apache::custom_config { 'no-acceptfilter':
+  apache::custom_config { 'no-acceptfilter':
     content => "AcceptFilter http none\nAcceptFilter https none",
   }
 
   # log Django messages only
-  ::apache::custom_config { 'error-log-format':
+  apache::custom_config { 'error-log-format':
     content => 'ErrorLogFormat "%M"',
   }
 
@@ -67,6 +67,8 @@
         rewrite_rule => ['^.*$ https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L]'],
       },
     ]
+  } else {
+    $rewrites = []
   }
 
   $common_aliases = [
@@ -118,26 +120,27 @@
     default => undef,
   }
 
-  ::apache::vhost { 'refinery':
-    servername                  => $site_url,
-    vhost_name                  => '*',
-    port                        => 80,
-    docroot                     => false,
-    manage_docroot              => false,
-    rewrites                    => $rewrites,
-    wsgi_script_aliases         => { '/' => "${django_root}/config/wsgi_${conf_mode}.py" },
-    wsgi_daemon_process         => 'refinery',
-    wsgi_daemon_process_options => {
-      user        => $app_user,
-      group       => $app_group,
-      python-path => "${django_root}:${virtualenv}/lib/python2.7/site-packages",
+  apache::vhost { 'refinery':
+    servername              => $site_url,
+    vhost_name              => '*',
+    port                    => 80,
+    docroot                 => false,
+    manage_docroot          => false,
+    rewrites                => $rewrites,
+    wsgi_script_aliases     => { '/' => "${django_root}/config/wsgi_${conf_mode}.py" },
+    wsgi_daemon_process     => {
+      'refinery' => {
+        'user'        => $app_user,
+        'group'       => $app_group,
+        'python-path' => "${django_root}:${virtualenv}/lib/python2.7/site-packages",
+      }
     },
-    wsgi_process_group          => 'refinery',
-    wsgi_pass_authorization     => 'On',
-    access_log_file             => 'refinery_access.log',
-    access_log_format           => $access_log_format,
-    error_log_file              => 'refinery_error.log',
-    aliases                     => $aliases,
-    directories                 => $directories,
+    wsgi_process_group      => 'refinery',
+    wsgi_pass_authorization => 'On',
+    access_log_file         => 'refinery_access.log',
+    access_log_format       => $access_log_format,
+    error_log_file          => 'refinery_error.log',
+    aliases                 => $aliases,
+    directories             => $directories,
   }
 }

deployment/puppet/refinery/manifests/init.pp

@@ -7,7 +7,7 @@
   $virtualenv             = $refinery::params::virtualenv,
   $docker_host            = $refinery::params::docker_host,
 ) inherits refinery::params {
-  sysctl { 'vm.swappiness': value => '10' }  # for better performance
+  sysctl::configuration { 'vm.swappiness': value => '10' }  # for better performance
 
   class { 'timezone':  # to make logs easier to read
     timezone => 'America/New_York',
@@ -25,6 +25,7 @@
   # workaround for CloudFront error 523 Origin Unreachable for https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
   class { '::rabbitmq':
     package_gpg_key  => 'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc',
+    manage_python    => false
   }
 
   package { 'memcached': }

deployment/puppet/refinery/manifests/postgresql.pp

@@ -7,7 +7,7 @@
   $rds_endpoint_address   = $refinery::params::rds_endpoint_address,
 ) inherits refinery::params {
   $server_version = '10'
-  $package_version = "${server_version}.7-1.pgdg14.04+1"
+  $package_version = "${server_version}.9-1.pgdg16.04+1"
 
   if $deployment_platform == 'aws' {
     $rds_settings = {
@@ -72,7 +72,7 @@
       # to make remote connections via SSH tunnel easier
       ipv4acls => ["host all ${db_user} 127.0.0.1/32 trust"],
     }
-
+    ->
     postgresql::server::role { $db_user:
       createdb => true,  # to allow automated testing
     }

deployment/puppet/refinery/manifests/solr.pp

@@ -14,7 +14,7 @@
   $solr_data_set_manager_data = "${solr_data_dir}/data_set_manager"
 
   package { 'java':
-    name => 'openjdk-7-jdk',
+    name => 'openjdk-8-jdk',
   }
 
   file { '/opt':

deployment/puppet/refinery/manifests/ui.pp

@@ -41,7 +41,7 @@
   }
   ->
   package {
-    'bower': ensure => '1.8.2', provider => 'npm';
+    'bower': ensure => '1.8.8', provider => 'npm';
     'grunt-cli': ensure => '0.1.13', provider => 'npm';
   }
   ->

deployment/requirements.txt

@@ -1,7 +1,7 @@
 Fabric==1.9.0
 ecdsa==0.11
 fabtools==0.19.0
-paramiko==1.14.0
+paramiko==2.5.0
 pycrypto==2.6.1
 awscli==1.16.108
 aws-shell==0.2.1

deployment/terraform/modules/ec2/main.tf

@@ -124,8 +124,8 @@ locals {
 
 resource "aws_instance" "app_server" {
   count                  = "${var.instance_count}"
-  # ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-20181203
-  ami                    = "ami-03597b1b84c02cf7b"
+  # ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20190514
+  ami                    = "ami-07b4156579ea1d7ba"
   instance_type          = "${var.instance_type}"
   key_name               = "${var.key_pair_name}"
   monitoring             = true
@@ -156,10 +156,14 @@ set -x
 export PROJECT_ROOT=/srv/refinery-platform
 
 # install dependencies
-/usr/bin/apt-get clean
+/usr/bin/wget -q -P /var/cache/apt/archives https://apt.puppetlabs.com/puppet5-release-xenial.deb
+/usr/bin/dpkg -i /var/cache/apt/archives/puppet5-release-xenial.deb
 /usr/bin/apt-get -qq update
+/usr/bin/apt-get -qq -y install git htop jq nmon puppet-agent ruby-dev tree
 /usr/bin/apt-get -y autoremove
-/usr/bin/apt-get -qq -y install git htop jq nmon puppet ruby-dev tree
+/usr/bin/apt-get clean
+
+PATH=/opt/puppetlabs/bin:$PATH
 
 # add extra SSH keys from Github
 for USERNAME in ${join(" ", var.ssh_users)}; do
@@ -173,11 +177,11 @@ su -c "git clone https://github.com/refinery-platform/refinery-platform.git $PRO
 su -c "cd $PROJECT_ROOT && /usr/bin/git checkout -q ${var.git_commit}" ubuntu
 
 # configure librarian-puppet
-/usr/bin/gem install librarian-puppet -v 2.2.3 --no-rdoc --no-ri
+/usr/bin/gem install librarian-puppet -v 3.0.0 --no-rdoc --no-ri
 cd $PROJECT_ROOT/deployment/puppet
 # need to set $HOME: https://github.com/rodjek/librarian-puppet/issues/258
 export HOME=/root
-/usr/local/bin/librarian-puppet config path /usr/share/puppet/modules --local
+/usr/local/bin/librarian-puppet config path /opt/puppetlabs/puppet/modules --local
 /usr/local/bin/librarian-puppet config tmp /tmp --local
 /usr/local/bin/librarian-puppet install
 
@@ -213,7 +217,7 @@ export FACTER_REFINERY_WELCOME_EMAIL_MESSAGE="${var.refinery_welcome_email_messa
 export FACTER_USER_FILES_COLUMNS="${var.refinery_user_files_columns}"
 
 # run Puppet
-/usr/bin/puppet apply $PROJECT_ROOT/deployment/puppet/manifests/site.pp
+/opt/puppetlabs/bin/puppet apply $PROJECT_ROOT/deployment/puppet/manifests/site.pp
 EOF
 }