Merge branch 'release-1.6.7'

# Resolved Conflicts:
#	CHANGELOG.md
#	refinery/core/tests.py
#	version.txt

.travis.yml

@@ -1,10 +1,6 @@
 sudo: required
 
 language: python
-addons:
-  apt:
-    packages:
-      - xvfb
 cache:
   directories:
     - /home/travis/.cache/pip
@@ -13,10 +9,12 @@ cache:
 python:
   - 2.7
 
-services:
-  - postgresql
-  - neo4j
-  - docker
+addons:
+  postgresql: "10"
+  apt:
+    packages:
+      - postgresql-10
+      - postgresql-client-10
 
 install:
   - node --version
@@ -40,13 +38,18 @@ env:
     - secure: oGdTYh3rnPdG7qVGYq3TgVESypir1btO2kmafumtKR6o+FYmMtb09pe+HtiT6Qfv4pDwnYZ2RTYqsTY4h6POigU5j7qQPSsZ5WkhHt9ybFyt/vfo2B95GaAIFt+8LVjTX5TItyw0/PPUCCUluQ1n8k/NCtSfMb0kIT/CuLrg7N8=
 
 before_script:
+  # PostgreSQL-10 is still not fully supported by Travis.
+  # See: https://github.com/travis-ci/travis-ci/issues/8537#issuecomment-416467506
+  # Use default postgres port
+  - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/10/main/postgresql.conf
+  # Use postgres 9.6 auth config:
+  - sudo cp /etc/postgresql/{9.6,10}/main/pg_hba.conf
+  - sudo service postgresql restart
+
   # Vagrant is not used on travis: This user is just for the database.
   - createuser --createdb --no-superuser --no-createrole vagrant
   - createdb -O vagrant refinery
 
-  - sudo mkdir /data # In vagrant, this is created by puppet; In production, this is an EBS mount.
-  - sudo chown $USER /data
-
   - cd refinery
 # See http://www.stuartellis.eu/articles/erb/#running-erb-from-the-command-line
   - erb config/config.json.erb > config/config.json
@@ -65,6 +68,7 @@ before_script:
   - pushd ui && grunt make && popd
   - python manage.py collectstatic --noinput -v 0
   - python manage.py runserver --insecure --settings config.settings.dev 2>&1 | grep -v 200 &
+
 script:
   - set -e # Any error will cause travis to exit early and report a failure.
 

README.md

@@ -10,7 +10,7 @@
 ## Installing and Launching for Development
 
 ### Prerequisites
-* Install [Git][gi] (2.3.2+), [Vagrant][va] (1.8.1+) and [Virtualbox][vb] (5.0.16+)
+* Install [Git][gi] (2.19.0+), [Vagrant][va] (2.2.0+) and [Virtualbox][vb] (5.2.20+)
 * [Add SSH key to your GitHub account](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/)
 * Note: this procedure has only been tested on local development machines running OS X 10.10+
 

deployment/aws-config/config.yaml.erb

@@ -4,12 +4,10 @@
 # The configuration is stored in two files: config.yaml and tags.yaml
 #
 # Neither of these files exist to begin with but `config.yaml` is required
-# Copy `config.yaml.template` to `config.yaml` and edit
 
-# RDS_SNAPSHOT and DATA_SNAPSHOT are only used when restoring a Refinery Platform stack
-# from the EBS and RDS snapshots of a previously created stack
-# If you don't specify these then fresh RDS instances and data volumes will be created
-# RDS_SNAPSHOT: refinery-web-XXX
+# DATA_SNAPSHOT is only used when restoring a Refinery Platform stack from the
+# EBS snapshot of a previously created stack
+# If you don't specify this then fresh data volume will be created
 # DATA_SNAPSHOT: snap-XXX
 
 # The password for the Django admin account.
@@ -37,8 +35,13 @@ STACK_NAME: "<%= tf.s3_bucket_name_base %>"
 COGNITO_IDENTITY_POOL_ID: "<%= tf.identity_pool_id %>"
 VPC_ID: "<%= tf.vpc_id %>"
 PUBLIC_SUBNET_ID: "<%= tf.public_subnet_id %>"
-RDS_DB_SUBNET_GROUP_NAME: "<%= tf.rds_db_subnet_group_name %>"
 REFINERY_DOCKER_HOST: "<%= tf.docker_hostname %>"
+ELB_SECURITY_GROUP_ID: "<%= tf.elb_security_group_id %>"
+APP_SERVER_SECURITY_GROUP_ID: "<%= tf.app_server_security_group_id %>"
+APP_SERVER_PROFILE_ID: "<%= tf.app_server_profile_id %>"
+RDS_ENDPOINT_ADDRESS: "<%= tf.db_hostname %>"
+RDS_SUPERUSER_PASSWORD: ""  # get from Terraform state file
+IAM_SMTP_USER: "<%= tf.iam_smtp_user %>"
 
 # Prefix for all S3 bucket names (must not be shared among stacks)
 # Constraint: DNS-compliant without periods (".")
@@ -67,29 +70,22 @@ DATA_VOLUME_TYPE: "st1"
 # The name of the EC2 key pair to use for SSH (must already exist in the AWS account)
 KEY_NAME: "id_rsa"
 
-# The name of the RDS instance (if not set, the stack name will be used by default)
-# RDS_NAME: rds-refinery-default
-
 # HTML-safe item to be displayed to the right of the `About` link in the navbar
 REFINERY_CUSTOM_NAVBAR_ITEM: '<a href="http://example.org/">Sample Entry</a>'
 
 REFINERY_WELCOME_EMAIL_MESSAGE: "Please fill out your user profile"
 REFINERY_WELCOME_EMAIL_SUBJECT: "Welcome to Refinery"
 
 REFINERY_GOOGLE_ANALYTICS_ID: ""
+REFINERY_GOOGLE_RECAPTCHA_SITE_KEY: ""
+REFINERY_GOOGLE_RECAPTCHA_SECRET_KEY: ""
 
 REFINERY_URL_SCHEME: "http"
 
 USER_FILES_COLUMNS: "name,filetype,sample_name,organism,technology,genotype,cell_type,antibody,experimenter,date_submitted"
 
 # == Rarely changed ==
 
-# Password for the RDS superuser account
-RDS_SUPERUSER_PASSWORD: "mypassword"
-
-# Name of the PostgreSQL role to create
-RDS_ROLE: "refinery"
-
 #List of github usernames to allow SSH access into the EC2 instance
 #SSH_USERS: <username_0> <username_1> ...
 

deployment/aws.sh

@@ -14,7 +14,7 @@ env
 GIT_BRANCH=${GIT_BRANCH:-develop}
 
 /usr/bin/apt-get -q -y install htop
-/usr/bin/apt-get -q -y install awscli jq postgresql-client-9.3
+/usr/bin/apt-get -q -y install awscli jq
 
 printf '%s' "${CONFIG_YAML}" | base64 -d > /home/ubuntu/config.yaml
 printf '%s' "${CONFIG_JSON}" | base64 -d > /home/ubuntu/config.json
@@ -54,18 +54,6 @@ HOME=/home/ubuntu sh /srv/refinery-platform/deployment/bin/fetch-github-ssh-keys
 # Tag the attached root volume
 sh /srv/refinery-platform/deployment/bin/fix-untagged-volumes
 
-# CloudFormation should have set the endpoint
-# (DNS address and port) for our PostgreSQL RDS.
-# Copy them so that puppet/facter can use it.
-: ${RDS_ENDPOINT_ADDRESS?RDS_ENDPOINT_ADDRESS must be set}
-: ${RDS_ENDPOINT_PORT?RDS_ENDPOINT_PORT must be set}
-
-# FACTER environment variables become facts for puppet;
-# see https://puppetlabs.com/blog/facter-part-1-facter-101
-export FACTER_RDS_HOST=$RDS_ENDPOINT_ADDRESS
-export FACTER_RDS_PORT=$RDS_ENDPOINT_PORT
-export FACTER_RDS_ROLE="$RDS_ROLE"
-
 # Create SMTP credentials and
 # place them in (facter) environment variables.
 . bin/create-smtp-credentials
@@ -78,16 +66,6 @@ export FACTER_EMAIL_HOST_PASSWORD="$EMAIL_HOST_PASSWORD"
 export FACTER_SITE_URL="$SITE_URL"
 export FACTER_SITE_NAME="$SITE_NAME"
 
-# Create RDS user and database here, instead of using puppet
-# (because drj couldn't work out how to do it in puppet)
-export FACTER_RDS_ROLE_PASSWORD=password
-# Already set by earlier part of userdata
-export RDS_SUPERUSER_PASSWORD
-export RDS_ENDPOINT_ADDRESS
-export RDS_ENDPOINT_PORT
-PASSWORD=$FACTER_RDS_ROLE_PASSWORD bin/ensure-postgresql-role
-bin/ensure-postgresql-database
-
 sudo su -c '/usr/local/bin/librarian-puppet install' ubuntu
 
 /usr/bin/puppet apply --modulepath=/srv/refinery-platform/deployment/modules /srv/refinery-platform/deployment/manifests/aws.pp

deployment/manifests/aws.pp

@@ -1,5 +1,8 @@
 $app_user = 'ubuntu'
 $app_group = $app_user
+$db_name = 'refinery'
+$db_user = $db_name
+$db_user_password = fqdn_rand_string(8)  # random alphanumeric string of length eight
 $virtualenv = "/home/${app_user}/.virtualenvs/refinery-platform"
 $project_root = '/srv/refinery-platform'
 $deployment_root = "${project_root}/deployment"
@@ -21,7 +24,10 @@
 # solr is in the /data volume on AWS:
 $solr_data_set_manager_data = "${data_dir}/solr/data_set_manager"
 $solr_core_data = "${data_dir}/solr/core"
-$django_docker_engine_data_dir = "${data_dir}/django-docker-engine-data"
+$django_docker_engine_mem_limit_mb = 3584
+# Based on t2.medium (specified in docker_host/main.tf).
+# 0.5GB is probably more than enough for everything non-docker.
+# (4GB - 0.5GB) * 1024MB/GB = 3584MB
 
 $deployment_platform = 'aws'
 

deployment/manifests/default.pp

@@ -1,5 +1,7 @@
 $app_user = 'vagrant'
 $app_group = $app_user
+$db_name = 'refinery'
+$db_user = $app_user
 $virtualenv = "/home/${app_user}/.virtualenvs/refinery-platform"
 $site_name = 'Refinery'
 $site_url = '192.168.50.50:8000'
@@ -15,8 +17,7 @@
 $conf_mode = 'dev'
 $django_settings_module = "config.settings.${conf_mode}"
 $ui_app_root = "${django_root}/ui"
-$data_dir = '/data'
-$django_docker_engine_data_dir = "${data_dir}/django-docker-engine-data"
+$django_docker_engine_mem_limit_mb = 20
 $docker_host = "tcp://127.0.0.1:2375"
 
 # to make logs easier to read
@@ -33,20 +34,6 @@
   require     => Exec['create_guest'],
 }
 
-# Django-docker-engine needs a place for ephemeral data.
-# In production, this is a separate EBS mount, so we don't need to create it locally.
-file { $::data_dir:
-  ensure => directory,
-  owner  => $app_user,
-  group  => $app_group
-}
-->
-file { $::django_docker_engine_data_dir:
-    ensure => directory,
-    owner  => $app_user,
-    group  => $app_group,
-    mode   => '0755',
-}
 
 # See code in refinery-modules/refinery/...
 include refinery

deployment/refinery-modules/refinery/manifests/aws.pp

@@ -69,13 +69,6 @@
   group => "$app_user",
   mode => "0755",
 }
-->
-file { $::django_docker_engine_data_dir:
-    ensure => directory,
-    owner => "$app_user",
-    group => "$app_user",
-    mode => "0755",
-}
 
 exec { "generate_superuser_json":
   command     => "${virtualenv}/bin/python /srv/refinery-platform/deployment/bin/generate-superuser > /srv/refinery-platform/refinery/core/fixtures/superuser.json.new",

deployment/refinery-modules/refinery/manifests/init.pp

@@ -72,12 +72,14 @@
   group       => $app_group,
 }
 ->
-
 file { "/opt":
   ensure => directory,
 }
 
-include '::rabbitmq'
+# workaround for CloudFront error 523 Origin Unreachable for https://www.rabbitmq.com/rabbitmq-release-signing-key.asc
+class { '::rabbitmq':
+  package_gpg_key  => 'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc',
+}
 
 class ui {
   apt::source { 'nodejs':