Merge pull request #726 from ndlib/docker-compose-rails-with-sync

Add support for developing the rails app within Docker

.gitattributes

@@ -0,0 +1,2 @@
+* text=auto
+*.sh text eol=lf

.gitignore

@@ -34,3 +34,4 @@ spec/jetty_generator
 .tags
 .byebug_history
 .DS_Store
+.docker-sync

README.md

@@ -6,8 +6,9 @@ This application will be deployed to [curate.nd.edu](http://curate.nd.edu).
 
 Its primary function is to be a self-deposit interface for our institutional digital repository.
 
-## Installation Notes
+Note: If using Docker, see README_DOCKER.md for instructions.
 
+## Installation Notes
 Installing the clamav gem on OS X is a trying process. It can be safely excluded
 from your development environment
 
@@ -18,12 +19,6 @@ bundle install --without headless
 ## Testing
 
 ### 1. Start dependencies
-#### With Docker
-```console
-docker-compose -f docker-compose-test.yml up -d
-```
-
-#### Without Docker
 ```console
 bundle exec rake curatend:jetty:start
 ```
@@ -53,16 +48,6 @@ Certain curation concern types are never created interactively:
 ### 1. Start dependencies
 Before you start the web-server you'll need to make sure Fedora and SOLR are running.
 
-#### With Docker
-
-You can run MySQL, Fedora and Solr via Docker:
-
-```console
-docker-compose up -d
-```
-
-#### Without Docker
-
 Start Fedora and SOLR via jetty:
 
 ```console
@@ -77,15 +62,13 @@ mysql.server start
 
 ### 2. Initialize the database
 Load the database schema into MySQL:
-
 ```console
 bundle exec rake db:schema:load
 ```
 
-To seed database with test data (skip this step when using Docker):
-
+To seed database with test data:
 ```console
-bundle exec db:seed:dev
+bundle exec rake db:seed:dev
 ```
 
 ### 3. Start Rails
@@ -102,35 +85,6 @@ If you don't need SSL, use the following command:
 bundle exec rails server
 ```
 
-## Rebuilding curate-jetty Docker image
-
-To rebuild the Docker image for running jetty, use the following command:
-
-```console
-docker build . -t ndlib/curate-jetty
-```
-
-To push your image to Dockerhub:
-
-```console
-docker login
-docker push ndlib/curate-jetty
-```
-
-To rebuild the image with pre-generated seed data:
-```console
-# First reset to the base image
-docker stop curate-jetty && docker rm curate-jetty
-docker run -p 8983:8983 -d --name curate-jetty -t ndlib/curate-jetty
-# Run seed scripts from the project root directory
-bundle exec rake db:schema:load db:seed:dev
-# Commit these changes to your image
-docker commit curate-jetty ndlib/curate-jetty-devseed
-# Push it up as the dev seed image
-docker login
-docker push ndlib/curate-jetty-devseed
-```
-
 ## Release Documentation
 
 See the [release documentation](https://docs.google.com/a/nd.edu/document/d/16weRctSzt8Iw2y55nwOKPBSgGDO_4lgti3CxaW3P2pc/edit?usp=sharing) for building a release.

README_DOCKER.md

@@ -0,0 +1,55 @@
+# Developing CurateND with Docker
+
+## Installation Notes
+Install Docker: https://www.docker.com/
+
+Install docker-sync: http://docker-sync.io/
+
+Start the stack:
+```console
+docker-sync-stack start
+```
+
+## Testing
+To execute the full test suite, run
+```console
+docker-compose exec rails bundle exec rake
+```
+To run individual specs, run
+```console
+docker-compose exec rails bundle exec rspec spec/path/to/file_spec.rb
+```
+
+## Debugging the application
+```console
+docker-compose exec rails bundle exec byebug --remote localhost:9876
+```
+
+## Rebuilding curate-jetty Docker image
+
+To rebuild the Docker image for running jetty, use the following command:
+
+```console
+docker build . -t ndlib/curate-jetty -f docker/Dockerfile.jetty
+```
+
+To push your image to Dockerhub:
+
+```console
+docker login
+docker push ndlib/curate-jetty
+```
+
+To rebuild the image with pre-generated seed data:
+```console
+# First reset to the base image
+docker stop curate-jetty && docker rm curate-jetty
+docker run -p 8983:8983 -d --name curate-jetty -t ndlib/curate-jetty
+# Run seed scripts from the project root directory
+bundle exec rake db:schema:load db:seed:dev
+# Commit these changes to your image
+docker commit curate-jetty ndlib/curate-jetty-devseed
+# Push it up as the dev seed image
+docker login
+docker push ndlib/curate-jetty-devseed
+```

config/database.yml

@@ -11,14 +11,15 @@
   ].detect { |f| f && File.exist?(f) }
 
   port = ENV["BOXEN_MYSQL_PORT"] || "3306"
+  host = ENV["MYSQL_HOST"] || "127.0.0.1"
 %>
 
 mysql_connection: &mysql_connection
 <% if socket %>
   host: localhost
   socket: <%= socket %>
 <% else %>
-  host: 127.0.0.1
+  host: <%= host %>
   port: <%= port %>
 <% end %>
 
@@ -37,7 +38,6 @@ local_user: &local_user
 development: &development
   <<: *local_user
   database: curate_nd_development
-  host:     127.0.0.1
   timeout: 5000
 
 development_remote_purl_database:

config/fedora.yml

@@ -2,14 +2,19 @@
 # FYI: This file is replaced by a copy for deploys
 # to ci, pre_production, and production.
 #
+<%
+  port = ENV["FEDORA_PORT"] || "8983"
+  host = ENV["FEDORA_HOST"] || "127.0.0.1"
+%>
+
 development:
   user: fedoraAdmin
   password: fedoraAdmin
-  url: http://127.0.0.1:8983/fedora
+  url: <%= "http://#{host}:#{port}/fedora" %>
 test: &TEST
   user: fedoraAdmin
   password: fedoraAdmin
-  url: <%= "http://127.0.0.1:#{ENV['TEST_JETTY_PORT'] || 8983}/fedora-test" %>
+  url: <%= "http://#{host}:#{port}/fedora-test" %>
 ci:
   <<: *TEST
 production:

config/initializers/byebug.rb

@@ -0,0 +1,7 @@
+if Rails.env.development?
+  require 'byebug/core'
+  #Byebug.wait_connection = true
+  port = ENV.fetch("BYEBUG_SERVER_PORT", 9876).to_i
+  print "Starting byebug server for remote debugging on port #{port}\n"
+  Byebug.start_server 'localhost', port
+end

config/solr.yml

@@ -5,10 +5,15 @@
 
 # This is a sample config file that does not have multiple solr instances. You will also need to be sure to
 # edit the fedora.yml file to match the solr URL for active-fedora.
+<%
+  port = ENV["SOLR_PORT"] || "8983"
+  host = ENV["SOLR_HOST"] || "127.0.0.1"
+%>
+
 development:
-  url: http://localhost:8983/solr/development
+  url: <%= "http://#{host}:#{port}/solr/development" %>
 test: &TEST
-  url: <%= "http://127.0.0.1:#{ENV['TEST_JETTY_PORT'] || 8983}/solr/test" %>
+  url: <%= "http://#{host}:#{port}/solr/test" %>
 cucumber:
   <<: *TEST
 ci:

docker-compose.yml

@@ -1,15 +1,48 @@
 version: '3'
 services:
   mysql:
-    image: mariadb
+    build:
+      context: ./docker/
+      dockerfile: Dockerfile.mariadb
     ports:
       - "3306:3306"
     environment:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_DATABASE: curate_nd_development
     volumes:
       - ./docker/mysql-utf8.cnf:/etc/mysql/conf.d/mysql-utf8.cnf
+    healthcheck:
+        interval: 30s
+        timeout: 10s
+        retries: 5
   jetty:
     image: ndlib/curate-jetty-devseed
     ports:
       - "8983:8983"
+  rails:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.rails
+    command: bash docker/rails_entry.sh
+    environment:
+      BUNDLE_PATH: "/bundle"
+      FEDORA_HOST: jetty
+      FEDORA_PORT: 8983
+      MYSQL_HOST: mysql
+      SOLR_HOST: jetty
+      SOLR_PORT: 8983
+      # Need to pass the user running docker into the container so that
+      # config/admin_usernames.yml pulls in the user as an admin. This is
+      # to replicate existing behavior on OSX and may not work correctly on
+      # another OS
+      USER: #{USER}
+    ports:
+      - "3000:3000"
+    volumes:
+      - curate_nd_rails_sync:/project_root:nocopy
+    depends_on:
+      - mysql
+      - jetty
+volumes:
+  curate_nd_rails_sync:
+    external: true

docker-sync.yml

@@ -0,0 +1,8 @@
+version: "2"
+
+options:
+  verbose: true
+syncs:
+  curate_nd_rails_sync:
+    src: '.'
+    sync_excludes: ['.git', 'tmp', 'jetty', 'vendor/bundle', 'log']

docker/Dockerfile.mariadb

@@ -0,0 +1,5 @@
+FROM mariadb
+
+COPY mysql-healthcheck /usr/local/bin/
+
+HEALTHCHECK CMD ["mysql-healthcheck"]

docker/Dockerfile.rails

@@ -0,0 +1,26 @@
+FROM ruby:2.1.10
+
+RUN apt-get update -qq && apt-get install -y build-essential libpq-dev nodejs unzip
+
+# Install Fits (which requires java)
+RUN apt-get install -y default-jre
+RUN mkdir /fits
+WORKDIR /fits
+RUN wget https://projects.iq.harvard.edu/files/fits/files/fits-0.6.2.zip
+RUN unzip fits-0.6.2.zip
+RUN chmod 755 /fits/fits-0.6.2/fits.sh
+ENV PATH="/fits/fits-0.6.2:${PATH}"
+
+# Put the installed gems outside of project_root so that the sync volume won't interfere
+RUN mkdir /bundle
+COPY Gemfile /bundle
+COPY Gemfile.lock /bundle
+WORKDIR /bundle
+RUN bundle install  --without headless --path /bundle
+
+RUN mkdir /project_root
+WORKDIR /project_root
+COPY . /project_root
+
+# Gemfile.lock may have changed after bundling, copy it back into the project_root
+RUN cp /bundle/Gemfile.lock /project_root/

docker/byebug.sh

@@ -0,0 +1 @@
+docker-compose exec rails bundle exec byebug --remote localhost:9876

docker/mysql-healthcheck

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eo pipefail
+
+if [ "$MYSQL_RANDOM_ROOT_PASSWORD" ] && [ -z "$MYSQL_USER" ] && [ -z "$MYSQL_PASSWORD" ]; then
+	# there's no way we can guess what the random MySQL password was
+	echo >&2 'healthcheck error: cannot determine random root password (and MYSQL_USER and MYSQL_PASSWORD were not set)'
+	exit 0
+fi
+
+host="$(hostname --ip-address || echo '127.0.0.1')"
+user="${MYSQL_USER:-root}"
+export MYSQL_PWD="${MYSQL_PASSWORD:-$MYSQL_ROOT_PASSWORD}"
+
+args=(
+	# force mysql to not use the local "mysqld.sock" (test "external" connectibility)
+	-h"$host"
+	-u"$user"
+	--silent
+)
+
+if select="$(echo 'SELECT 1' | mysql "${args[@]}")" && [ "$select" = '1' ]; then
+	exit 0
+fi
+
+exit 1

docker/rails_entry.sh

@@ -0,0 +1,10 @@
+# All the things that will execute when starting the rails service
+# Copy the generated lock file back into the project root. This is to sync this change back to the
+# application after the container has mounted the sync volume
+cp /bundle/Gemfile.lock ./
+bash docker/wait-for-it.sh mysql:3306
+bundle exec rake db:schema:load
+# If we find people are ok with just starting a shell in the container and running the db create/load before running specs, then we can remove this
+RAILS_ENV=test bundle exec rake db:create db:schema:load
+
+exec bundle exec thin start -p 3000 --ssl --ssl-key-file dev_server_keys/server.key --ssl-cert-file dev_server_keys/server.crt