Merge pull request #107 from datahuborg/feature/openid-connect

Let users authenticate with MIT OpenID Connect and social logins

.codeclimate.yml

@@ -3,6 +3,8 @@ engines:
     enabled: true
   radon:
     enabled: true
+    config:
+      threshold: 'C'
   fixme:
     enabled: true
     checks:

.gitignore

@@ -85,6 +85,9 @@ DerivedData
 # Pods - for those of you who use CocoaPods
 Pods
 
+# Sublime Text
+*.sublime-project
+
 # Vagrant
 .vagrant
 
@@ -93,3 +96,4 @@ ghostdriver.log
 
 # Developer Specific
 devnotes.txt
+src/functional_tests/screenshots/*

.travis.yml

@@ -7,11 +7,15 @@ before_install:
   - sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev ant default-jdk
   - travis_retry wget http://people.apache.org/~jfarrell/thrift/0.9.2/contrib/ubuntu/thrift-compiler_0.9.2_amd64.deb -O thrift-compiler.deb
   - sudo dpkg -i thrift-compiler.deb
+  - sudo mkdir /user_data
+  - sudo chmod -R 777 /user_data/
+  - whoami
 
 install:
   - thrift --version
   - source src/setup.sh
   - pip install -r requirements.txt
+  - python src/scripts/generate_secret_key.py
   - cd src
 
 addons:
@@ -30,5 +34,5 @@ script:
   - python manage.py test browser    # tests datahub core views
   - python manage.py test core       # tests datahub core database access
   # a DB problem requires the Functional Tests to be run seperately
-  - python manage.py test functional_tests.test_login_auth
-  - python manage.py test functional_tests.test_layout_and_styling
+  # - python manage.py test functional_tests.test_login_auth
+  # - python manage.py test functional_tests.test_layout_and_styling

Dockerfile

@@ -5,7 +5,8 @@ RUN apt-get update && apt-get install -y \
         python-pip \
         python-dev \
         libpq-dev \
-        libpq5 && \
+        libpq5 \
+        libffi-dev && \
         apt-get clean
 RUN mkdir /datahub
 WORKDIR /datahub

Getting Started.rst

@@ -226,6 +226,8 @@ You can run the functional tests with:
     $ cd src
     $ python manage.py test functional_tests
 
+Browser screenshots are saved in ``src/functional_tests/screenshots`` on teardown
+
 You can exit the testing container with control-d or via the command ``exit``.
 
 ----------
@@ -244,8 +246,6 @@ Run them from the ``/src`` directory:
     $ cd src
     $ python manage.py test
 
-Alternatively, they can be run individually:
-
 .. code-block:: bash
 
     $ vagrant ssh
@@ -330,6 +330,7 @@ Because the server is containerized, most server commands must be run in a conta
     $ sudo sh provisions/docker/create-dev-containers.sh
     $ sudo sh provisions/docker/rebuild-and-collect-static-files.sh
     $ sudo sh provisions/docker/restore-database.sh
+    $ sudo sh provisions/docker/run-test-container.sh
     $ sudo sh provisions/docker/start-containers.sh
     $ sudo sh provisions/docker/stop-containers.sh
 
@@ -355,6 +356,9 @@ Example Docker commands:
       --volumes-from app \
       datahuborg/datahub \
       python src/manage.py collectstatic --noinput
+
+    # Pip install -r requirements.txt
+    $ sudo docker exec app pip install -r requirements.txt
     
     # Note that `--rm` means it creates an ephemeral container. A new
     # lightweight VM is created just for that command, and is then

Vagrantfile

@@ -16,5 +16,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     vm.name = "datahub-local"
     vm.memory = 2048
     #vm.cpus = 2
+    # Set the timesync threshold to 1 minute, instead of the default 20 minutes.
+    vm.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 60000]
   end
 end

provisions/docker/back-up-all-databases.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+FILE="/tmp/backup.gz"
+echo "Backing up all databases from db container into $FILE on host machine..."
+if [ ! -f $FILE ]; then
+    (set -x;
+    docker run --rm -t --link db:db -v /tmp:/tmp datahuborg/postgres \
+        /bin/bash -c \
+        "pg_dumpall --clean --if-exists --host db --username postgres | gzip > $FILE")
+else
+    echo "$FILE already exists. Cancelling backup."
+fi
+echo "Done."

provisions/docker/back-up-single-database.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+FILE="/tmp/$1-backup.gz"
+echo "Backing up database $1 from db container into $FILE on host machine..."
+if [ ! -f $FILE ]; then
+    (set -x;
+    docker run --rm -t --link db:db -v /tmp:/tmp datahuborg/postgres \
+        /bin/bash -c \
+        "pg_dump --clean --create --if-exists --host db --username postgres -d $1 | gzip > $FILE")
+else
+    echo "$FILE already exists. Cancelling backup."
+fi
+echo "Done."

provisions/docker/build-images.sh

@@ -2,10 +2,12 @@
 
 # Build the project's Docker images if you don't want to pull the prebuilt ones from Docker Hub
 echo "Building Docker images..."
-echo "Building datahuborg/postgres (1/3)"
+echo "Building datahuborg/postgres (1/4)"
 docker build -t datahuborg/postgres provisions/postgres/
-echo "Building datahuborg/nginx (2/3)"
+echo "Building datahuborg/nginx (2/4)"
 docker build -t datahuborg/nginx provisions/nginx/
-echo "Building datahuborg/datahub (3/3)"
+echo "Building datahuborg/datahub (3/4)"
 docker build -t datahuborg/datahub .
+echo "Pulling latest phantomjs (4/4)"
+docker pull wernight/phantomjs:latest
 echo "Done."

provisions/docker/create-dev-containers.sh

@@ -20,6 +20,9 @@
 # data path is declared in the base postgres image, so having one requires the
 # other.
 
+echo "Creating \"datahub_dev\" Docker network if needed..."
+docker network create datahub_dev 2> /dev/null
+
 echo "Creating Docker containers..."
 echo "(1/6) Creating \"logs\" - Data container for all server logs"
 docker create --name logs \
@@ -36,27 +39,28 @@ echo "(3/6) Creating \"db\" - Postgres server"
 docker create --name db \
     --volumes-from logs \
     --volumes-from data \
+    --net=datahub_dev \
     datahuborg/postgres
 echo "(4/6) Creating \"app\" - gunicorn server hosting DataHub"
 docker create --name app \
     --env 'USER=vagrant' \
     --volumes-from logs \
     --volumes-from data \
-    --link db:db \
+    --net=datahub_dev \
     -v /vagrant:/datahub \
     datahuborg/datahub gunicorn --config=provisions/gunicorn/config_dev.py browser.wsgi
 echo "(5/6) Creating \"web\" - nginx http proxy"
 docker create --name web \
     --volumes-from logs \
     --volumes-from app \
     -v /ssl/:/etc/nginx/ssl/ \
-    --link app:app \
+    --net=datahub_dev \
     -p 80:80 -p 443:443 \
     datahuborg/nginx
-echo "(6/6) Creating \"test browser\" - phantomjs"
+echo "(6/6) Creating \"phantomjs\" - PhantomJS remote web driver for Selenium tests"
 docker create --name phantomjs \
     --env 'USER=vagrant' \
-    --link web:web \
+    --net=datahub_dev \
     wernight/phantomjs \
     phantomjs --webdriver=8910
 echo "Done."

provisions/docker/rebuild-and-collect-static-files.sh

@@ -1,2 +1,7 @@
 #!/bin/bash
-docker run --rm -t --volumes-from app datahuborg/datahub /bin/bash -c "make html && python src/manage.py collectstatic --noinput"
+
+(set -x; docker run --rm -t \
+    --volumes-from app \
+    datahuborg/datahub \
+    /bin/bash -c \
+    "make html && python src/manage.py collectstatic --noinput")

provisions/docker/restore-database.sh

@@ -1,8 +1,12 @@
-echo "Wiping existing Postgres data and restoring from /tmp/backup.gz on host machine to db container..."
-if [ -f /tmp/backup.gz ]; then
+#!/bin/bash
+
+FILE="/tmp/backup.gz"
+echo "Wiping existing Postgres data and restoring from $FILE on host machine to db container..."
+if [ -f $FILE ]; then
+    (set -x;
     docker run --rm -t --link db:db -v /tmp:/tmp datahuborg/postgres \
         /bin/bash -c \
-        "gunzip -c /tmp/backup.gz | psql --host db --username postgres postgres"
+        "gunzip -c $FILE | psql --host db --username postgres postgres")
 else
     echo "/tmp/backup.gz missing. Cancelling restore."
 fi

provisions/docker/run-functional-tests.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd /datahub/src
+set -x
+python manage.py test functional_tests.test_login_auth
+python manage.py test functional_tests.test_layout_and_styling
+python manage.py test functional_tests.test_database_capabilities

provisions/docker/run-test-container.sh

@@ -1,14 +1,27 @@
+#!/bin/bash
+
 echo "Starting phantomjs container..."
 if ! `docker inspect -f {{.State.Running}} phantomjs`; then
-    docker start phantomjs
+    (set -x; docker start phantomjs)
     echo "phantomjs started."
 else
     echo "phantomjs already running. Skipping."
 fi
-echo "Creating testing container..."
-docker run -ti --rm \
-    --link db:db \
-    --link phantomjs:phantomjs \
+echo "Stopping app container..."
+(set -x; docker stop app)
+echo "Spinning up test app container..."
+echo "*** Run unit tests with 'sh /datahub/provisions/docker/run-unit-tests.sh'."
+echo "*** Run functional tests with 'sh /datahub/provisions/docker/run-functional-tests.sh'."
+echo "*** Run specific tests with commands like 'python manage.py test core'."
+echo "*** Run a debuggable server with 'python manage.py runserver 0.0.0.0:8000'."
+(set -x; docker run -ti --rm \
     -e "DATAHUB_DOCKER_TESTING=true" \
+    -e "DJANGO_LIVE_TEST_SERVER_ADDRESS=0.0.0.0:8000" \
+    --volumes-from logs \
+    --volumes-from data \
+    --net=datahub_dev \
     -v /vagrant:/datahub \
-    datahuborg/datahub /bin/bash
+    -w /datahub/src \
+    datahuborg/datahub /bin/bash)
+echo "Bringing back app container..."
+(set -x; docker start app)

provisions/docker/run-unit-tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd /datahub/src
+set -x
+python manage.py test inventory
+python manage.py test www
+python manage.py test account
+python manage.py test browser
+python manage.py test core

provisions/docker/start-containers.sh

@@ -6,7 +6,13 @@ echo "Waiting 5 seconds for db to spin up..."
 sleep 5
 # Make sure the database is up to date (should move this to a docker entrypoint script)
 echo "Running Django migrations..."
-docker run --rm --env 'USER=vagrant' --link db:db --volumes-from app datahuborg/datahub python src/manage.py migrate
+docker run \
+    --rm \
+    --env 'USER=vagrant' \
+    --volumes-from app \
+    --net=datahub_dev \
+    datahuborg/datahub \
+    /bin/bash -c "python src/manage.py migrate --noinput"
 docker start app
 docker start web
 echo "Done."

provisions/nginx/default.conf

@@ -54,7 +54,7 @@ server {
         # proxy_set_header X-Forwarded-Host $host:$port_to_forward;
         # proxy_set_header X-Forwarded-Server $server_name;
         # proxy_set_header X-Forwarded-Port $port_to_forward;
-        
+
         # Hide "Connection: keep-alive" from gunicorn so it closes requests
         # properly. The current setup of gunicorn does not handle keep-alive
         # requests and leaves clients hanging, expecting more data.

provisions/postgres/02_set_user_data_permissions.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Make sure the app container (www-data) and db container (postgres) are able
+# to create and delete files and directories in the shared data volume
+# (/user_data).
+gosu root chown -R www-data:www-data /user_data/
+gosu root chmod -R 3775 /user_data/
+gosu root setfacl -d -R -m g:www-data:rwx /user_data
+gosu root setfacl -d -R -m g:postgres:rwx /user_data

provisions/postgres/Dockerfile

@@ -4,4 +4,5 @@ ENV DB_NAME datahub
 ENV DB_USER postgres
 ADD 00_create_database_if_not_exists.sh /docker-entrypoint-initdb.d/
 ADD 01_enable_postgres_logging.sh /docker-entrypoint-initdb.d/
+ADD 02_set_user_data_permissions.sh /docker-entrypoint-initdb.d/
 VOLUME /user_data

requirements.txt

@@ -15,7 +15,13 @@ viz==0.0.5
 Sphinx==1.3.1
 pdb==0.1
 coverage
+
+requests>=2.5.1
+python-social-auth==0.2.13
+pyjwkest==1.0.1
+django-crispy-forms==1.5.2
+
 selenium
-requests
+factory_boy
 gunicorn==19.3.0
 mock==1.3.0

src/account/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'account.config.DataHubAccountConfig'