Migrate oauth to GitHub app

.gitignore

@@ -2,7 +2,8 @@
 *~
 *.pem
 deploy/janrain.key
-deploy/GITHUB_CLIENT_SECRET
+GITHUB_CLIENT_SECRET
+GITHUB_APP_PRIVATE_KEY_PEM
 deploy/setup/CONFIG
 *.pyc
 *gz

deploy/as-admin.sh

@@ -202,28 +202,6 @@ if [ `which virtualenv`x = x ]; then
     apt_get_install python-virtualenv virtualenv
 fi
 
-# ---------- JAVA ----------
-# N.b. Java 8 isn't available for woody, and is available for jessie
-# only with the addition of unstable to /etc/apt/sources.list and a
-# corresponding prioritization in /etc/apt/preferences (to prevent
-# unstable from taking over the whole system).
-
-if [ `which javac`x != x ] && ( javac -version 2>&1 | egrep -q 1.8 ); then
-    echo "Java 8 OK"
-elif apt-cache policy openjdk-8-jre-headless | grep -q "Installed.*none"; then
-    apt_get_install openjdk-8-jre-headless
-    apt_get_install openjdk-8-jdk
-    sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-    sudo update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-amd64/bin/javac
-elif [ `which javac`x != x ] && ( javac -version 2>&1 | egrep -q 1.7 ); then
-    echo path is $PATH
-    echo "Java 7 OK"
-elif [ `which javac`x != x ]; then
-    echo "** Possible wrong version of java"
-else
-    apt_get_install openjdk-7-jre-headless
-    apt_get_install openjdk-7-jdk
-fi
 
 # ---------- MAVEN 3 ----------
 if [ `which mvn`x = x ]; then
@@ -242,6 +220,22 @@ if [ ! -r /etc/ntp.conf ]; then
 fi
 
 
+# ---------- BACKUP EXISTING CONFIG FILES ---------
+
+# Copy all possible Open Tree config files. A given target machine will have
+# some but not all of these, so suppress "file not found" messages.
+CONFIG_BACKUP_DIR="/home/admin/otol-backups.$(date +'%b-%d-%Y-%H%M%S')"
+mkdir -p $CONFIG_BACKUP_DIR
+cp --parents --preserve=all \
+        /etc/apache2/sites-available/opentree.conf \
+        /etc/apache2/sites-available/opentree-ssl.conf \
+        /etc/apache2/opentree-shared.conf \
+        /home/opentree/repo/opentree/curator/private/config \
+        /home/opentree/repo/opentree/webapp/private/config \
+        /home/phylesystemapi/repo/phylesystem-api/private/config \
+        $CONFIG_BACKUP_DIR 2>/dev/null
+
+
 # ---------- APACHE VHOST ----------
 
 # Set up apache so that web2py takes over the vhost

deploy/push.sh

@@ -72,8 +72,8 @@ done
 [ "x$CONFIGFILE" != x ] || err "No configuration file given (need -c {filename})"
 
 # Configurable parameters
-[ "x$CERTIFICATE_FILE" != x ] || CERTIFICATE_FILE=/etc/ssl/certs/opentree/STAR_opentreeoflife_org.pem
-[ "x$CERTIFICATE_KEY_FILE" != x ] || CERTIFICATE_KEY_FILE=/etc/ssl/private/opentreeoflife.org.key
+[ "x$CERTIFICATE_FILE" != x ] || CERTIFICATE_FILE=/etc/letsencrypt/live/opentreeoflife.org/fullchain.pem
+[ "x$CERTIFICATE_KEY_FILE" != x ] || CERTIFICATE_KEY_FILE=/etc/letsencrypt/live/opentreeoflife.org/privkey.pem
 
 # OPENTREE_HOST (the server being set up) must always be specified, e.g.
 # OPENTREE_HOST=devapi.opentreeoflife.org
@@ -107,15 +107,17 @@ done
 
 # Used by oauth
 if [ "x$OPENTREE_PUBLIC_DOMAIN" = x ]; then
-    echo "Defaulting OPENTREE_PUBLIC_DOMAIN to $OPENTREE_HOST"
+    echo "Defaulting OPENTREE_PUBLIC_DOMAIN to $OPENTREE_HOST" || exit 1
     OPENTREE_PUBLIC_DOMAIN=$OPENTREE_HOST
 fi
 # WEBAPP_BASE_URL is only needed for defaulting other things
 if [ "x$WEBAPP_BASE_URL" = x ]; then
     WEBAPP_BASE_URL=https://$OPENTREE_PUBLIC_DOMAIN
 fi
+[ "x$CURATION_GITHUB_APP_ID" != x ] || CURATION_GITHUB_APP_ID=ID_NOT_PROVIDED
 [ "x$CURATION_GITHUB_CLIENT_ID" != x ] || CURATION_GITHUB_CLIENT_ID=ID_NOT_PROVIDED
 [ "x$CURATION_GITHUB_REDIRECT_URI" != x ] || CURATION_GITHUB_REDIRECT_URI=$WEBAPP_BASE_URL/webapp/user/login
+[ "x$TREEVIEW_GITHUB_APP_ID" != x ] || TREEVIEW_GITHUB_APP_ID=ID_NOT_PROVIDED
 [ "x$TREEVIEW_GITHUB_CLIENT_ID" != x ] || TREEVIEW_GITHUB_CLIENT_ID=ID_NOT_PROVIDED
 [ "x$TREEVIEW_GITHUB_REDIRECT_URI" != x ] || TREEVIEW_GITHUB_REDIRECT_URI=$WEBAPP_BASE_URL/curator/user/login
 
@@ -144,13 +146,18 @@ ASSH="ssh -i ${ADMIN_IDENTITY}"
 # For unprivileged actions to server
 OT_USER=$OPENTREE_USER
 
-echo "host=$OPENTREE_HOST, admin=$OPENTREE_ADMIN, pem=$OPENTREE_IDENTITY, controller=$CONTROLLER"
-
+echo "host=$OPENTREE_HOST, admin=$OPENTREE_ADMIN, pem=$OPENTREE_IDENTITY, controller=$CONTROLLER" || exit 1
+installing_web2py=no
+update_apache_config=yes
 restart_apache=no
 
 function process_arguments {
     sync_system
     docommand $*
+    if [ $installing_web2py = "yes" ]; then
+        # TODO: is there a uniform action here?
+        :  # for now, do nothing
+    fi
     if [ $restart_apache = "yes" ]; then
         restart_apache
     fi
@@ -185,10 +192,14 @@ function docommand {
         ;;
     index  | indexoti | index-db)
         index_doc_store
-            ;;
+        ;;
     apache)
         restart_apache=yes
-            ;;
+        # if used as a standalone command, don't change configuration!
+        if [ $OPENTREE_COMPONENTS = "apache" ]; then
+            update_apache_config=no
+        fi
+        ;;
     echo)
         # Test ability to do remote commands inline...
         ${SSH} "$OT_USER@$OPENTREE_HOST" bash <<EOF
@@ -204,7 +215,7 @@ EOF
         err "Unrecognized command, or component not in OPENTREE_COMPONENTS: $command"
     fi
         # Default if not a recognized command: treat as component name
-        docomponent $command
+        docomponent $command || exit 1
     esac
 }
 
@@ -214,33 +225,36 @@ function docomponent {
     component=$1
     case $component in
     opentree)
-        push_webapps
+        push_webapps || exit 1
+        installing_web2py=yes
         restart_apache=yes
         ;;
     phylesystem-api | api)
         # 'api' option is for backward compatibility
-        push_phylesystem_api
+        push_phylesystem_api || exit 1
+        installing_web2py=yes
         restart_apache=yes
         ;;
     oti)
-        push_neo4j oti
+        push_neo4j oti || exit 1
         ;;
     treemachine)
-        push_neo4j treemachine
+        push_neo4j treemachine || exit 1
         # restart apache to clear the RAM cache (stale results)
         restart_apache=yes
         ;;
     taxomachine)
-        push_neo4j taxomachine
+        push_neo4j taxomachine || exit 1
         # restart apache to clear the RAM cache (stale results)
         restart_apache=yes
         ;;
     smasher)
-        push_smasher
+        push_smasher || exit 1
         ;;
     otcetera)
-	push_otcetera
-	;;
+        push_otcetera || exit 1
+        restart_apache=yes
+        ;;
     *)
         echo "Unrecognized component: $component"
         ;;
@@ -280,7 +294,7 @@ function restart_apache {
     if [ $DRYRUN = "yes" ]; then echo "[restarting apache]"; return; fi
     scp -p -i "${ADMIN_IDENTITY}" restart-apache.sh "$ADMIN@$OPENTREE_HOST":
     ${ASSH} "$ADMIN@$OPENTREE_HOST" bash restart-apache.sh "$OT_USER" "$OPENTREE_HOST" \
-      "$CERTIFICATE_FILE" "$CERTIFICATE_KEY_FILE" "$OTINDEX_BASE_URL"
+      "$CERTIFICATE_FILE" "$CERTIFICATE_KEY_FILE" "$OTINDEX_BASE_URL" "$PHYLAPI_BASE_URL" "$installing_web2py"
 }
 
 # Commands
@@ -294,26 +308,28 @@ function push_neo4j_db {
         err "Usage: $0 -c {configfile} push-db {tarball} {application}"
     fi
     HEREBALL=downloads/$APP.db.tgz
-    time rsync -vax -e "${SSH}" $TARBALL "$OT_USER@$OPENTREE_HOST":$HEREBALL
-    install_neo4j_db $HEREBALL $APP
+    time rsync -vax -e "${SSH}" $TARBALL "$OT_USER@$OPENTREE_HOST":$HEREBALL || exit 1
+    install_neo4j_db $HEREBALL $APP || exit 1
 }
 
 function install_neo4j_db {
     HEREBALL=$1
     APP=$2
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-db.sh $HEREBALL $APP $CONTROLLER
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-db.sh $HEREBALL $APP $CONTROLLER || exit 1
 }
 
 function index_doc_store {
     if [ $DRYRUN = "yes" ]; then echo "[index_doc_store]"; return; fi
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/index-doc-store.sh $OPENTREE_API_BASE_URL $CONTROLLER
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/index-doc-store.sh $OPENTREE_API_BASE_URL $CONTROLLER || exit 1
 }
 
 # Component installation
 
 function push_webapps {
 
+    if [ $CURATION_GITHUB_APP_ID = ID_NOT_PROVIDED ]; then echo "WARNING: Missing GitHub app ID! Curation feedback will be disabled."; fi
     if [ $CURATION_GITHUB_CLIENT_ID = ID_NOT_PROVIDED ]; then echo "WARNING: Missing GitHub client ID! Curation UI will be disabled."; fi
+    if [ $TREEVIEW_GITHUB_APP_ID = ID_NOT_PROVIDED ]; then echo "WARNING: Missing GitHub app ID! Tree-view feedback will be disabled."; fi
     if [ $TREEVIEW_GITHUB_CLIENT_ID = ID_NOT_PROVIDED ]; then echo "WARNING: Missing GitHub client ID! Tree-view feedback will be disabled."; fi
     # We could default these (used by webapps), but for some reason we don't
     [ "x$TREEMACHINE_BASE_URL" != x ] || err "TREEMACHINE_BASE_URL not configured"
@@ -322,24 +338,36 @@ function push_webapps {
     [ "x$CONFLICT_BASE_URL"    != x ] || err "CONFLICT_BASE_URL not configured"
 
     if [ $DRYRUN = "yes" ]; then echo "[opentree]"; return; fi
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-web2py-apps.sh "$OPENTREE_HOST" "${OPENTREE_PUBLIC_DOMAIN}" "${OPENTREE_DEFAULT_APPLICATION}" "$CONTROLLER" "${CURATION_GITHUB_CLIENT_ID}" "${CURATION_GITHUB_REDIRECT_URI}" "${TREEVIEW_GITHUB_CLIENT_ID}" "${TREEVIEW_GITHUB_REDIRECT_URI}" "${TREEMACHINE_BASE_URL}" "${TAXOMACHINE_BASE_URL}" "${OTI_BASE_URL}" "${OPENTREE_API_BASE_URL}" "${COLLECTIONS_API_BASE_URL}" "${AMENDMENTS_API_BASE_URL}" "${FAVORITES_API_BASE_URL}" "${CONFLICT_API_BASE_URL}"
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-web2py-apps.sh "$OPENTREE_HOST" "${OPENTREE_PUBLIC_DOMAIN}" "${OPENTREE_DEFAULT_APPLICATION}" "$CONTROLLER" "${CURATION_GITHUB_APP_ID}" "${CURATION_GITHUB_CLIENT_ID}" "${CURATION_GITHUB_REDIRECT_URI}" "${CURATION_GITHUB_INSTALLATION_ID}" "${TREEVIEW_GITHUB_APP_ID}" "${TREEVIEW_GITHUB_CLIENT_ID}" "${TREEVIEW_GITHUB_REDIRECT_URI}" "${TREEVIEW_GITHUB_INSTALLATION_ID}" "${TREEMACHINE_BASE_URL}" "${TAXOMACHINE_BASE_URL}" "${OTI_BASE_URL}" "${OPENTREE_API_BASE_URL}" "${COLLECTIONS_API_BASE_URL}" "${AMENDMENTS_API_BASE_URL}" "${FAVORITES_API_BASE_URL}" "${CONFLICT_API_BASE_URL}"
     # place the files with secret GitHub API keys for curator and webapp (tree browser feedback) apps
     # N.B. This includes the final domain name, since we'll need different keys for dev.opentreeoflife.org, www.opentreeoflife.org, etc.
     keyfile=${OPENTREE_SECRETS}/treeview-GITHUB_CLIENT_SECRET-$OPENTREE_PUBLIC_DOMAIN
     if [ -r $keyfile ]; then
-        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/webapp/private/GITHUB_CLIENT_SECRET
+        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/webapp/private/GITHUB_CLIENT_SECRET || exit 1
     else
-        echo "** Cannot find GITHUB_CLIENT_SECRET file $keyfile"
+        echo "** Cannot find GITHUB_CLIENT_SECRET file $keyfile" || exit 1
+    fi
+    keyfile=${OPENTREE_SECRETS}/treeview-GITHUB_APP_PRIVATE_KEY_PEM-$OPENTREE_PUBLIC_DOMAIN
+    if [ -r $keyfile ]; then
+        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/webapp/private/GITHUB_APP_PRIVATE_KEY_PEM
+    else
+        echo "** Cannot find GITHUB_APP_PRIVATE_KEY_PEM file $keyfile"
     fi
     keyfile=${OPENTREE_SECRETS}/curation-GITHUB_CLIENT_SECRET-$OPENTREE_PUBLIC_DOMAIN
     if [ -r $keyfile ]; then
-        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/curator/private/GITHUB_CLIENT_SECRET
+        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/curator/private/GITHUB_CLIENT_SECRET || exit 1
+    else
+        echo "** Cannot find GITHUB_CLIENT_SECRET file $keyfile" || exit 1
+    fi
+    keyfile=${OPENTREE_SECRETS}/curation-GITHUB_APP_PRIVATE_KEY_PEM-$OPENTREE_PUBLIC_DOMAIN
+    if [ -r $keyfile ]; then
+        rsync -pr -e "${SSH}" $keyfile "$OT_USER@$OPENTREE_HOST":repo/opentree/curator/private/GITHUB_APP_PRIVATE_KEY_PEM
     else
-        echo "** Cannot find GITHUB_CLIENT_SECRET file $keyfile"
+        echo "** Cannot find GITHUB_APP_PRIVATE_KEY_PEM file $keyfile"
     fi
 
     # we’re using the bot for “anonymous” comments in the synth-tree explorer
-    push_bot_identity
+    push_bot_identity || exit 1
 }
 
 # Utility for all the webapps.
@@ -349,11 +377,13 @@ function push_bot_identity {
     # place an OAuth token for GitHub API by bot user 'opentreeapi'
     tokenfile=${OPENTREE_SECRETS}/OPENTREEAPI_OAUTH_TOKEN
     if [ -r $tokenfile ]; then
-        rsync -pr -e "${SSH}" $tokenfile "$OT_USER@$OPENTREE_HOST":.ssh/OPENTREEAPI_OAUTH_TOKEN
-        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/OPENTREEAPI_OAUTH_TOKEN
+        rsync -pr -e "${SSH}" $tokenfile "$OT_USER@$OPENTREE_HOST":.ssh/OPENTREEAPI_OAUTH_TOKEN || exit 1
+        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/OPENTREEAPI_OAUTH_TOKEN || exit 1
     else
-        echo "** Cannot find OPENTREEAPI_OAUTH_TOKEN file $tokenfile"
+        echo "** Cannot find OPENTREEAPI_OAUTH_TOKEN file $tokenfile" || exit 1
     fi
+
+
 }
 
 # Set up server's clone of phylesystem repo, and the web API
@@ -366,49 +396,50 @@ function push_phylesystem_api {
     [ "x$AMENDMENTS_REPO"   != "x" ] || err "AMENDMENTS_REPO not configured"
     [ "x$FAVORITES_REPO"    != "x" ] || err "FAVORITES_REPO not configured"
     [ "x$OTI_BASE_URL"      != "x" ] || err "OTI_BASE_URL not configured"
+    [ "x$READ_ONLY_MODE"    != "x" ] || err "READ_ONLY_MODE not configured"
     [ "x$OTINDEX_BASE_URL"      != "x" ] || err "OTINDEX_BASE_URL not configured"
 
-    push_bot_identity
+    push_bot_identity || exit 1
 
     # Place private key for GitHub access
     if [ "x$OPENTREE_GH_IDENTITY" = "x" ]; then
-        echo "Warning: OPENTREE_GH_IDENTITY not specified"
+        echo "Warning: OPENTREE_GH_IDENTITY not specified" || exit 1
     elif [ ! -r $OPENTREE_GH_IDENTITY ]; then
-        echo "Warning: $OPENTREE_GH_IDENTITY not found"
+        echo "Warning: $OPENTREE_GH_IDENTITY not found" || exit 1
     else
-        rsync -p -e "${SSH}" "$OPENTREE_GH_IDENTITY" "$OT_USER@$OPENTREE_HOST":.ssh/opentree
-        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/opentree
+        rsync -p -e "${SSH}" "$OPENTREE_GH_IDENTITY" "$OT_USER@$OPENTREE_HOST":.ssh/opentree || exit 1
+        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/opentree || exit 1
     fi
 
     # Try to place an OAuth token for GitHub API by bot user 'opentreeapi'
     tokenfile=${OPENTREE_SECRETS}/OPENTREEAPI_OAUTH_TOKEN
     if [ -r $tokenfile ]; then
-        rsync -p -e "${SSH}" $tokenfile "$OT_USER@$OPENTREE_HOST":.ssh/OPENTREEAPI_OAUTH_TOKEN
-        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/OPENTREEAPI_OAUTH_TOKEN
+        rsync -p -e "${SSH}" $tokenfile "$OT_USER@$OPENTREE_HOST":.ssh/OPENTREEAPI_OAUTH_TOKEN || exit 1
+        ${SSH} "$OT_USER@$OPENTREE_HOST" chmod 600 .ssh/OPENTREEAPI_OAUTH_TOKEN || exit 1
     else
         echo "****************************\n  OAuth token file (${tokenfile}) not found!\n  Falling back to any existing token on the server, OR a prompt for manual creation of webhooks.\n****************************"
     fi
 
     ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-api.sh "$OPENTREE_HOST" \
-           $OPENTREE_DOCSTORE $COLLECTIONS_REPO $AMENDMENTS_REPO $FAVORITES_REPO $CONTROLLER $OTI_BASE_URL $OPENTREE_API_BASE_URL $COLLECTIONS_API_BASE_URL $AMENDMENTS_API_BASE_URL $FAVORITES_API_BASE_URL $OPENTREE_DEFAULT_APPLICATION $OTINDEX_BASE_URL 
+           $OPENTREE_DOCSTORE $COLLECTIONS_REPO $AMENDMENTS_REPO $FAVORITES_REPO $CONTROLLER $OTI_BASE_URL $OPENTREE_API_BASE_URL $COLLECTIONS_API_BASE_URL $AMENDMENTS_API_BASE_URL $FAVORITES_API_BASE_URL $OPENTREE_DEFAULT_APPLICATION $OTINDEX_BASE_URL $READ_ONLY_MODE || exit 1
 }
 
 function push_neo4j {
     APP=$1
     if [ $DRYRUN = "yes" ]; then echo "[neo4j app: $APP]"; return; fi
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-neo4j-app.sh $CONTROLLER $APP $FORCE_COMPILE
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-neo4j-app.sh $CONTROLLER $APP $FORCE_COMPILE || exit 1
 }
 
 function push_smasher {
     if [ $DRYRUN = "yes" ]; then echo "[push_smasher]"; return; fi
     echo push_smasher: ${OPENTREE_WEBAPI_BASE_URL}
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-smasher.sh ${CONTROLLER} ${OPENTREE_WEBAPI_BASE_URL}
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-smasher.sh ${CONTROLLER} ${OPENTREE_WEBAPI_BASE_URL} || exit 1
 }
 
 function push_otcetera {
     if [ $DRYRUN = "yes" ]; then echo "[push_otcetera]"; return; fi
     echo push_otcetera:
-    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-otcetera.sh ${CONTROLLER} ${OPENTREE_WEBAPI_BASE_URL}
+    ${SSH} "$OT_USER@$OPENTREE_HOST" ./setup/install-otcetera.sh ${CONTROLLER} ${OPENTREE_WEBAPI_BASE_URL} || exit 1
 }
 
-process_arguments $*
+process_arguments $* || exit 1