Development

.envrc

@@ -1,6 +1,7 @@
 export FLASK_APP=main:app
 export GAE_VERSION=development-`cat .travis.yml  | grep 'VERSION_NUM=' | cut -f 2 -d '='`
-export GOOGLE_APPLICATION_CREDENTIALS=env_config/client-secret.json
+export CLOUD_CONFIG=1
+export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/env_config/client-secret.json
 export PYTHONPATH=$(pwd)
 export WERKZEUG_DEBUG_PIN=off
 export TEMPLATES_AUTO_RELOAD=1

.gitignore

@@ -61,7 +61,7 @@ target/
 #Random
 *.DS_Store
 
-*.json
+package-lock.json
 
 tmp/
 
@@ -72,6 +72,10 @@ mapping_worker/.envrc
 
 env_config/
 env_config.zip
+env_config_mti/
+env_config_mti.zip
+env_config_al/
+env_config_al.zip
 
 
 .python-version
@@ -92,4 +96,10 @@ photos/*
 *.done
 
 # Heritability run go tool
-invoke
+invoke
+
+.vscode/launch.json
+cloud_functions/heritability_run/strain_data.tsv
+base/bam_bai_signed_download_script.sh
+
+uploads/

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "external/generate-thumbnails"]
+	path = external/generate-thumbnails
+	url = https://github.com/AndersenLab/generate-thumbnails
+[submodule "external/h2calc"]
+	path = external/h2calc
+	url = https://github.com/AndersenLab/h2calc
+[submodule "external/nscalc"]
+	path = external/nscalc
+	url = https://github.com/AndersenLab/nscalc
+[submodule "external/ipcalc"]
+	path = external/ipcalc
+	url = https://github.com/AndersenLab/ipcalc
+[submodule "external/dockerfile"]
+	path = external/dockerfile
+	url = https://github.com/AndersenLab/dockerfile
+[submodule "external/NemaScan"]
+	path = external/NemaScan
+	url = https://github.com/AndersenLab/NemaScan

.travis-al.yml

@@ -0,0 +1,29 @@
+language: bash
+
+env:
+  - GOOGLE_CLOUD_BUCKET="elegansvariation.org" GOOGLE_CLOUD_PROJECT_ID="andersen-lab"
+
+install:
+- openssl aes-256-cbc -K $encrypted_f7a2e30d9b29_key -iv $encrypted_f7a2e30d9b29_iv -in env_config_al.zip.enc -out env_config.zip -d
+- unzip -qo env_config.zip
+- mv env_config_al env_config
+- export VERSION_NUM=1-5-9
+- export APP_CONFIG=master
+- export CLOUD_CONFIG=1
+- if [ "${TRAVIS_BRANCH}" != "master" ]; then export APP_CONFIG=development; fi;
+- export GAE_VERSION=${APP_CONFIG}-${VERSION_NUM}
+- export GOOGLE_APPLICATION_CREDENTIALS=env_config/client-secret.json
+- export GOOGLE_CLOUD_BUCKET=${GOOGLE_CLOUD_BUCKET}
+- export GOOGLE_CLOUD_PROJECT_ID=${GOOGLE_CLOUD_PROJECT_ID}
+
+
+deploy:
+  provider: gae
+  version: "${GAE_VERSION}"
+  project: "${GOOGLE_CLOUD_PROJECT_ID}"
+  keyfile: env_config/client-secret.json
+  on:
+    all_branches: true
+  no_promote: true
+  no_stop_previous_version: true
+  skip_cleanup: true

.travis-mti.yml

@@ -0,0 +1,28 @@
+language: bash
+
+env:
+  - GOOGLE_CLOUD_BUCKET="elegansvariation" GOOGLE_CLOUD_PROJECT_ID="andersen-lab-302418"
+
+install:
+- openssl aes-256-cbc -K $encrypted_eb81f51f2e9b_key -iv $encrypted_eb81f51f2e9b_iv -in env_config_mti.zip.enc -out env_config.zip -d
+- unzip -qo env_config.zip
+- mv env_config_mti env_config
+- export VERSION_NUM=1-5-9
+- export APP_CONFIG=master
+- export CLOUD_CONFIG=1
+- if [ "${TRAVIS_BRANCH}" != "master" ]; then export APP_CONFIG=development; fi;
+- export GAE_VERSION=${APP_CONFIG}-${VERSION_NUM}
+- export GOOGLE_APPLICATION_CREDENTIALS=env_config/client-secret.json
+- export GOOGLE_CLOUD_BUCKET=${GOOGLE_CLOUD_BUCKET}
+- export GOOGLE_CLOUD_PROJECT_ID=${GOOGLE_CLOUD_PROJECT_ID}
+
+deploy:
+  provider: gae
+  version: "${GAE_VERSION}"
+  project: "${GOOGLE_CLOUD_PROJECT_ID}"
+  keyfile: env_config/client-secret.json
+  on:
+    all_branches: true
+  no_promote: true
+  no_stop_previous_version: true
+  skip_cleanup: true

.travis.yml

@@ -1,18 +1,24 @@
 language: bash
 
+env:
+  - GOOGLE_CLOUD_BUCKET="elegansvariation.org" GOOGLE_CLOUD_PROJECT_ID="andersen-lab"
+
 install:
-- openssl aes-256-cbc -K $encrypted_53077b9a3e95_key -iv $encrypted_53077b9a3e95_iv -in env_config.zip.enc -out env_config.zip -d
+- openssl aes-256-cbc -K $encrypted_86f5a1ab1ccf_key -iv $encrypted_86f5a1ab1ccf_iv -in env_config.zip.enc -out env_config.zip -d
 - unzip -qo env_config.zip
-- export VERSION_NUM=1-5-3
-- export APP_CONFIG=master
+- export VERSION_NUM=9-9-9-9
+- export APP_CONFIG=development
+- export CLOUD_CONFIG=1
 - if [ "${TRAVIS_BRANCH}" != "master" ]; then export APP_CONFIG=development; fi;
 - export GAE_VERSION=${APP_CONFIG}-${VERSION_NUM}
 - export GOOGLE_APPLICATION_CREDENTIALS=env_config/client-secret.json
+- export GOOGLE_CLOUD_BUCKET=${GOOGLE_CLOUD_BUCKET}
+- export GOOGLE_CLOUD_PROJECT_ID=${GOOGLE_CLOUD_PROJECT_ID}
 
 deploy:
   provider: gae
   version: "${GAE_VERSION}"
-  project: andersen-lab
+  project: "${GOOGLE_CLOUD_PROJECT_ID}"
   keyfile: env_config/client-secret.json
   on:
     all_branches: true

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 ## Making Changes
 
-* Propose an issue or change you would like ot make using the issue tracker.
+* Propose an issue or change you would like to make using the issue tracker.
 * Create a fork of the development branch. Once you have completed your work, create a pull request.
 * Rebase the latest development branch changes if updates were made in the interim.
-* email danielecook@gmail.com if you have further questions.
+* email danielecook@gmail.com or sam.wachspress@gmail.com if you have further questions.

Dockerfile

@@ -16,6 +16,11 @@ tabix \
 graphviz \
 libgraphviz-dev \
 pkg-config \
+libxml2 \
+xmlsec1 \
+libxml2-dev \
+libxmlsec1-dev \
+libxmlsec1-openssl \
 && rm -rf /var/lib/apt/lists/*
 
 ENV BCFTOOLS_BIN="bcftools-1.10.tar.bz2" \
@@ -56,6 +61,6 @@ ADD . /app
 RUN FLASK_APP=main:app GAE_VERSION=blank-blank flask
 
 # Download the database; GAE_VERSION set as dummy variable
-RUN FLASK_APP=main:app GAE_VERSION=blank-blank flask download_db
+# RUN FLASK_APP=main:app GAE_VERSION=blank-blank flask download_db
 
 CMD gunicorn -b :$PORT main:app

app.yaml

@@ -1,6 +1,10 @@
+beta_settings:
+  cloud_sql_instances: andersen-lab:us-central1:cendr
+
 runtime: custom
+service: dev
 env: flex
-entrypoint: gunicorn -b :$PORT main:app
+entrypoint: gunicorn -b :$PORT main:app --ssl-version TLSv1_2
 runtime_config:
   python_version: 3
 
@@ -32,15 +36,15 @@ resources:
 
 liveness_check:
   path: "/liveness_check"
-  check_interval_sec: 30
-  timeout_sec: 4
+  check_interval_sec: 60
+  timeout_sec: 10
   failure_threshold: 2
   success_threshold: 2
 
 readiness_check:
   path: "/readiness_check"
-  check_interval_sec: 5
-  timeout_sec: 4
+  check_interval_sec: 120
+  timeout_sec: 10
   failure_threshold: 2
   success_threshold: 2
   app_start_timeout_sec: 300

base/application.py

@@ -1,31 +1,40 @@
+from datetime import datetime
 import os
 import json
 import requests
+
 from os.path import basename
-from base.config import config
 from flask import Flask, render_template
 from flask_wtf.csrf import CSRFProtect
-from base.utils.text_utils import render_markdown
 from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.exceptions import HTTPException
 
+
+from base.constants import GOOGLE_CLOUD_BUCKET
+from base.config import config
+from base.utils.text_utils import render_markdown
 from base.manage import (initdb,
                          update_strains,
                          update_credentials,
-                         decrypt_credentials,
-                         download_db)
+                         decrypt_credentials)
 
 # --------- #
 #  Routing  #
 # --------- #
 from base.views.about import about_bp
 from base.views.primary import primary_bp
-from base.views.strains import strain_bp
+from base.views.strains import strains_bp
 from base.views.order import order_bp
 from base.views.data import data_bp
 from base.views.mapping import mapping_bp
 from base.views.gene import gene_bp
 from base.views.user import user_bp
+from base.views.maintenance import maintenance_bp
+from base.views.admin.admin import admin_bp
+from base.views.admin.users import users_bp
+from base.views.admin.data import data_admin_bp
+
+
 
 # Tools
 from base.views.tools import (tools_bp,
@@ -42,9 +51,9 @@
 from base.views.api.api_data import api_data_bp
 
 # Auth
-from base.auth import (auth_bp,
-                       google_bp,
-                       github_bp)
+from base.views.auth import (auth_bp, 
+                             google_bp, 
+                             saml_bp)
 
 
 # ---- End Routing ---- #
@@ -54,7 +63,8 @@
                              cache,
                              debug_toolbar,
                              sslify,
-                             sqlalchemy)
+                             sqlalchemy,
+                             jwt)
 
 # Template filters
 from base.filters import (comma, format_release)
@@ -86,7 +96,7 @@ def configure_ssl(app):
         # Running on server
         app.debug = False
         # Ignore leading slash of urls; skips must use start of path
-        sslify(app)
+        sslify(app, skips=['tasks'])
     elif app.config['DEBUG']:
         debug_toolbar(app)
         app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = True
@@ -97,8 +107,7 @@ def register_commands(app):
     for command in [initdb,
                     update_strains,
                     update_credentials,
-                    decrypt_credentials,
-                    download_db]:
+                    decrypt_credentials]:
         app.cli.add_command(command)
 
 
@@ -110,20 +119,26 @@ def register_template_filters(app):
 def register_extensions(app):
     markdown(app)
     cache.init_app(app, config={'CACHE_TYPE': 'base.utils.cache.datastore_cache'})
-    sqlalchemy(app)
-    CSRFProtect(app)
-    app.config['csrf'] = CSRFProtect(app)
-
+    sqlalchemy.init_app(app)
+    # protect all routes (except the ones listed) from cross site request forgery
+    csrf = CSRFProtect(app)
+    csrf.exempt(auth_bp)
+    csrf.exempt(saml_bp)
+    csrf.exempt(maintenance_bp)
+    app.config['csrf'] = csrf
+    jwt.init_app(app)
 
 def register_blueprints(app):
     """Register blueprints with the Flask application."""
     app.register_blueprint(primary_bp, url_prefix='')
     app.register_blueprint(about_bp, url_prefix='/about')
-    app.register_blueprint(strain_bp, url_prefix='/strain')
+    app.register_blueprint(strains_bp, url_prefix='/strains')
     app.register_blueprint(order_bp, url_prefix='/order')
     app.register_blueprint(data_bp, url_prefix='/data')
     app.register_blueprint(mapping_bp, url_prefix='')
     app.register_blueprint(gene_bp, url_prefix='/gene')
+
+    # User
     app.register_blueprint(user_bp, url_prefix='/user')
 
     # Tools
@@ -138,33 +153,51 @@ def register_blueprints(app):
     app.register_blueprint(api_data_bp, url_prefix='/api')
 
     # Auth
-    app.register_blueprint(auth_bp, url_prefix='')
+    app.register_blueprint(auth_bp, url_prefix='/auth')
+    app.register_blueprint(saml_bp, url_prefix='/saml')
     app.register_blueprint(google_bp, url_prefix='/login')
-    app.register_blueprint(github_bp, url_prefix='/login')
 
-    # Healthchecks
+    # Admin
+    app.register_blueprint(admin_bp, url_prefix='/admin')
+    app.register_blueprint(users_bp, url_prefix='/admin/users')
+    app.register_blueprint(data_admin_bp, url_prefix='/admin/data')
+
+    # Healthchecks/Maintenance
+    app.register_blueprint(maintenance_bp, url_prefix='/tasks')
     app.register_blueprint(check_bp, url_prefix='')
 
 
 def gs_static(url, prefix='static'):
-    return f"https://storage.googleapis.com/elegansvariation.org/{prefix}/{url}"
+    return f"https://storage.googleapis.com/{GOOGLE_CLOUD_BUCKET}/{prefix}/{url}"
 
 
 def configure_jinja(app):
     # Injects "contexts" into templates
     @app.context_processor
     def inject():
-        return dict(version=os.environ.get("GAE_VERSION", "-9-9-9").split("-", 1)[1].replace("-", "."),
-                    json=json,
-                    list=list,
-                    str=str,
-                    int=int,
-                    len=len,
-                    gs_static=gs_static,
-                    basename=basename,
-                    render_markdown=render_markdown)
-
-
+      return dict(version=os.environ.get("GAE_VERSION", "-9-9-9").split("-", 1)[1].replace("-", "."),
+                  json=json,
+                  list=list,
+                  str=str,
+                  int=int,
+                  len=len,
+                  gs_static=gs_static,
+                  basename=basename,
+                  render_markdown=render_markdown)
+
+    # Datetime filters for Jinja
+    @app.template_filter('date_format')
+    def _jinja2_filter_datetime(date, fmt=None):
+      if fmt:
+        return date.strftime(fmt)
+      else:
+        return date.strftime('%c')
+
+'''
+2021-04-14 17:26:51.348674+00:00
+
+'%Y-%m-%d %H:%M:%S.%f+%z'
+'''
 def register_errorhandlers(app):
 
     def render_error(e="generic"):