Merge pull request #778 from NeuroVault/upgrade

Merge CI Fix to master
NeuroVault · Apr 14, 2023 · dacd5e2 · dacd5e2
2 parents 2fb0038 + 6744e3a
commit dacd5e2
Show file tree

Hide file tree

Showing 5 changed files with 171 additions and 60 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -0,0 +1,95 @@
+name: Building and Testing Workflow
+on: [workflow_dispatch,push]
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_neurovault:
+   runs-on: ubuntu-latest
+   steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Configuration
+        run: |
+          cp .env.example .env
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - 
+        name: set lower case owner name
+        run: |
+          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+        env:
+          OWNER: '${{ github.repository_owner }}'
+      -
+        name: Build and push
+        uses: docker/bake-action@v2
+        with:
+          files: docker-compose.yml
+          push: true
+          load: false
+          pull: false
+          set: |
+              worker.tags=ghcr.io/${{ env.OWNER_LC }}/neurovault:${{ hashFiles('**/neurovault/**') }}
+              worker.cache-from=type=registry,ref=ghcr.io/${{ env.OWNER_LC }}/neurovault:${{ hashFiles('**/neurovault/**') }}
+              worker.cache-from=type=gha,scope=cached-neurovault
+              worker.cache-to=type=gha,scope=cached-neurovault,mode=max
+              django.tags=ghcr.io/${{ env.OWNER_LC }}/neurovault_fs:${{ hashFiles('**/neurovault/**') }}
+              django.cache-from=type=registry,ref=ghcr.io/${{ env.OWNER_LC }}/neurovault_fs:${{ hashFiles('**/neurovault/**') }}
+              django.cache-from=type=gha,scope=cached-neurovault_fs
+              django.cache-to=type=gha,scope=cached-neurovault_fs,mode=max
+
+  neurovault_backend_tests:
+    runs-on: ubuntu-latest
+    needs: build_neurovault
+    steps:
+      - 
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Configuration
+        run: |
+          cp .env.example .env
+      -
+        name: load images
+        uses: docker/bake-action@v2
+        with:
+          files: docker-compose.yml
+          push: false
+          load: true
+          set: |
+              worker.cache-from=type=gha,scope=cached-neurovault
+              django.cache-from=type=gha,scope=cached-neurovault_fs
+      - 
+        name: spin up backend
+        run: |
+          # pull specific images since an old version of neurovault is on dockerhub
+          docker compose pull postgres redis
+          docker compose \
+            -f docker-compose.yml \
+            up -d --no-build
+      - 
+        name: Migrate Database
+        run: |
+          until docker compose exec -T \
+          postgres pg_isready -U postgres; do sleep 1; done
+
+          docker compose exec -T django python manage.py migrate
+      -
+        name: Backend Tests
+        run: |
+          docker compose exec -T django python manage.py test
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
diff --git a/compose/django/Dockerfile_fs b/compose/django/Dockerfile_fs
@@ -17,6 +17,7 @@ RUN apt-get -y update \
     --exclude='freesurfer/lib/cuda' \
     --exclude='freesurfer/lib/qt' && \
     apt-get install -y tcsh bc tar libgomp1 perl-modules curl  && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Set up the environment

diff --git a/compose/django/Dockerfile_multi b/compose/django/Dockerfile_multi
@@ -0,0 +1,68 @@
+FROM python:3.10-slim AS neurovault
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  # dependencies for building Python packages
+  build-essential \
+  # psycopg2 dependencies
+  libpq-dev \
+  git
+
+COPY ./compose/django/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+WORKDIR /code
+
+EXPOSE 8000
+
+FROM neurovault AS neurovault_fs
+
+RUN apt-get -y update \
+    && apt-get install -y wget subversion && \
+    wget -qO- ftp://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
+    --exclude='freesurfer/trctrain' \
+    --exclude='freesurfer/subjects/fsaverage_sym' \
+    --exclude='freesurfer/subjects/fsaverage3' \
+    --exclude='freesurfer/subjects/fsaverage4' \
+    --exclude='freesurfer/subjects/fsaverage5' \
+    --exclude='freesurfer/subjects/fsaverage6' \
+    --exclude='freesurfer/subjects/cvs_avg35' \
+    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
+    --exclude='freesurfer/subjects/bert' \
+    --exclude='freesurfer/subjects/V1_average' \
+    --exclude='freesurfer/average/mult-comp-cor' \
+    --exclude='freesurfer/lib/cuda' \
+    --exclude='freesurfer/lib/qt' && \
+    apt-get install -y tcsh bc tar libgomp1 perl-modules curl  && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Set up the environment
+ENV OS Linux
+ENV FS_OVERRIDE 0
+ENV FIX_VERTEX_AREA=
+ENV SUBJECTS_DIR /opt/freesurfer/subjects
+ENV FSF_OUTPUT_FORMAT nii.gz
+ENV MNI_DIR /opt/freesurfer/mni
+ENV LOCAL_DIR /opt/freesurfer/local
+ENV FREESURFER_HOME /opt/freesurfer
+ENV FSFAST_HOME /opt/freesurfer/fsfast
+ENV MINC_BIN_DIR /opt/freesurfer/mni/bin
+ENV MINC_LIB_DIR /opt/freesurfer/mni/lib
+ENV MNI_DATAPATH /opt/freesurfer/mni/data
+ENV FMRI_ANALYSIS_DIR /opt/freesurfer/fsfast
+ENV PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5
+ENV MNI_PERL5LIB /opt/freesurfer/mni/lib/perl5/5.8.5
+ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools:/opt/freesurfer/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
+RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
+
+RUN svn export --force https://github.com/NeuroVault/neurovault_data/trunk/pycortex_datastore /usr/local/share/pycortex/
+RUN apt-get update && apt-get -y install tcsh libglu1-mesa libxmu6
+RUN /opt/freesurfer/bin/mri_convert /opt/freesurfer/subjects/fsaverage/mri/brain.mgz /opt/freesurfer/subjects/fsaverage/mri/brain.nii.gz
+RUN mkdir /usr/local/share/pycortex/db/fsaverage/transforms/
+RUN pip install tornado
+
+RUN apt-get update && apt-get -y install zip
+RUN wget https://ndownloader.figshare.com/files/6891069 -O icbm.zip && unzip icbm.zip -d /opt/freesurfer/subjects/ && rm icbm.zip
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,7 +4,8 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: ./compose/django/Dockerfile_fs
+      dockerfile: ./compose/django/Dockerfile_multi
+      target: neurovault_fs
     image: neurovault/neurovault_fs
     command: python manage.py runserver 0.0.0.0:8000
     ports: 
@@ -15,6 +16,7 @@ services:
     depends_on:
       - redis
       - postgres
+      - worker
     env_file:
       - ./.env
 
@@ -31,6 +33,10 @@ services:
     command: redis-server --appendonly yes
 
   worker:
+    build:
+      context: .
+      dockerfile: ./compose/django/Dockerfile_multi
+      target: neurovault
     image: neurovault/neurovault
     command: celery -A neurovault.celery worker -Q default -n default@%h
     volumes: