Merge pull request #3260 from infotroph/allow-depends-update

[Docker, CI] Allow building pecan/depends from pecan/depends

.github/workflows/book.yml

@@ -5,13 +5,11 @@ on:
     branches:
       - master
       - develop
-
     tags:
       - '*'
-
   pull_request:
-
   merge_group:
+  workflow_dispatch:
 
 jobs:
   bookdown:

.github/workflows/ci-weekly.yml

@@ -3,8 +3,9 @@ name: Weekly Checks
 on:
   #  every Monday at 4:30 AM
   # (hopefully after the 1:30 AM `depends` build has completed)
-   schedule:
+  schedule:
     - cron: '30 4 * * 1'
+  workflow_dispatch:
 
 jobs:
   test:

.github/workflows/ci.yml

@@ -8,14 +8,12 @@ on:
 
     tags:
       - '*'
-
   pull_request:
-
   merge_group:
-
   issue_comment:
     types:
       - created
+  workflow_dispatch:
 
 env:
   R_LIBS_USER: /usr/local/lib/R/site-library

.github/workflows/docker.yml

@@ -18,14 +18,12 @@ on:
     branches:
       - master
       - develop
-
   pull_request:
-
   merge_group:
-
   issue_comment:
     types:
       - created
+  workflow_dispatch:
 
 # Certain actions will only run when this is the master repo.
 env:
@@ -74,14 +72,28 @@ jobs:
             echo "PECAN_VERSION=develop" >> $GITHUB_ENV
             echo "PECAN_TAGS=develop" >> $GITHUB_ENV
           fi
-
+      # If any dependencies changed in this PR, try to update depends image
+      # Only runs on pull_request -- others it's not clear what base to compare to
+      - name: check for changed dependencies
+        if: github.base_ref != ''
+        uses: dorny/paths-filter@v3
+        id: findchanges
+        with:
+          filters: |
+            deps:
+              - docker/depends/**
+      - if: steps.findchanges.outputs.deps == 'true'
+        run: echo "UPDATE_DEPENDS_FROM_TAG=${GITHUB_BASE_REF##*/}" >> $GITHUB_ENV
+        env:
+          GITHUB_BASE_REF: ${{ github.base_ref }}
       # use shell script to build, there is some complexity in this
       - name: create images
         run: ./docker.sh -i github
         env:
           PECAN_GIT_CHECKSUM: ${{ github.sha }}
           PECAN_GIT_BRANCH: ${GITHUB_BRANCH}
           VERSION: ${{ env.PECAN_VERSION }}
+          UPDATE_DEPENDS_FROM_TAG: ${{ env.UPDATE_DEPENDS_FROM_TAG }}
 
       # push all images to github
       - name: Publish to GitHub

book_source/03_topical_pages/94_docker/04_models.Rmd

@@ -67,10 +67,11 @@ docker build \
 Example of a Dockerfile, in this case to build the ED2 model.
 
 ```dockerfile
+ARG IMAGE_VERSION="latest"
 # ----------------------------------------------------------------------
 # BUILD MODEL BINARY
 # ----------------------------------------------------------------------
-FROM debian:stretch as model-binary
+FROM pecan/models:${IMAGE_VERSION} as model-binary
 
 # Some variables that can be used to set control the docker build
 ARG MODEL_VERSION=git

docker.sh

@@ -115,6 +115,9 @@ echo "#"
 echo "# The docker image for dependencies takes a long time to build. You"
 echo "# can use a prebuilt version (default) or force a new version to be"
 echo "# built locally using: DEPEND=build $0"
+echo "#"
+echo "# EXPERIMENTAL: To attempt updating an existing dependency image"
+echo "# instead of building from scratch, use UPDATE_DEPENDS_FROM_TAG=<tag>"
 echo "# ----------------------------------------------------------------------"
 
 # not building dependencies image, following command will build this
@@ -125,6 +128,17 @@ if [ "${DEPEND}" == "build" ]; then
         --build-arg R_VERSION=${R_VERSION} ${GITHUB_WORKFLOW_ARG} \
         --tag pecan/depends:${IMAGE_VERSION} \
         docker/depends
+elif [ "${UPDATE_DEPENDS_FROM_TAG}" != "" ]; then
+    echo "# Attempting to update from existing pecan/depends:${UPDATE_DEPENDS_FROM_TAG}."
+    echo "# This is experimental. if it fails, please instead use"
+    echo "# 'DEPEND=build' to start from a known clean state."
+    ${DEBUG} docker build \
+        --pull \
+        --secret id=github_token,env=GITHUB_PAT \
+        --build-arg FROM_IMAGE="pecan/depends" \
+        --build-arg R_VERSION=${UPDATE_DEPENDS_FROM_TAG} ${GITHUB_WORKFLOW_ARG} \
+        --tag pecan/depends:${IMAGE_VERSION} \
+        docker/depends
 else
     if [ "$( docker image ls -q pecan/depends:${IMAGE_VERSION} )" == "" ]; then
         if [ "${PECAN_GIT_BRANCH}" != "master" ]; then

docker/depends/Dockerfile

@@ -1,20 +1,12 @@
-ARG R_VERSION="4.0.2"
+ARG R_VERSION="4.1"
+ARG FROM_IMAGE="rocker/tidyverse"
 
 # ----------------------------------------------------------------------
 # PECAN FOR MODEL BASE IMAGE
 # ----------------------------------------------------------------------
-FROM rocker/tidyverse:${R_VERSION}
+FROM ${FROM_IMAGE}:${R_VERSION}
 MAINTAINER Rob Kooper <kooper@illinois.edu>
 
-# ----------------------------------------------------------------------
-# UPDATE GIT
-# This is needed for stretch and github actions
-# ----------------------------------------------------------------------
-RUN if [ "$(lsb_release -s -c)" = "stretch" ]; then \
-      echo 'deb http://deb.debian.org/debian stretch-backports main' >> /etc/apt/sources.list \
-      && apt-get update \
-      && apt-get -t stretch-backports upgrade -y git \
-    ; fi
 
 # ----------------------------------------------------------------------
 # INSTALL BINARY/LIBRARY DEPENDENCIES