Merge pull request #1353 from LLNL/v2022.10.0-RC

V2022.10.0 RC to main

.dockerignore

@@ -0,0 +1 @@
+Dockerfile

.gitignore

@@ -1,3 +1,6 @@
+.cache
+.idea
+*.sync-conflict-*
 *.pyc
 *.o
 *.a

.gitlab-ci.yml

@@ -1,6 +1,6 @@
 ###############################################################################
-# Copyright (c) 2016-2020, Lawrence Livermore National Security, LLC
-# and RAJA project contributors. See the RAJA/COPYRIGHT file for details.
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 ###############################################################################
@@ -10,93 +10,56 @@
 # at Lawrence Livermore National Laboratory (LLNL).
 #
 # This entire pipeline is LLNL-specific
-# #############################################################################
-
-# We define the following GitLab pipeline variables:
-#
-# GIT_SUBMODULE_STRATEGY:
-# Tells Gitlab to recursively update the submodules when cloning umpire
 #
-# ALLOC_NAME:
-# On LLNL's ruby, this pipeline creates only one allocation shared among jobs
-# in order to save time and resources. This allocation has to be uniquely named
-# so that we are sure to retrieve it.
+# Important note: This file is a template provided by
+# llnl/radiuss-shared-ci. It should not require any change from the project to
+# get started but could feature project-specific stages.
 #
-# BUILD_ROOT:
-# The path to the shared resources between all jobs. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-#
-# DEFAULT_TIME:
-# Default time to let the Lassen jobs run will be 30 minutes. However, if it is
-# a job that requires more time, it will be overwritten in the lassen template 
-# file.
-# TODO: add a clean-up mechanism
+# However, each project should provide:
+# - .gitlab/custom-jobs-and-variables.yml
+# - .gitlab/subscribed-pipelines.yml
+# - .gitlab/${MACHINE}-build-and-test-extra.yml
+###############################################################################
 
+# We define the following GitLab pipeline variables:
 variables:
+  MP_BRANCH: "develop"
+# Use a service user to run CI. This prevents from running pipelines as an
+# actual user.
+  LLNL_SERVICE_USER: ""
+# Use a service user workspace. Solves permission issues, stores everything
+# at the same location whoever triggers a pipeline.
+#  CUSTOM_CI_BUILDS_DIR: ""
+# Tells Gitlab to recursively update the submodules when cloning the project.
   GIT_SUBMODULE_STRATEGY: recursive
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+# We build the projects in the CI clone directory.
+# TODO: add a clean-up mechanism
   BUILD_ROOT: ${CI_PROJECT_DIR}
-  DEFAULT_TIME: 30
-  MP_BRANCH: "develop"
 
-# Normally, stages are blocking in Gitlab. However, using the keyword "needs" we
-# can express dependencies between job that break the ordering of stages, in
-# favor of a DAG.
-# In practice r_*, l_* and b_* stages are independently run and start immediately.
+# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# corresponds to a test batch on a given machine.
 
+# High level stages
 stages:
-  - r_allocate_resources
-  - r_build_and_test
-  - r_release_resources
-  - l_build_and_test
-  - b_build_and_test
-  - c_build_and_test
+  - build-and-test
   - multi_project
 
-# This is the rules that drives the activation of "advanced" jobs. All advanced
-# jobs will share this through a template mechanism.
-.advanced_pipeline:
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "develop" || $ALL_TARGETS == "ON"' #run only if ...
-
-# These are also templates (.name) that define project specific build commands.
-# If an allocation exist with the name defined in this pipeline, the job will
-# use it (slurm specific).
-.build_toss_3_x86_64_ib_script:
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t ${DEFAULT_TIME} -N 1 scripts/gitlab/build_and_test.sh
-  artifacts:
-    reports:
-      junit: junit.xml
-
-.build_toss_4_x86_64_ib_corona_script:
-  script:
-    - srun -p pbatch -t 30 -N 1 scripts/gitlab/build_and_test.sh
-
-# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-.build_blueos_3_ppc64le_ib_script:
-  script:
-    - lalloc 1 -W ${DEFAULT_TIME} scripts/gitlab/build_and_test.sh
-  artifacts:
-    reports:
-      junit: junit.xml
-
-.build_blueos_3_ppc64le_ib_ats_disabled_script:
-  script:
-    - lalloc 1 --atsdisable -W ${DEFAULT_TIME} scripts/gitlab/build_and_test.sh
-  artifacts:
-    reports:
-      junit: junit.xml
-
-.build_blueos_3_ppc64le_ib_p9_script:
-  extends: .build_blueos_3_ppc64le_ib_script
+# Template for jobs triggering a build-and-test sub-pipelines:
+.build-and-test:
+  stage: build-and-test
+  trigger:
+    include:
+      - local: '.gitlab/custom-jobs-and-variables.yml'
+      - project: 'radiuss/radiuss-shared-ci'
+        ref: v2022.09.0
+        file: '${CI_MACHINE}-build-and-test.yml'
+      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+    strategy: depend
+    forward:
+      pipeline_variables: true
 
-# If testing develop branch, trigger CHAI pipeline with this version of RAJA.
+# If testing develop branch, trigger RAJAPerf pipeline with this version of
+# RAJA.
 # TODO: Once spack allows to clone a specific commit on demand, then point to the exact commit.
 #       This will prevent from sticking to a branch (here develop).
 #       MP_BRANCH is short for "Multi-Project Branch" and will usually be develop.
@@ -111,11 +74,6 @@ trigger-rajaperf:
     branch: develop
     strategy: depend
 
-# This is where jobs are included.
+# pipelines subscribed by the project
 include:
-  - local: .gitlab/ruby-templates.yml
-  - local: .gitlab/ruby-jobs.yml
-  - local: .gitlab/lassen-templates.yml
-  - local: .gitlab/lassen-jobs.yml
-  - local: .gitlab/corona-templates.yml
-  - local: .gitlab/corona-jobs.yml
+  - local: .gitlab/subscribed-pipelines.yml

.gitlab/corona-build-and-test-extra.yml

@@ -0,0 +1,28 @@
+#############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# and RAJA project contributors. See the RAJA/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+#############################################################################
+
+########################
+# Overridden shared jobs
+########################
+# We duplicate the shared jobs description and add necessary changes for RAJA.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+# No overridden jobs so far.
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
+# describe the spec here.
+
+rocm_5_1_1_clang_13_0_0_desul_atomics:
+  variables:
+    SPEC: " +rocm ~openmp +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop"
+  extends: .build_and_test_on_corona
+

.gitlab/custom-jobs-and-variables.yml

@@ -0,0 +1,52 @@
+###############################################################################
+# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+###############################################################################
+
+# We define the following GitLab pipeline variables:
+variables:
+# On LLNL's ruby, this pipeline creates only one allocation shared among jobs
+# in order to save time and resources. This allocation has to be uniquely named
+# so that we are sure to retrieve it and avoid collisions.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Ruby
+# Arguments for top level allocation
+  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=60 --nodes=1"
+# Arguments for job level allocation
+  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=45 --nodes=1"
+# Project specific variants for ruby
+  PROJECT_RUBY_VARIANTS: "+openmp "
+# Project specific deps for ruby
+  PROJECT_RUBY_DEPS: ""
+
+# Corona
+# Arguments for top level allocation
+  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--time-limit=60m --nodes=1"
+# Arguments for job level allocation
+  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1"
+# Project specific variants for corona
+  PROJECT_CORONA_VARIANTS: "~openmp "
+# Project specific deps for corona
+  PROJECT_CORONA_DEPS: "^blt@develop "
+
+# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# Arguments for job level allocation
+  LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
+# Project specific variants for lassen
+  PROJECT_LASSEN_VARIANTS: "+openmp "
+# Project specific deps for lassen
+  PROJECT_LASSEN_DEPS: ""
+
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+# See Umpire for an example (export junit test reports).
+.custom_build_and_test:
+  artifacts:
+    reports:
+      junit: junit.xml