diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml index 99a108309e..ec40843f0e 100644 --- a/ci/.gitlab-ci.yml +++ b/ci/.gitlab-ci.yml @@ -21,7 +21,8 @@ include: - local: 'ci/cpu/gcc11_release_stdexec.yml' - local: 'ci/cpu/gcc11_debug_stdexec.yml' - local: 'ci/cpu/gcc12_release_cxx20.yml' - - local: 'ci/cpu/gcc13_codecov.yml' + # https://github.com/eth-cscs/DLA-Future/pull/1311 + # - local: 'ci/cpu/gcc13_codecov.yml' - local: 'ci/cpu/gcc13_release.yml' - local: 'ci/cuda/gcc13_release.yml' - local: 'ci/cuda/gcc13_release_scalapack.yml' diff --git a/ci/base-images/amdgpu-cray-mpich/HOWTO.md b/ci/base-images/amdgpu-cray-mpich/HOWTO.md deleted file mode 100644 index a9f7f902a4..0000000000 --- a/ci/base-images/amdgpu-cray-mpich/HOWTO.md +++ /dev/null @@ -1,43 +0,0 @@ -# Modified base image to allow building cray-mpich - -## Preparation steps - -``` -mkdir lib64 -cp -a /usr/lib64/libxpmem.* lib64/ - -git clone https://github.com/eth-cscs/alps-cluster-config.git - -cp alps-cluster-config/beverin/packages.yaml packages.yaml -``` - -## Edit cluster config files - -Modify `packages.yaml`: -``` - patchelf: - require: "@:0.17" - libfabric: -- buildable: false -- externals: -- - spec: libfabric@1.15.2.0 -- prefix: /opt/cray/libfabric/1.15.2.0/ -+ require: "@1.15.2" - slurm: - buildable: false - externals: -``` -Note: The container engine (CE) will replace libfabric with the system one when running the container. -Make sure to use the same version. - -## Build and push - -``` -export TAG="v0.2" -CSCS_REGISTRY="jfrog.svc.cscs.ch/docker-ci-ext/4700071344751697" -podman login jfrog.svc.cscs.ch - -podman build -f build.Dockerfile -t $CSCS_REGISTRY/base-images/rocm_6.0.2-dev-ubuntu-22.04:$TAG - -podman push $CSCS_REGISTRY/base-images/rocm_6.0.2-dev-ubuntu-22.04:$TAG -``` diff --git a/ci/base-images/amdgpu-cray-mpich/build.Dockerfile b/ci/base-images/amdgpu-cray-mpich/build.Dockerfile deleted file mode 100644 index ce4064284f..0000000000 --- a/ci/base-images/amdgpu-cray-mpich/build.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker.io/rocm/dev-ubuntu-22.04:6.0.2 - -COPY lib64 /usr/lib -COPY packages.yaml /root/.spack/packages.yaml -COPY alps-cluster-config/site /root/site diff --git a/ci/base-images/gh200-cray-mpich/HOWTO.md b/ci/base-images/gh200-cray-mpich/HOWTO.md deleted file mode 100644 index 3083eb39e1..0000000000 --- a/ci/base-images/gh200-cray-mpich/HOWTO.md +++ /dev/null @@ -1,64 +0,0 @@ -# Modified base image to allow building cray-mpich - -## Preparation steps - -``` -mkdir lib64 -cp -a /usr/lib64/libcuda.* lib64/ -cp -a /usr/lib64/libxpmem.* lib64/ - -git clone https://github.com/eth-cscs/alps-cluster-config.git - -cp alps-cluster-config/daint/packages.yaml packages.yaml -``` - -## Edit cluster config files - -Modify `packages.yaml`: -``` - xpmem: - buildable: false - externals: - - spec: xpmem@2.9.6 - prefix: /usr - libfabric: -- buildable: false -- externals: -- - spec: libfabric@1.15.2.0 -- prefix: /opt/cray/libfabric/1.15.2.0/ -+ require: "@1.15.2.0" - slurm: - buildable: false - externals: - - spec: slurm@23-11-7 - prefix: /usr -``` -Note: The container engine (CE) will replace libfabric with the system one when running the container. -Make sure to use the same version. - - -Modify `alps-cluster-config/site/repo/packages/cray-gtl/package.py` -``` - patchelf("--force-rpath", "--set-rpath", rpath, f, fail_on_error=False) - # The C compiler wrapper can fail because libmpi_gtl_cuda refers to the symbol - # __gxx_personality_v0 but wasn't linked against libstdc++. -- if "libmpi_gtl_cuda.so" in str(f): -- patchelf("--add-needed", "libstdc++.so", f, fail_on_error=False) - if "@8.1.27+cuda" in self.spec: - patchelf("--add-needed", "libcudart.so", f, fail_on_error=False) - patchelf("--add-needed", "libcuda.so", f, fail_on_error=False) -``` -Note: the library links `libstdc++.so` from version 8.1.23. All the available aarch64 libraries already link with it, -therefore we can safely remove it for gh200. - -## Build and push - -``` -export TAG="v1.4" -CSCS_REGISTRY="jfrog.svc.cscs.ch/docker-ci-ext/4700071344751697" -podman login jfrog.svc.cscs.ch - -podman build -f build.Dockerfile -t $CSCS_REGISTRY/base-images/cuda_12.6.1-devel-ubuntu24.04:$TAG - -podman push $CSCS_REGISTRY/base-images/cuda_12.6.1-devel-ubuntu24.04:$TAG -``` diff --git a/ci/base-images/gh200-cray-mpich/build.Dockerfile b/ci/base-images/gh200-cray-mpich/build.Dockerfile deleted file mode 100644 index 4b95562c40..0000000000 --- a/ci/base-images/gh200-cray-mpich/build.Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker.io/nvidia/cuda:12.6.1-devel-ubuntu24.04 - -COPY lib64 /usr/lib -COPY packages.yaml /root/.spack/packages.yaml -COPY alps-cluster-config/site /root/site diff --git a/ci/ci-ext-custom.yml b/ci/ci-ext-custom.yml index 4fda4c9f9a..a7953d2b12 100644 --- a/ci/ci-ext-custom.yml +++ b/ci/ci-ext-custom.yml @@ -14,6 +14,8 @@ include: extends: .container-runner-eiger variables: SLURM_CONSTRAINT: mc + # Need to reset as remote adds SLURM_MPI_TYPE: pmi2 + SLURM_MPI_TYPE: cray_shasta .dlaf-container-runner-daint-gh200: extends: .container-runner-daint-gh200 diff --git a/ci/common-ci.yml b/ci/common-ci.yml index 73433dfba0..706fa31475 100644 --- a/ci/common-ci.yml +++ b/ci/common-ci.yml @@ -30,10 +30,12 @@ variables: - TAG_APTGET=`echo ${EXTRA_APTGET} | sha256sum - | head -c 6` - TAG_COMPILER=`echo ${COMPILER}_CXX${CXXSTD} | sed 's/[@]//g'` - TAG_DOCKERFILE=`sha256sum $DOCKERFILE | head -c 12` - - TAG_SPACK=`echo $SPACK_SHA | sed "s/develop-//g" | head -c 16` + - TAG_SPACK=`echo $SPACK_SHA | sed "s/develop-//g" | head -c 8` + - TAG_SPACK_PACKAGES=`echo $SPACK_PACKAGES_SHA | sed "s/develop-//g" | head -c 8` + - TAG_ALPS_CLUSTER_CONFIG=`echo $ALPS_CLUSTER_CONFIG_SHA | head -c 8` - TAG_REPO=`find $SPACK_DLAF_REPO -type f -exec sha256sum {} \; | sha256sum - | head -c 12` - TAG_ENVIRONMENT=`cat $SPACK_ENVIRONMENT $COMMON_SPACK_ENVIRONMENT | sha256sum | head -c 12` - - TAG=${TAG_IMAGE}-${TAG_APTGET}-${TAG_COMPILER}-MKL${USE_MKL}-${TAG_DOCKERFILE}-${TAG_SPACK}-${TAG_REPO}-${TAG_ENVIRONMENT} + - TAG=${TAG_IMAGE}-${TAG_APTGET}-${TAG_COMPILER}-MKL${USE_MKL}-${TAG_DOCKERFILE}-${TAG_SPACK}-${TAG_SPACK_PACKAGES}-${TAG_ALPS_CLUSTER_CONFIG}-${TAG_REPO}-${TAG_ENVIRONMENT} - export PERSIST_IMAGE_NAME=$DEPS_IMAGE:$TAG - echo "DEPS_IMAGE=$PERSIST_IMAGE_NAME" > build.env - echo "USE_MKL=$USE_MKL" >> build.env @@ -44,11 +46,15 @@ variables: reports: dotenv: build.env variables: - SPACK_SHA: develop-2025-02-09 + SPACK_SHA: fbadf557433ae3d8cb8a2b63a81303de42a9fd6f + SPACK_PACKAGES_SHA: bf71e784ba28818a05468ce4ffef405fa7c3d151 + ALPS_CLUSTER_CONFIG_SHA: 3c26eb25bdfa3151b637c040154a55bb9da31ba1 SPACK_DLAF_REPO: ./spack DOCKER_BUILD_ARGS: '[ "BASE_IMAGE", "SPACK_SHA", + "SPACK_PACKAGES_SHA", + "ALPS_CLUSTER_CONFIG_SHA", "EXTRA_APTGET", "COMPILER", "CXXSTD", diff --git a/ci/cpu/asan_ubsan_lsan.yml b/ci/cpu/asan_ubsan_lsan.yml index dddf575f5c..97c8047984 100644 --- a/ci/cpu/asan_ubsan_lsan.yml +++ b/ci/cpu/asan_ubsan_lsan.yml @@ -14,7 +14,7 @@ cpu asan ubsan lsan deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-18 libclang-rt-18-dev libomp-18-dev llvm-18" - COMPILER: clang@18 + COMPILER: llvm@18 SPACK_ENVIRONMENT: ci/docker/asan-ubsan-lsan.yaml USE_MKL: "ON" DEPS_IMAGE: $CSCS_REGISTRY_PATH/cpu-asan-ubsan-lsan/deps diff --git a/ci/cpu/clang15_release.yml b/ci/cpu/clang15_release.yml index a7d2e6cf66..4932ecb286 100644 --- a/ci/cpu/clang15_release.yml +++ b/ci/cpu/clang15_release.yml @@ -14,7 +14,7 @@ cpu clang15 release deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-15" - COMPILER: clang@15 + COMPILER: llvm@15 SPACK_ENVIRONMENT: ci/docker/release-cpu-serial.yaml USE_MKL: "ON" DEPS_IMAGE: $CSCS_REGISTRY_PATH/cpu-clang15-release/deps diff --git a/ci/cpu/clang15_release_cxx20.yml b/ci/cpu/clang15_release_cxx20.yml index 32428f0330..f39a2ba812 100644 --- a/ci/cpu/clang15_release_cxx20.yml +++ b/ci/cpu/clang15_release_cxx20.yml @@ -14,7 +14,7 @@ cpu clang15 cxx20 release deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-15 libomp-15-dev" - COMPILER: clang@15 + COMPILER: llvm@15 CXXSTD: 20 SPACK_ENVIRONMENT: ci/docker/release-cpu.yaml USE_MKL: "ON" diff --git a/ci/cpu/clang15_release_stdexec.yml b/ci/cpu/clang15_release_stdexec.yml index f2c7e87e88..2f98dc7446 100644 --- a/ci/cpu/clang15_release_stdexec.yml +++ b/ci/cpu/clang15_release_stdexec.yml @@ -14,7 +14,7 @@ cpu clang15 stdexec release deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-15 libomp-15-dev" - COMPILER: clang@15 + COMPILER: llvm@15 CXXSTD: 20 SPACK_ENVIRONMENT: ci/docker/release-cpu-stdexec.yaml USE_MKL: "ON" diff --git a/ci/cpu/clang16_release.yml b/ci/cpu/clang16_release.yml index 0739f9de38..da88397224 100644 --- a/ci/cpu/clang16_release.yml +++ b/ci/cpu/clang16_release.yml @@ -14,7 +14,7 @@ cpu clang16 release deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-16 libomp-16-dev" - COMPILER: clang@16 + COMPILER: llvm@16 SPACK_ENVIRONMENT: ci/docker/release-cpu.yaml USE_MKL: "ON" DEPS_IMAGE: $CSCS_REGISTRY_PATH/cpu-clang16-release/deps diff --git a/ci/cpu/clang18_release.yml b/ci/cpu/clang18_release.yml index cc431be028..0f60fc65a4 100644 --- a/ci/cpu/clang18_release.yml +++ b/ci/cpu/clang18_release.yml @@ -14,7 +14,7 @@ cpu clang18 release deps: extends: .build_deps_common variables: EXTRA_APTGET: "clang-18 libomp-18-dev" - COMPILER: clang@18 + COMPILER: llvm@18 SPACK_ENVIRONMENT: ci/docker/release-cpu.yaml USE_MKL: "ON" DEPS_IMAGE: $CSCS_REGISTRY_PATH/cpu-clang18-release/deps diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile index 742a82b84f..ea8cfea64b 100644 --- a/ci/docker/build.Dockerfile +++ b/ci/docker/build.Dockerfile @@ -33,31 +33,25 @@ RUN apt-get -yqq update && \ ${EXTRA_APTGET} && \ rm -rf /var/lib/apt/lists/* -# Install MKL and remove static libs (to keep image smaller) -ARG USE_MKL=ON -ARG MKL_VERSION=2024.0 -ARG MKL_SPEC=2024.0.0 -RUN if [ "$USE_MKL" = "ON" ]; then \ - wget -qO - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB 2>/dev/null > /etc/apt/trusted.gpg.d/intel.asc && \ - apt-add-repository 'deb https://apt.repos.intel.com/oneapi all main' && \ - apt-get install -y -qq --no-install-recommends intel-oneapi-mkl-devel-${MKL_VERSION} && \ - rm -rf /var/lib/apt/lists/* && \ - find "/opt/intel/oneapi" -name "*.a" -delete ; \ - fi - -# This is the spack version we want to have +# These are the versions of spack and spack-packages we want to have ARG SPACK_SHA ENV SPACK_SHA=$SPACK_SHA +ARG SPACK_PACKAGES_SHA +ENV SPACK_PACKAGES_SHA=$SPACK_PACKAGES_SHA # Install the specific ref of Spack provided by the user and find compilers RUN mkdir -p /opt/spack && \ - curl -Ls "https://api.github.com/repos/spack/spack/tarball/$SPACK_SHA" | tar --strip-components=1 -xz -C /opt/spack + curl -Ls "https://api.github.com/repos/spack/spack/tarball/$SPACK_SHA" | tar --strip-components=1 -xz -C /opt/spack && \ + mkdir -p /opt/spack-packages && \ + curl -Ls "https://api.github.com/repos/spack/spack-packages/tarball/$SPACK_PACKAGES_SHA" | tar --strip-components=1 -xz -C /opt/spack-packages +RUN spack repo add --scope site /opt/spack-packages/repos/spack_repo/builtin -# Find compilers + Add gfortran to clang specs + Define which compiler we want to use +# Find compilers + Define which compiler we want to use ARG COMPILER -RUN spack compiler find && \ - gawk -i inplace '$0 ~ "compiler:" {flag=0} $0 ~ "spec:.*clang" {flag=1} flag == 1 && $1 ~ "^f[c7]" && $2 ~ "null" {gsub("null","/usr/bin/gfortran",$0)} {print $0}' /root/.spack/linux/compilers.yaml && \ - spack config add "packages:all:require:[\"%${COMPILER}\"]" +RUN spack external find gcc llvm && \ + spack config add "packages:cxx:require:'${COMPILER}'" && \ + spack config add "packages:c:require:'${COMPILER}'" && \ + spack config add "packages:fortran:require:gcc" RUN spack external find \ autoconf \ @@ -74,10 +68,7 @@ RUN spack external find \ perl \ pkg-config \ python \ - xz && \ - if [ "$USE_MKL" = "ON" ]; then \ - echo -e " intel-oneapi-mkl:\n externals:\n - spec: \"intel-oneapi-mkl@$MKL_SPEC mpi_family=mpich\"\n prefix: /opt/intel/oneapi\n buildable: False" >> ~/.spack/packages.yaml ; \ - fi + xz # Add our custom spack repo from here ARG SPACK_DLAF_REPO @@ -86,10 +77,12 @@ COPY $SPACK_DLAF_REPO /user_repo RUN spack repo add --scope site /user_repo ### Workaround until CE provides full MPI substitution. -# Add ~/site/repo if it exists in the base image -RUN if [ -d ~/site/repo ]; then \ - spack repo add --scope site ~/site/repo; \ - fi +ARG ALPS_CLUSTER_CONFIG_SHA +ENV ALPS_CLUSTER_CONFIG_SHA=$ALPS_CLUSTER_CONFIG_SHA +RUN mkdir -p /opt/alps-cluster-config && \ + curl -Ls "https://api.github.com/repos/eth-cscs/alps-cluster-config/tarball/$ALPS_CLUSTER_CONFIG_SHA" | \ + tar --strip-components=1 -xz -C /opt/alps-cluster-config && \ + spack repo add --scope site /opt/alps-cluster-config/site/spack_repo/alps # Set this to a spack.yaml file which contains a spec # e.g. --build-arg SPACK_ENVIRONMENT=ci/spack/my-env.yaml diff --git a/ci/docker/common-amdgpu.yaml b/ci/docker/common-amdgpu.yaml index f5265844bb..037f3a9640 100644 --- a/ci/docker/common-amdgpu.yaml +++ b/ci/docker/common-amdgpu.yaml @@ -22,6 +22,17 @@ packages: mpi: ### Workaround until CE provides full MPI substitution. require: 'cray-mpich' + xpmem: + require: + - '~kernel-module' + cray-gtl: + buildable: True + cray-mpich: + buildable: True + cray-pals: + buildable: True + cray-pmi: + buildable: True blaspp: variants: - '~cuda' @@ -29,17 +40,13 @@ packages: - '~rocm' intel-oneapi-mkl: require: - - 'threads=openmp' + - 'threads=none' openblas: variants: - 'threads=openmp' - openssl: - # without version spack concretizes openssl@3.3.1 that doesn't build due to deprecation - require: - - '@3.3.2' libfabric: require: - - '@1.15.2' + - '@1.22.0' hwloc: variants: - '~libxml2' diff --git a/ci/docker/common-gh200.yaml b/ci/docker/common-gh200.yaml index b9d472e897..9e85b39950 100644 --- a/ci/docker/common-gh200.yaml +++ b/ci/docker/common-gh200.yaml @@ -24,6 +24,17 @@ packages: mpi: ### Workaround until CE provides full MPI substitution. require: 'cray-mpich' + xpmem: + require: + - '~kernel-module' + cray-gtl: + buildable: True + cray-mpich: + buildable: True + cray-pals: + buildable: True + cray-pmi: + buildable: True blaspp: variants: - '~cuda' @@ -38,14 +49,10 @@ packages: openblas: variants: - 'threads=openmp' - openssl: - # without version spack concretizes openssl@3.3.1 that doesn't build due to deprecation - require: - - '@3.3.2' libfabric: # Fix version to be compatible with version on todi require: - - '@1.15.2' + - '@1.22.0' hwloc: variants: - '~libxml2' diff --git a/ci/docker/common.yaml b/ci/docker/common.yaml index fb8e197404..07eff5b616 100644 --- a/ci/docker/common.yaml +++ b/ci/docker/common.yaml @@ -20,7 +20,7 @@ packages: scalapack: require: 'intel-oneapi-mkl' mpi: - require: 'mpich' + require: 'cray-mpich' blaspp: variants: - '~cuda' @@ -28,10 +28,24 @@ packages: - '~rocm' intel-oneapi-mkl: require: - - 'threads=openmp' + - 'threads=none' openblas: variants: - 'threads=openmp' + libfabric: + require: + - '@1.22.0' + xpmem: + require: + - '~kernel-module' + cray-gtl: + buildable: True + cray-mpich: + buildable: True + cray-pals: + buildable: True + cray-pmi: + buildable: True mpich: # Requirement for ABI compatibility on the test system require: diff --git a/ci/docker/release-rocm602-stdexec.yaml b/ci/docker/release-rocm602-stdexec.yaml index 1edfa39873..3272c54a75 100644 --- a/ci/docker/release-rocm602-stdexec.yaml +++ b/ci/docker/release-rocm602-stdexec.yaml @@ -40,6 +40,11 @@ spack: externals: - spec: llvm-amdgpu@6.0.2 ~rocm-device-libs prefix: /opt/rocm-6.0.2/llvm + extra_attributes: + compilers: + c: /opt/rocm-6.0.2/llvm/bin/amdclang + cxx: /opt/rocm-6.0.2/llvm/bin/amdclang++ + fortran: /opt/rocm-6.0.2/llvm/bin/amdflang buildable: false rocm-device-libs: externals: diff --git a/ci/docker/release-rocm602.yaml b/ci/docker/release-rocm602.yaml index c185a68913..fcf4fed37c 100644 --- a/ci/docker/release-rocm602.yaml +++ b/ci/docker/release-rocm602.yaml @@ -37,6 +37,11 @@ spack: externals: - spec: llvm-amdgpu@6.0.2 ~rocm-device-libs prefix: /opt/rocm-6.0.2/llvm + extra_attributes: + compilers: + c: /opt/rocm-6.0.2/llvm/bin/amdclang + cxx: /opt/rocm-6.0.2/llvm/bin/amdclang++ + fortran: /opt/rocm-6.0.2/llvm/bin/amdflang buildable: false rocm-device-libs: externals: diff --git a/test/unit/common/test_pipeline.cpp b/test/unit/common/test_pipeline.cpp index ddfc8a7583..6e7e557a0a 100644 --- a/test/unit/common/test_pipeline.cpp +++ b/test/unit/common/test_pipeline.cpp @@ -398,8 +398,14 @@ TEST(SubPipeline, BasicReadonlyParentAccess) { // The first parent access and all sub pipeline accesses should complete here. The last parent access // should not complete until the sub pipeline has been reset. auto checkpointparent_last_started = ex::ensure_started(std::move(checkpointparent_last)); + // pika 0.33.0 changed the order in which continuations are triggered. +#if PIKA_VERSION_FULL >= 0x002100 // >= 0.33.0 + tt::sync_wait(ex::when_all(std::move(checkpoint2), std::move(checkpoint0), std::move(checkpoint1), + std::move(checkpointparent_first))); +#else tt::sync_wait(ex::when_all(std::move(checkpoint1), std::move(checkpoint0), std::move(checkpoint2), std::move(checkpointparent_first))); +#endif // Since the last parent access will be run as an inline continuation and the access was eagerly // started, it should be triggered by the reset of the sub pipeline even without a sync_wait. EXPECT_TRUE(first_parent_access_done);