From c36412c81a733fa352d35606990d7db2234c7a83 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 20:50:10 +0100 Subject: [PATCH 001/122] updates to base branch for nessi.no-2022.11-dev --- EESSI-pilot-install-software.sh | 481 +++++++++++++++++--------------- eessi-2022.11.yml | 5 + 2 files changed, 262 insertions(+), 224 deletions(-) create mode 100644 eessi-2022.11.yml diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 803ac57554..5d8ab7678f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -137,7 +137,19 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -REQ_EB_VERSION='4.5.0' +echo "EESSI_CVMFS_REPO=${EESSI_CVMFS_REPO}" +case ${EESSI_CVMFS_REPO} in + /cvmfs/pilot.eessi-hpc.org*) + REQ_EB_VERSION='4.5.0' + ;; + /cvmfs/pilot.nessi.no*) + REQ_EB_VERSION='4.6.2' + ;; + *) + fatal_error "unsupported CVMFS repository '${EESSI_CVMFS_REPO}'" +esac +echo "REQ_EB_VERSION=${REQ_EB_VERSION}" +module avail 2>&1 | grep -i easybuild echo ">> Checking for EasyBuild module..." ml_av_easybuild_out=$TMPDIR/ml_av_easybuild.out @@ -206,230 +218,247 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -ok_msg="Java installed, off to a good (?) start!" -fail_msg="Failed to install Java, woopsie..." -$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -export GCC_EC="GCC-9.3.0.eb" -echo ">> Starting slow with ${GCC_EC}..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -echo ">> Install CMake with fixed easyblock to take into account --sysroot" -ok_msg="CMake installed!" -fail_msg="Installation of CMake failed, what the ..." -$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -echo ">> Installing OpenBLAS..." -ok_msg="Done with OpenBLAS!" -fail_msg="Installation of OpenBLAS failed!" -if [[ $GENERIC -eq 1 ]]; then - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -else - openblas_include_easyblocks_from_pr='' -fi -$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OpenMPI..." -ok_msg="OpenMPI installed, w00!" -fail_msg="Installation of OpenMPI failed, that's not good..." -$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -echo ">> Install Python 2.7.18 and Python 3.8.2..." -ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -fail_msg="Installation of Python failed, oh no..." -$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Perl..." -ok_msg="Perl installed, making progress..." -fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -echo ">> Installing SciPy-bundle" -ok_msg="SciPy-bundle installed, yihaa!" -fail_msg="SciPy-bundle installation failed, bummer..." -SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -if [[ "$(uname -m)" == "aarch64" ]]; then - $EB $SCIPY_EC --robot --skip-test-step -else - $EB $SCIPY_EC --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing GROMACS..." -ok_msg="GROMACS installed, wow!" -fail_msg="Installation of GROMACS failed, damned..." -$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -echo ">> Installing OpenFOAM (twice!)..." -ok_msg="OpenFOAM installed, now we're talking!" -fail_msg="Installation of OpenFOAM failed, we were so close..." -$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing TensorFlow 2.3.1..." -ok_msg="TensorFlow 2.3.1 installed, w00!" -fail_msg="Installation of TensorFlow failed, why am I not surprised..." -$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Horovod 0.21.3..." -ok_msg="Horovod installed! Go do some parallel training!" -fail_msg="Horovod installation failed. There comes the headache..." -$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - - echo ">> Installing code-server 3.7.3..." - ok_msg="code-server 3.7.3 installed, now you can use VS Code!" - fail_msg="Installation of code-server failed, that's going to be hard to fix..." - $EB code-server-3.7.3.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - -echo ">> Installing RStudio-Server 1.3.1093..." -ok_msg="RStudio-Server installed, enjoy!" -fail_msg="Installation of RStudio-Server failed, might be OS deps..." -$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing IPython 7.15.0..." -ok_msg="IPython installed, launch your Jupyter Notebooks!" -fail_msg="Installation of IPython failed, that's unexpected..." -$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing R 4.1.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing EasyBuild 4.5.1..." -ok_msg="EasyBuild v4.5.1 installed" -fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -echo ">> Installing SciPy-bundle with foss/2021a..." -ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -fi - -$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - +################################################################################ +# COMMENT OUT EB install sections below to not hit GH rate limit because of +# *from-pr arguments +################################################################################ +##### install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, +##### see https://github.com/EESSI/software-layer/issues/123 +##### and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 +####ok_msg="Java installed, off to a good (?) start!" +####fail_msg="Failed to install Java, woopsie..." +####$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install GCC for foss/2020a +####export GCC_EC="GCC-9.3.0.eb" +####echo ">> Starting slow with ${GCC_EC}..." +####ok_msg="${GCC_EC} installed, yippy! Off to a good start..." +####fail_msg="Installation of ${GCC_EC} failed!" +##### pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, +##### which includes patch to fix build of GCC 9.3 when recent kernel headers are in place +####$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install CMake with custom easyblock that patches CMake when --sysroot is used +####echo ">> Install CMake with fixed easyblock to take into account --sysroot" +####ok_msg="CMake installed!" +####fail_msg="Installation of CMake failed, what the ..." +####$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 +####echo ">> Installing OpenBLAS..." +####ok_msg="Done with OpenBLAS!" +####fail_msg="Installation of OpenBLAS failed!" +####if [[ $GENERIC -eq 1 ]]; then +#### echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." +#### openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" +####else +#### openblas_include_easyblocks_from_pr='' +####fi +####$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OpenMPI..." +####ok_msg="OpenMPI installed, w00!" +####fail_msg="Installation of OpenMPI failed, that's not good..." +####$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install Python +####echo ">> Install Python 2.7.18 and Python 3.8.2..." +####ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" +####fail_msg="Installation of Python failed, oh no..." +####$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Perl..." +####ok_msg="Perl installed, making progress..." +####fail_msg="Installation of Perl failed, this never happens..." +##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +####$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Qt5..." +####ok_msg="Qt5 installed, phieuw, that was a big one!" +####fail_msg="Installation of Qt5 failed, that's frustrating..." +####$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### skip test step when installing SciPy-bundle on aarch64, +##### to dance around problem with broken numpy tests; +##### cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 +####echo ">> Installing SciPy-bundle" +####ok_msg="SciPy-bundle installed, yihaa!" +####fail_msg="SciPy-bundle installation failed, bummer..." +####SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb +####if [[ "$(uname -m)" == "aarch64" ]]; then +#### $EB $SCIPY_EC --robot --skip-test-step +####else +#### $EB $SCIPY_EC --robot +####fi +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing GROMACS..." +####ok_msg="GROMACS installed, wow!" +####fail_msg="Installation of GROMACS failed, damned..." +####$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! +##### 32GB is sufficient to build with 16 cores +####echo ">> Installing OpenFOAM (twice!)..." +####ok_msg="OpenFOAM installed, now we're talking!" +####fail_msg="Installation of OpenFOAM failed, we were so close..." +####$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +#### echo ">> Installing QuantumESPRESSO..." +#### ok_msg="QuantumESPRESSO installed, let's go quantum!" +#### fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" +#### $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot +#### check_exit_code $? "${ok_msg}" "${fail_msg}" +####fi +#### +####echo ">> Installing R 4.0.0 (better be patient)..." +####ok_msg="R installed, wow!" +####fail_msg="Installation of R failed, so sad..." +####$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Bioconductor 3.11 bundle..." +####ok_msg="Bioconductor installed, enjoy!" +####fail_msg="Installation of Bioconductor failed, that's annoying..." +####$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing TensorFlow 2.3.1..." +####ok_msg="TensorFlow 2.3.1 installed, w00!" +####fail_msg="Installation of TensorFlow failed, why am I not surprised..." +####$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Horovod 0.21.3..." +####ok_msg="Horovod installed! Go do some parallel training!" +####fail_msg="Horovod installation failed. There comes the headache..." +####$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +#### +#### echo ">> Installing code-server 3.7.3..." +#### ok_msg="code-server 3.7.3 installed, now you can use VS Code!" +#### fail_msg="Installation of code-server failed, that's going to be hard to fix..." +#### $EB code-server-3.7.3.eb --robot +#### check_exit_code $? "${ok_msg}" "${fail_msg}" +####fi +#### +####echo ">> Installing RStudio-Server 1.3.1093..." +####ok_msg="RStudio-Server installed, enjoy!" +####fail_msg="Installation of RStudio-Server failed, might be OS deps..." +####$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." +####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +####$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Spark 3.1.1..." +####ok_msg="Spark installed, set off the fireworks!" +####fail_msg="Installation of Spark failed, no fireworks this time..." +####$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing IPython 7.15.0..." +####ok_msg="IPython installed, launch your Jupyter Notebooks!" +####fail_msg="Installation of IPython failed, that's unexpected..." +####$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing WRF 3.9.1.1..." +####ok_msg="WRF installed, it's getting hot in here!" +####fail_msg="Installation of WRF failed, that's unexpected..." +####OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing R 4.1.0 (better be patient)..." +####ok_msg="R installed, wow!" +####fail_msg="Installation of R failed, so sad..." +####$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Nextflow 22.10.1..." +####ok_msg="Nextflow installed, the work must flow..." +####fail_msg="Installation of Nextflow failed, that's unexpected..." +####$EB -r --from-pr 16531 Nextflow-22.10.1.eb +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." +####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +####$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing EasyBuild 4.5.1..." +####ok_msg="EasyBuild v4.5.1 installed" +####fail_msg="EasyBuild v4.5.1 failed to install" +####$EB --from-pr 14545 --include-easyblocks-from-pr 2805 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 +####check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" +#### +####echo ">> Installing SciPy-bundle with foss/2021a..." +####ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" +####fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." +##### use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 +##### which includes patch to fix installation with recent Linux kernel headers +####$EB --from-pr 14454 GCCcore-10.3.0.eb --robot +##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +####$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 +##### use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set +##### from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 +####$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +##### use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 +##### that includes patch to fix bootstrap problem when using alternate sysroot +####$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +##### use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 +##### which includes a patch to fix installation on POWER +####$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot +##### ignore failing FlexiBLAS tests when building on POWER; +##### some tests are failing due to a segmentation fault due to "invalid memory reference", +##### see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; +##### using -fstack-protector-strong -fstack-clash-protection should fix that, +##### but it doesn't for some reason when building for ppc64le/generic... +####if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then +#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure +####else +#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb +####fi +#### +####$EB SciPy-bundle-2021.05-foss-2021a.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" + + +################################################################################ ### add packages here +################################################################################ +## example block showing a few debugging means +#echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." +#ok_msg="CaDiCaL installed. Nice!" +#fail_msg="Installation of CaDiCaL failed, that's unexpected..." +#$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot --disable-cleanup-tmpdir +#exit_code=$? +#$EB --last-log +#cat $($EB --last-log) +#check_exit_code $exit_code "${ok_msg}" "${fail_msg}" + echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" @@ -442,5 +471,9 @@ $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} $TOPDIR/check_missing_installations.sh +# TODO do not clean up by default (or provide a cmd line arg and when run +# from bot, do not clean up) echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} + +exit 0 diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml new file mode 100644 index 0000000000..16941b6211 --- /dev/null +++ b/eessi-2022.11.yml @@ -0,0 +1,5 @@ +software: + OpenMPI: + toolchains: + GCC-9.3.0: + version: ['4.0.3'] From 78f325ed3559286c61843576b0dc8d83ec9df9f7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 20:58:01 +0100 Subject: [PATCH 002/122] fix issues in eessi_container.sh --- eessi_container.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1eb630a096..09691c822b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -252,7 +252,6 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then # skip creating a new tmp directory, just set environment variables echo "Resuming from previous run using temporary storage at ${RESUME}" EESSI_HOST_STORAGE=${RESUME} - echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" else # we need a tmp location (and possibly init it with ${RESUME} if it was not # a directory @@ -284,6 +283,7 @@ else EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." fi +echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then @@ -488,7 +488,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi # if INFO is set to 1 (arg --info), add argument '-q' -RUN_QUIET=${INFO:--q} +if [[ -z ${INFO} ]]; then + RUN_QUIET='-q' +else + RUN_QUIET='' +fi + echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level From 0ce3e09019e1021936ab77374d33e381500a90e4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 21:18:28 +0100 Subject: [PATCH 003/122] fix easystack typo in eessi-2022.11.yml --- eessi-2022.11.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 16941b6211..75e6a61a55 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -2,4 +2,4 @@ software: OpenMPI: toolchains: GCC-9.3.0: - version: ['4.0.3'] + versions: ['4.0.3'] From 32aa71135a75a7ce8ec799f5fbff49c3369cb025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Sat, 4 Feb 2023 15:07:23 +0100 Subject: [PATCH 004/122] fix RUN_QUIET setting --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 09691c822b..9a85300bc1 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -487,8 +487,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 1 (arg --info), add argument '-q' -if [[ -z ${INFO} ]]; then +# if INFO is set to 0 (no arg --info), add argument '-q' +if [[ ${INFO} -eq 0 ]]; then RUN_QUIET='-q' else RUN_QUIET='' From a524d91131163d52cc4c631d6b8b2b31cc8aaf06 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 4 Feb 2023 15:54:08 +0100 Subject: [PATCH 005/122] adding GCC/11.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 5d8ab7678f..26ac9f7852 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -459,6 +459,15 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #cat $($EB --last-log) #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" +# install GCC for foss/2022a +export GCC_EC="GCC-11.3.0.eb" +echo ">> Starting slow with ${GCC_EC} using '--from-pr 14453' ..." +ok_msg="${GCC_EC} installed, yippy! Off to a good start..." +fail_msg="Installation of ${GCC_EC} failed!" +# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, +# which includes patch to fix build of GCC 11.3.0 when recent kernel headers are in place +$EB --robot --from-pr 14453 ${GCC_EC} +check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 8e2f09cdf4bed1b281e8b0eb63e6c42b3149c991 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 4 Feb 2023 16:14:51 +0100 Subject: [PATCH 006/122] Revert "adding GCC/11.3.0 to NESSI/2022.11" This reverts commit a524d91131163d52cc4c631d6b8b2b31cc8aaf06. --- EESSI-pilot-install-software.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 26ac9f7852..5d8ab7678f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -459,15 +459,6 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #cat $($EB --last-log) #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" -# install GCC for foss/2022a -export GCC_EC="GCC-11.3.0.eb" -echo ">> Starting slow with ${GCC_EC} using '--from-pr 14453' ..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 11.3.0 when recent kernel headers are in place -$EB --robot --from-pr 14453 ${GCC_EC} -check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 7eb8308361117ec4aea8cabbdd70ed2a365aa2d6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 6 Feb 2023 11:14:39 +0100 Subject: [PATCH 007/122] adding latest EB to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 5d8ab7678f..4b0d30a7c1 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -459,6 +459,18 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #cat $($EB --last-log) #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" +## add latest EasyBuild to stack +echo ">> Adding latest EasyBuild to stack. So far we have:" +module --ignore_cache avail EasyBuild 2>&1 | grep EasyBuild +ok_msg="Latest EasyBuild got installed ... great!" +fail_msg="Installation of latest EasyBuild failed! Disappointed." +$EB --robot --install-latest-eb-release +exit_code=$? +# get what versions are available now +echo ">> Now, we have:" +module --ignore_cache avail EasyBuild 2>&1 | grep EasyBuild +check_exit_code $? "${ok_msg}" "${fail_msg}" + echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 23e773ce0d8c6504e1440629a4178318265b73d2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 Feb 2023 19:07:54 +0100 Subject: [PATCH 008/122] improvements to resuming job environment --- bot/build.sh | 10 +++++----- eessi_container.sh | 7 +++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8565511bfa..2c15b904f3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -mkdir -p previous_tmp +mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh --access rw" @@ -151,7 +151,7 @@ echo " ${HTTPS_PROXY_OPT}" echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg @@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg --info \ --mode run \ ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} @@ -190,7 +190,7 @@ echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/tarball_step" echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ @@ -200,7 +200,7 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT --mode run \ ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/tarball_step \ ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index 9a85300bc1..b8a96d462c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -354,6 +354,9 @@ else # standard EESSI repositories) cfg_load ${EESSI_REPOS_CFG_FILE} + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } # @@ -487,8 +490,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 0 (no arg --info), add argument '-q' -if [[ ${INFO} -eq 0 ]]; then +# if INFO is set to 1 (arg --info), add argument '-q' +if [[ -z ${INFO} ]]; then RUN_QUIET='-q' else RUN_QUIET='' From d4504c235436bafba42b7ed023ea9c9ede718508 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 Feb 2023 19:27:07 +0100 Subject: [PATCH 009/122] address requested change --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index b8a96d462c..3c017d7510 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -490,8 +490,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 1 (arg --info), add argument '-q' -if [[ -z ${INFO} ]]; then +# if INFO is set to 0 (no arg --info), add argument '-q' +if [[ ${INFO} -eq 0 ]]; then RUN_QUIET='-q' else RUN_QUIET='' From f7f79808a08c806e6d970d5ac467821d7fc3d473 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 10 Feb 2023 15:11:17 +0100 Subject: [PATCH 010/122] removing unnecessary debug information --- EESSI-pilot-install-software.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4b0d30a7c1..fe2e3a3f2f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -460,16 +460,12 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" ## add latest EasyBuild to stack -echo ">> Adding latest EasyBuild to stack. So far we have:" -module --ignore_cache avail EasyBuild 2>&1 | grep EasyBuild +echo ">> Adding latest EasyBuild to stack..." ok_msg="Latest EasyBuild got installed ... great!" fail_msg="Installation of latest EasyBuild failed! Disappointed." $EB --robot --install-latest-eb-release exit_code=$? -# get what versions are available now -echo ">> Now, we have:" -module --ignore_cache avail EasyBuild 2>&1 | grep EasyBuild -check_exit_code $? "${ok_msg}" "${fail_msg}" +check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" echo ">> Creating/updating Lmod cache..." From ad0b37773a6c6435795aa60dd10679cc9ab0da17 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 10 Feb 2023 17:11:07 +0100 Subject: [PATCH 011/122] special handling of EB when building for EESSI/2021.12 --- EESSI-pilot-install-software.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index fe2e3a3f2f..df9f9d7188 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -463,7 +463,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL echo ">> Adding latest EasyBuild to stack..." ok_msg="Latest EasyBuild got installed ... great!" fail_msg="Installation of latest EasyBuild failed! Disappointed." -$EB --robot --install-latest-eb-release +if [[ ${EESSI_CVMFS_REPO} == /cvmfs/pilot.eessi-hpc.org ]]; then + $EB --from-pr 14545 --include-easyblocks-from-pr 2805 --robot --install-latest-eb-release +else + $EB --robot --install-latest-eb-release +fi exit_code=$? check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" From 35d02612f206469789a5432576053b75972566b4 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 14 Feb 2023 11:14:19 +0000 Subject: [PATCH 012/122] Setting EasyBuild version_to 4.7.0 and modifying the YML file format --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index df9f9d7188..0c33cabc39 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -143,7 +143,7 @@ case ${EESSI_CVMFS_REPO} in REQ_EB_VERSION='4.5.0' ;; /cvmfs/pilot.nessi.no*) - REQ_EB_VERSION='4.6.2' + REQ_EB_VERSION='4.7.0' ;; *) fatal_error "unsupported CVMFS repository '${EESSI_CVMFS_REPO}'" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 75e6a61a55..2add7fb98b 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,5 +1 @@ -software: - OpenMPI: - toolchains: - GCC-9.3.0: - versions: ['4.0.3'] +easyconfigs: From 0496c8d33f30e73c9e1ee8eb110b63cec14500fa Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 14 Feb 2023 11:59:23 +0000 Subject: [PATCH 013/122] modifying the YML file format --- eessi-2022.11.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2add7fb98b..5614c3d17f 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1 +1,2 @@ easyconfigs: + - EasyBuild-4.7.0.eb From 3cff9c61324d37a88cf7390d82fe549384bc244b Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 14 Feb 2023 18:57:50 +0000 Subject: [PATCH 014/122] CMake-3.20.1-GCCcore-10.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 0c33cabc39..d8daad899f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -424,7 +424,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL ####$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 ##### use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set ##### from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -####$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ##### use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 ##### that includes patch to fix bootstrap problem when using alternate sysroot ####$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 5614c3d17f..41b0656a8f 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,2 +1,3 @@ easyconfigs: - EasyBuild-4.7.0.eb + - CMake-3.20.1-GCCcore-10.3.0.eb From e7e4cea1bf93ff2ff1901060792a1f05ae7dd6d6 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Feb 2023 08:17:22 +0000 Subject: [PATCH 015/122] replaced #### with # --- EESSI-pilot-install-software.sh | 450 ++++++++++++++++---------------- 1 file changed, 225 insertions(+), 225 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index d8daad899f..bfd95c144d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -222,234 +222,234 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL # COMMENT OUT EB install sections below to not hit GH rate limit because of # *from-pr arguments ################################################################################ -##### install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -##### see https://github.com/EESSI/software-layer/issues/123 -##### and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -####ok_msg="Java installed, off to a good (?) start!" -####fail_msg="Failed to install Java, woopsie..." -####$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### install GCC for foss/2020a -####export GCC_EC="GCC-9.3.0.eb" -####echo ">> Starting slow with ${GCC_EC}..." -####ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -####fail_msg="Installation of ${GCC_EC} failed!" -##### pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -##### which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -####$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### install CMake with custom easyblock that patches CMake when --sysroot is used -####echo ">> Install CMake with fixed easyblock to take into account --sysroot" -####ok_msg="CMake installed!" -####fail_msg="Installation of CMake failed, what the ..." -####$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -####echo ">> Installing OpenBLAS..." -####ok_msg="Done with OpenBLAS!" -####fail_msg="Installation of OpenBLAS failed!" -####if [[ $GENERIC -eq 1 ]]; then -#### echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." -#### openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -####else -#### openblas_include_easyblocks_from_pr='' -####fi -####$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing OpenMPI..." -####ok_msg="OpenMPI installed, w00!" -####fail_msg="Installation of OpenMPI failed, that's not good..." -####$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### install Python -####echo ">> Install Python 2.7.18 and Python 3.8.2..." -####ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -####fail_msg="Installation of Python failed, oh no..." -####$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Perl..." -####ok_msg="Perl installed, making progress..." -####fail_msg="Installation of Perl failed, this never happens..." -##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -####$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Qt5..." -####ok_msg="Qt5 installed, phieuw, that was a big one!" -####fail_msg="Installation of Qt5 failed, that's frustrating..." -####$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### skip test step when installing SciPy-bundle on aarch64, -##### to dance around problem with broken numpy tests; -##### cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -####echo ">> Installing SciPy-bundle" -####ok_msg="SciPy-bundle installed, yihaa!" -####fail_msg="SciPy-bundle installation failed, bummer..." -####SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -####if [[ "$(uname -m)" == "aarch64" ]]; then -#### $EB $SCIPY_EC --robot --skip-test-step -####else -#### $EB $SCIPY_EC --robot -####fi -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing GROMACS..." -####ok_msg="GROMACS installed, wow!" -####fail_msg="Installation of GROMACS failed, damned..." -####$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -##### note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -##### 32GB is sufficient to build with 16 cores -####echo ">> Installing OpenFOAM (twice!)..." -####ok_msg="OpenFOAM installed, now we're talking!" -####fail_msg="Installation of OpenFOAM failed, we were so close..." -####$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then -#### echo ">> Installing QuantumESPRESSO..." -#### ok_msg="QuantumESPRESSO installed, let's go quantum!" -#### fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" -#### $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot -#### check_exit_code $? "${ok_msg}" "${fail_msg}" -####fi -#### -####echo ">> Installing R 4.0.0 (better be patient)..." -####ok_msg="R installed, wow!" -####fail_msg="Installation of R failed, so sad..." -####$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Bioconductor 3.11 bundle..." -####ok_msg="Bioconductor installed, enjoy!" -####fail_msg="Installation of Bioconductor failed, that's annoying..." -####$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing TensorFlow 2.3.1..." -####ok_msg="TensorFlow 2.3.1 installed, w00!" -####fail_msg="Installation of TensorFlow failed, why am I not surprised..." -####$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Horovod 0.21.3..." -####ok_msg="Horovod installed! Go do some parallel training!" -####fail_msg="Horovod installation failed. There comes the headache..." -####$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then -#### -#### echo ">> Installing code-server 3.7.3..." -#### ok_msg="code-server 3.7.3 installed, now you can use VS Code!" -#### fail_msg="Installation of code-server failed, that's going to be hard to fix..." -#### $EB code-server-3.7.3.eb --robot -#### check_exit_code $? "${ok_msg}" "${fail_msg}" -####fi -#### -####echo ">> Installing RStudio-Server 1.3.1093..." -####ok_msg="RStudio-Server installed, enjoy!" -####fail_msg="Installation of RStudio-Server failed, might be OS deps..." -####$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -####$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Spark 3.1.1..." -####ok_msg="Spark installed, set off the fireworks!" -####fail_msg="Installation of Spark failed, no fireworks this time..." -####$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing IPython 7.15.0..." -####ok_msg="IPython installed, launch your Jupyter Notebooks!" -####fail_msg="Installation of IPython failed, that's unexpected..." -####$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing WRF 3.9.1.1..." -####ok_msg="WRF installed, it's getting hot in here!" -####fail_msg="Installation of WRF failed, that's unexpected..." -####OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing R 4.1.0 (better be patient)..." -####ok_msg="R installed, wow!" -####fail_msg="Installation of R failed, so sad..." -####$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing Nextflow 22.10.1..." -####ok_msg="Nextflow installed, the work must flow..." -####fail_msg="Installation of Nextflow failed, that's unexpected..." -####$EB -r --from-pr 16531 Nextflow-22.10.1.eb -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -####$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####echo ">> Installing EasyBuild 4.5.1..." -####ok_msg="EasyBuild v4.5.1 installed" -####fail_msg="EasyBuild v4.5.1 failed to install" -####$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -####check_exit_code $? "${ok_msg}" "${fail_msg}" -#### -####LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -####check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" -#### -####echo ">> Installing SciPy-bundle with foss/2021a..." -####ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -####fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -##### use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -##### which includes patch to fix installation with recent Linux kernel headers -####$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -####$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -##### use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -##### from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 +# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, +# see https://github.com/EESSI/software-layer/issues/123 +# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 +#ok_msg="Java installed, off to a good (?) start!" +#fail_msg="Failed to install Java, woopsie..." +#$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# install GCC for foss/2020a +#export GCC_EC="GCC-9.3.0.eb" +#echo ">> Starting slow with ${GCC_EC}..." +#ok_msg="${GCC_EC} installed, yippy! Off to a good start..." +#fail_msg="Installation of ${GCC_EC} failed!" +## pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, +## which includes patch to fix build of GCC 9.3 when recent kernel headers are in place +#$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# install CMake with custom easyblock that patches CMake when --sysroot is used +#echo ">> Install CMake with fixed easyblock to take into account --sysroot" +#ok_msg="CMake installed!" +#fail_msg="Installation of CMake failed, what the ..." +#$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 +#echo ">> Installing OpenBLAS..." +#ok_msg="Done with OpenBLAS!" +#fail_msg="Installation of OpenBLAS failed!" +#if [[ $GENERIC -eq 1 ]]; then +# echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." +# openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" +#else +# openblas_include_easyblocks_from_pr='' +#fi +#$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing OpenMPI..." +#ok_msg="OpenMPI installed, w00!" +#fail_msg="Installation of OpenMPI failed, that's not good..." +#$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# install Python +#echo ">> Install Python 2.7.18 and Python 3.8.2..." +#ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" +#fail_msg="Installation of Python failed, oh no..." +#$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Perl..." +#ok_msg="Perl installed, making progress..." +#fail_msg="Installation of Perl failed, this never happens..." +# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +## to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +#$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Qt5..." +#ok_msg="Qt5 installed, phieuw, that was a big one!" +#fail_msg="Installation of Qt5 failed, that's frustrating..." +#$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# skip test step when installing SciPy-bundle on aarch64, +# to dance around problem with broken numpy tests; +# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 +#echo ">> Installing SciPy-bundle" +#ok_msg="SciPy-bundle installed, yihaa!" +#fail_msg="SciPy-bundle installation failed, bummer..." +#SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb +#if [[ "$(uname -m)" == "aarch64" ]]; then +# $EB $SCIPY_EC --robot --skip-test-step +#else +# $EB $SCIPY_EC --robot +#fi +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing GROMACS..." +#ok_msg="GROMACS installed, wow!" +#fail_msg="Installation of GROMACS failed, damned..." +#$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! +# 32GB is sufficient to build with 16 cores +#echo ">> Installing OpenFOAM (twice!)..." +#ok_msg="OpenFOAM installed, now we're talking!" +#fail_msg="Installation of OpenFOAM failed, we were so close..." +#$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +# echo ">> Installing QuantumESPRESSO..." +# ok_msg="QuantumESPRESSO installed, let's go quantum!" +# fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" +# $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot +# check_exit_code $? "${ok_msg}" "${fail_msg}" +#fi + +#echo ">> Installing R 4.0.0 (better be patient)..." +#ok_msg="R installed, wow!" +#fail_msg="Installation of R failed, so sad..." +#$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Bioconductor 3.11 bundle..." +#ok_msg="Bioconductor installed, enjoy!" +#fail_msg="Installation of Bioconductor failed, that's annoying..." +#$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing TensorFlow 2.3.1..." +#ok_msg="TensorFlow 2.3.1 installed, w00!" +#fail_msg="Installation of TensorFlow failed, why am I not surprised..." +#$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Horovod 0.21.3..." +#ok_msg="Horovod installed! Go do some parallel training!" +#fail_msg="Horovod installation failed. There comes the headache..." +#$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then + +# echo ">> Installing code-server 3.7.3..." +# ok_msg="code-server 3.7.3 installed, now you can use VS Code!" +# fail_msg="Installation of code-server failed, that's going to be hard to fix..." +# $EB code-server-3.7.3.eb --robot +# check_exit_code $? "${ok_msg}" "${fail_msg}" +#fi + +#echo ">> Installing RStudio-Server 1.3.1093..." +#ok_msg="RStudio-Server installed, enjoy!" +#fail_msg="Installation of RStudio-Server failed, might be OS deps..." +#$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." +#ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +#fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +#$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Spark 3.1.1..." +#ok_msg="Spark installed, set off the fireworks!" +#fail_msg="Installation of Spark failed, no fireworks this time..." +#$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing IPython 7.15.0..." +#ok_msg="IPython installed, launch your Jupyter Notebooks!" +#fail_msg="Installation of IPython failed, that's unexpected..." +#$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing WRF 3.9.1.1..." +#ok_msg="WRF installed, it's getting hot in here!" +#fail_msg="Installation of WRF failed, that's unexpected..." +#OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing R 4.1.0 (better be patient)..." +#ok_msg="R installed, wow!" +#fail_msg="Installation of R failed, so sad..." +#$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing Nextflow 22.10.1..." +#ok_msg="Nextflow installed, the work must flow..." +#fail_msg="Installation of Nextflow failed, that's unexpected..." +#$EB -r --from-pr 16531 Nextflow-22.10.1.eb +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." +#ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +#fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +#$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#echo ">> Installing EasyBuild 4.5.1..." +#ok_msg="EasyBuild v4.5.1 installed" +#fail_msg="EasyBuild v4.5.1 failed to install" +#$EB --from-pr 14545 --include-easyblocks-from-pr 2805 +#check_exit_code $? "${ok_msg}" "${fail_msg}" + +#LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 +#check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" + +#echo ">> Installing SciPy-bundle with foss/2021a..." +#ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" +#fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." +# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 +# which includes patch to fix installation with recent Linux kernel headers +#$EB --from-pr 14454 GCCcore-10.3.0.eb --robot +# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +#$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 +#use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set +#from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -##### use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -##### that includes patch to fix bootstrap problem when using alternate sysroot -####$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -##### use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -##### which includes a patch to fix installation on POWER -####$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -##### ignore failing FlexiBLAS tests when building on POWER; -##### some tests are failing due to a segmentation fault due to "invalid memory reference", -##### see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -##### using -fstack-protector-strong -fstack-clash-protection should fix that, -##### but it doesn't for some reason when building for ppc64le/generic... -####if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then -#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -####else -#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -####fi -#### -####$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -####check_exit_code $? "${ok_msg}" "${fail_msg}" +# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 +# that includes patch to fix bootstrap problem when using alternate sysroot +#$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 +# which includes a patch to fix installation on POWER +#$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot +# ignore failing FlexiBLAS tests when building on POWER; +# some tests are failing due to a segmentation fault due to "invalid memory reference", +# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; +# using -fstack-protector-strong -fstack-clash-protection should fix that, +# but it doesn't for some reason when building for ppc64le/generic... +#if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then +# $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure +#else +# $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb +#fi +# +#$EB SciPy-bundle-2021.05-foss-2021a.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" -################################################################################ +##################### ### add packages here -################################################################################ -## example block showing a few debugging means +##################### +# example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" #fail_msg="Installation of CaDiCaL failed, that's unexpected..." @@ -459,7 +459,7 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 #cat $($EB --last-log) #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" -## add latest EasyBuild to stack +# add latest EasyBuild to stack echo ">> Adding latest EasyBuild to stack..." ok_msg="Latest EasyBuild got installed ... great!" fail_msg="Installation of latest EasyBuild failed! Disappointed." From 999708e4a9a3e9ee3f621fa7e285fe470791a1e7 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Feb 2023 08:27:11 +0000 Subject: [PATCH 016/122] replaced #### with # --- EESSI-pilot-install-software.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index bfd95c144d..9eeff3b6ca 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -1,5 +1,5 @@ #!/bin/bash -# + # Script to install EESSI pilot software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at @@ -235,8 +235,8 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #echo ">> Starting slow with ${GCC_EC}..." #ok_msg="${GCC_EC} installed, yippy! Off to a good start..." #fail_msg="Installation of ${GCC_EC} failed!" -## pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -## which includes patch to fix build of GCC 9.3 when recent kernel headers are in place +# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, +# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place #$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb #check_exit_code $? "${ok_msg}" "${fail_msg}" @@ -277,7 +277,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #ok_msg="Perl installed, making progress..." #fail_msg="Installation of Perl failed, this never happens..." # use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -## to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) #$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 #check_exit_code $? "${ok_msg}" "${fail_msg}" From 4c9c3e7589c8ea3f1ae5bcf7a2e5a1b3f4dbcde2 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Feb 2023 15:56:52 +0000 Subject: [PATCH 017/122] Add Python-3.9.5-GCCcore-10.3.0-bare.eb to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 3 ++- eessi-2022.11.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 9eeff3b6ca..42f062e25d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -424,7 +424,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 #use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set #from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +#$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 # use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 # that includes patch to fix bootstrap problem when using alternate sysroot #$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot @@ -449,6 +449,7 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ##################### ### add packages here ##################### +$EB Python-3.9.5-GCCcore-10.3.0-bare.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 41b0656a8f..9301b75ac1 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,3 +1,3 @@ easyconfigs: - EasyBuild-4.7.0.eb - - CMake-3.20.1-GCCcore-10.3.0.eb + - Python-3.9.5-GCCcore-10.3.0-bare.eb From 69151fe14021fc3f366eaa6a1a29fb7655736ef6 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 16 Feb 2023 11:34:52 +0000 Subject: [PATCH 018/122] Added back CMake in both EESSI-pilot-install-software.sh and eessi-2022.11.yml + went back to default Python WITHOUT the bare version --- EESSI-pilot-install-software.sh | 4 ++-- eessi-2022.11.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 42f062e25d..328f0df30e 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -424,7 +424,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 #use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set #from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -#$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 # use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 # that includes patch to fix bootstrap problem when using alternate sysroot #$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot @@ -449,7 +449,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL ##################### ### add packages here ##################### -$EB Python-3.9.5-GCCcore-10.3.0-bare.eb --robot +$EB Python-3.9.5-GCCcore-10.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 9301b75ac1..d637b2ec6c 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,3 +1,4 @@ easyconfigs: - EasyBuild-4.7.0.eb + - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0-bare.eb From 9f93226342c6e0690816131cc95f64149a6a9ae7 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 16 Feb 2023 11:41:35 +0000 Subject: [PATCH 019/122] Fixed Python version in yml file --- eessi-2022.11.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index d637b2ec6c..878fe4f12c 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,4 +1,4 @@ easyconfigs: - EasyBuild-4.7.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - - Python-3.9.5-GCCcore-10.3.0-bare.eb + - Python-3.9.5-GCCcore-10.3.0.eb From 58d28c6ae827dd00a4b398f4dce052b9f6a1e16a Mon Sep 17 00:00:00 2001 From: Dan Jonsson Date: Fri, 17 Feb 2023 11:06:03 +0100 Subject: [PATCH 020/122] Adding Qt5/5.14.1-GCCcore-9.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 10 +++++----- eessi-2022.11.yml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 328f0df30e..67f940745c 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -281,11 +281,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 #check_exit_code $? "${ok_msg}" "${fail_msg}" -#echo ">> Installing Qt5..." -#ok_msg="Qt5 installed, phieuw, that was a big one!" -#fail_msg="Installation of Qt5 failed, that's frustrating..." -#$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing Qt5..." +ok_msg="Qt5 installed, phieuw, that was a big one!" +fail_msg="Installation of Qt5 failed, that's frustrating..." +$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot +check_exit_code $? "${ok_msg}" "${fail_msg}" # skip test step when installing SciPy-bundle on aarch64, # to dance around problem with broken numpy tests; diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 878fe4f12c..4577f108b3 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.7.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0.eb + - Qt5-5.14.1-GCCcore-9.3.0.eb From a9622e254f514f7551f9360d8dbc539b6833c690 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 20 Feb 2023 08:22:43 +0000 Subject: [PATCH 021/122] Add OpenMPI/4.1.1 with GCC/10.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 1 + eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 328f0df30e..8fa3394d97 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -450,6 +450,7 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ### add packages here ##################### $EB Python-3.9.5-GCCcore-10.3.0.eb --robot +$EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 878fe4f12c..7833906b7e 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.7.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0.eb + - OpenMPI-4.1.1-GCC-10.3.0.eb From 67df600116e4550aa22a562ba6ee35a9ff615a76 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 20 Feb 2023 08:53:19 +0000 Subject: [PATCH 022/122] Add ImageMagick/7.0.11.14 with GCC/10.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 1 + eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 328f0df30e..b3a7f022d8 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -450,6 +450,7 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ### add packages here ##################### $EB Python-3.9.5-GCCcore-10.3.0.eb --robot +$EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 878fe4f12c..e9e72c2027 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.7.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0.eb + - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb From 6919962d1631aa0ed8a74d262815156a9fec37aa Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 22 Feb 2023 13:34:48 +0000 Subject: [PATCH 023/122] added a clarification comment in EESSI-pilot-install-software.sh --- EESSI-pilot-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f79961d2a8..8a646c07f5 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -451,6 +451,7 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ##################### $EB Python-3.9.5-GCCcore-10.3.0.eb --robot $EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot +# this Package has been added to reduce the complexity of building large packages such as R $EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." From 947eac995cf98314767a180cd3214221a67a74eb Mon Sep 17 00:00:00 2001 From: parosen Date: Thu, 23 Feb 2023 08:23:00 +0100 Subject: [PATCH 024/122] Adding Nextflow-22.10.1.eb to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8fa3394d97..6df4921860 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -392,11 +392,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental #check_exit_code $? "${ok_msg}" "${fail_msg}" -#echo ">> Installing Nextflow 22.10.1..." -#ok_msg="Nextflow installed, the work must flow..." -#fail_msg="Installation of Nextflow failed, that's unexpected..." -#$EB -r --from-pr 16531 Nextflow-22.10.1.eb -#check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing Nextflow 22.10.1..." +ok_msg="Nextflow installed, the work must flow..." +fail_msg="Installation of Nextflow failed, that's unexpected..." +$EB -r --from-pr 16531 Nextflow-22.10.1.eb +check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." #ok_msg="OSU-Micro-Benchmarks installed, yihaa!" From f2bb51ec9138fd20b84f4bc6cd3ad51b5e3f9dac Mon Sep 17 00:00:00 2001 From: parosen Date: Thu, 23 Feb 2023 10:58:18 +0100 Subject: [PATCH 025/122] Added ebconfig to yml file and commented use of --from-pr --- EESSI-pilot-install-software.sh | 1 + eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f79f6576fa..06869d3218 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -395,6 +395,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL echo ">> Installing Nextflow 22.10.1..." ok_msg="Nextflow installed, the work must flow..." fail_msg="Installation of Nextflow failed, that's unexpected..." +# Comment from Axel: PR 16531 was merged so --from-pr not needed anymore (but was used in this build) $EB -r --from-pr 16531 Nextflow-22.10.1.eb check_exit_code $? "${ok_msg}" "${fail_msg}" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2a10bd0198..f99ab0901e 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,3 +4,4 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - Nextflow-22.10.1.eb From 1a017cd007f19746b94fc6a6563de5236dc47877 Mon Sep 17 00:00:00 2001 From: parosen Date: Thu, 23 Feb 2023 11:38:43 +0100 Subject: [PATCH 026/122] Adding QuantumESPRESSO to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 14 +++++++------- eessi-2022.11.yml | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8a646c07f5..14ce91e791 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -315,13 +315,13 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" -#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then -# echo ">> Installing QuantumESPRESSO..." -# ok_msg="QuantumESPRESSO installed, let's go quantum!" -# fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" -# $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot -# check_exit_code $? "${ok_msg}" "${fail_msg}" -#fi +if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then + echo ">> Installing QuantumESPRESSO..." + ok_msg="QuantumESPRESSO installed, let's go quantum!" + fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" + $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" +fi #echo ">> Installing R 4.0.0 (better be patient)..." #ok_msg="R installed, wow!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2a10bd0198..b52fd53f68 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,3 +4,4 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - QuantumESPRESSO-6.6-foss-2020a.eb From c0d71380c01476c9b91ef8636bac590076d79c40 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 23 Feb 2023 11:59:53 +0000 Subject: [PATCH 027/122] Add Spark/3.1.1 with GCC/9.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8a646c07f5..3a2cf9542a 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -371,7 +371,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #echo ">> Installing Spark 3.1.1..." #ok_msg="Spark installed, set off the fireworks!" #fail_msg="Installation of Spark failed, no fireworks this time..." -#$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r #check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing IPython 7.15.0..." diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2a10bd0198..1b045247b1 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,3 +4,4 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - Spark-3.1.1-foss-2020a-Python-3.8.2.eb From 5e5f97dec86d67d6a0a43166306274c7bb2f5f13 Mon Sep 17 00:00:00 2001 From: trz42 Date: Thu, 23 Feb 2023 23:56:28 +0100 Subject: [PATCH 028/122] add --generic arg when running install script --- bot/build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 2c15b904f3..0a39b72169 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,6 +141,11 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +GENERIC_OPT= +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then + GENERIC_OPT="--generic" +fi + mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" @@ -153,7 +158,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -165,7 +170,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} + ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") From 30d6e7ec6b2cf88510c7e0b8adefa5b9e56c446c Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:17:57 +0100 Subject: [PATCH 029/122] fix error in regex --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0a39b72169..a7a7f2d589 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -142,7 +142,7 @@ if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi GENERIC_OPT= -if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi From 16094cf1135a00540230af5328c9b452e3095c02 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:42:29 +0100 Subject: [PATCH 030/122] add command separator -- --- eessi_container.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 3c017d7510..c9e4340f43 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -114,6 +114,8 @@ SAVE= HTTP_PROXY=${http_proxy:-} HTTPS_PROXY=${https_proxy:-} +COMMAND_SEPARATOR=0 + POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do @@ -172,8 +174,17 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; + --) + COMMAND_SEPARATOR=1 + shift + ;; -*|--*) - fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + if [[ ${COMMAND_SEPARATOR} -eq 0 ]]; then + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + else + POSITIONAL_ARGS+=("$1") # save positional arg + shift + fi ;; *) # No more options POSITIONAL_ARGS+=("$1") # save positional arg From 46b726890e4d2fc954adeb36a898aa64b41d31d8 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:43:23 +0100 Subject: [PATCH 031/122] use command separator -- --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a7a7f2d589..d5f81cd193 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -158,7 +158,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -170,7 +170,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") From 7f087c449cefb76da93897ba986430fc60f4c046 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 06:57:42 +0100 Subject: [PATCH 032/122] retain eb tmp dir when building Qt5 --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 79d93701f1..5d8efc6c4d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -284,7 +284,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL echo ">> Installing Qt5..." ok_msg="Qt5 installed, phieuw, that was a big one!" fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot +$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot --disable-cleanup-tmpdir check_exit_code $? "${ok_msg}" "${fail_msg}" # skip test step when installing SciPy-bundle on aarch64, From 8a181115fc051d82e8365edb40b8a400a86d4568 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 27 Feb 2023 07:35:45 +0000 Subject: [PATCH 033/122] Add OSU-Micro-Benchmarks/5.6.3 with GCC/9.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8a646c07f5..59a44483b3 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -365,7 +365,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." #ok_msg="OSU-Micro-Benchmarks installed, yihaa!" #fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -#$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r +$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r #check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing Spark 3.1.1..." diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2a10bd0198..9213827345 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,3 +4,4 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb From b58ed879cd82378830ccd371ee404402e41564b1 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 28 Feb 2023 09:06:57 +0000 Subject: [PATCH 034/122] Add OSU-Micro-Benchmarks/5.7.1 with GCC/10.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8a646c07f5..f33ec9bb7e 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -401,7 +401,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." #ok_msg="OSU-Micro-Benchmarks installed, yihaa!" #fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -#$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r #check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing EasyBuild 4.5.1..." diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 2a10bd0198..a4ec98fd0f 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,3 +4,4 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb From 1087943c744c62c28d00d526bc0c505e37e600d8 Mon Sep 17 00:00:00 2001 From: parosen Date: Thu, 2 Mar 2023 10:40:26 +0100 Subject: [PATCH 035/122] added --fetch before installing to mitigate possible corrupted sources --- EESSI-pilot-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 14ce91e791..78bba682db 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -319,6 +319,7 @@ if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then echo ">> Installing QuantumESPRESSO..." ok_msg="QuantumESPRESSO installed, let's go quantum!" fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" + $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot --fetch --force $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi From e516ada807312c1585a2c40beb56cd87f616d8cd Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 3 Mar 2023 07:08:36 +0000 Subject: [PATCH 036/122] Add WRF/3.9.1.1 with GCC/9.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 2 +- eessi-2022.11.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 06869d3218..bdc9cde48f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -383,7 +383,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #echo ">> Installing WRF 3.9.1.1..." #ok_msg="WRF installed, it's getting hot in here!" #fail_msg="Installation of WRF failed, that's unexpected..." -#OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 +OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 #check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing R 4.1.0 (better be patient)..." diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index f99ab0901e..d5109cd48c 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -5,3 +5,4 @@ easyconfigs: - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb - Nextflow-22.10.1.eb + - WRF-3.9.1.1-foss-2020a-dmpar.eb From 6eab9c623fc7c5538e109a026796c2ec461c27e4 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 6 Mar 2023 09:26:43 +0000 Subject: [PATCH 037/122] uncommented the check and some additional information about the package while building WRF --- EESSI-pilot-install-software.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index bdc9cde48f..80c482bf25 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -380,11 +380,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r #check_exit_code $? "${ok_msg}" "${fail_msg}" -#echo ">> Installing WRF 3.9.1.1..." -#ok_msg="WRF installed, it's getting hot in here!" -#fail_msg="Installation of WRF failed, that's unexpected..." +echo ">> Installing WRF 3.9.1.1..." +ok_msg="WRF installed, it's getting hot in here!" +fail_msg="Installation of WRF failed, that's unexpected..." OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -#check_exit_code $? "${ok_msg}" "${fail_msg}" +check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing R 4.1.0 (better be patient)..." #ok_msg="R installed, wow!" From f59bfdededf7b56bc95dc64f19021a3770d8ac61 Mon Sep 17 00:00:00 2001 From: trz42 Date: Tue, 7 Mar 2023 15:33:24 +0100 Subject: [PATCH 038/122] fix check for missing installations --- check_missing_installations.sh | 20 ++++++++++++++++++-- scripts/utils.sh | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e4c75aebd8..45f276dee1 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" + +# the above assesses the installed software for each easyconfig provided in +# the easystack file and then print messages such as +# `No missing modules!` +# or +# `2 out of 3 required modules missing:` +# depending on the result of the assessment. Hence, we need to check if the +# output does not contain any line with ` required modules missing:` + +grep " required modules missing:" ${eb_missing_out} > /dev/null + +# we need to process the result (from finding `No missing modules` to NOT finding +# ` required modules missing:` and no other error happened) +# +# if grep returns 1 (` required modules missing:` was NOT found), we set +# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 +[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" diff --git a/scripts/utils.sh b/scripts/utils.sh index d0da95e87f..1f46ae69b4 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - fatal_error "${fail_msg}" + echo_red "${fail_msg}" fi } From 7bb7c1de90b2bfb47a11124702d6dd968531267a Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:07:04 +0100 Subject: [PATCH 039/122] add test with missing package --- .github/workflows/test_eessi.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 92b1f71cad..1735bb77ed 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -40,4 +40,13 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + ./check_missing_installations.sh + [[ $? -eq 0 ]] && exit 1 || exit 0 From 68a068b64ea5af8fad5a69ff812a5942cacd7167 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:25:04 +0100 Subject: [PATCH 040/122] separated test cases + updated evaluation of test result --- .github/workflows/test_eessi.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1735bb77ed..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -42,11 +42,26 @@ jobs: env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + + - name: Test check_missing_installations.sh with missing package + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml - ./check_missing_installations.sh - [[ $? -eq 0 ]] && exit 1 || exit 0 + if ./check_missing_installations.sh; then + echo "captured missing package; test PASSED" + exit 0 + else + echo "did NOT capture missing package; test FAILED" + exit 1 + fi From 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:30:17 +0100 Subject: [PATCH 041/122] fix small bug in env var name --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 094e82df3b0de5a195d99fa914674024aa2e8145 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:35:29 +0100 Subject: [PATCH 042/122] set software subdir override before sourcing init script --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..d0b530e2e4 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 60cf7e2715b7231810479a3fd91d20499c9b7670 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:01 +0100 Subject: [PATCH 043/122] Revert "set software subdir override before sourcing init script" This reverts commit 094e82df3b0de5a195d99fa914674024aa2e8145. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index d0b530e2e4..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 6f471520b63b0cee6b9d94818e7f42938dbc1bf8 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:51 +0100 Subject: [PATCH 044/122] Revert "fix small bug in env var name" This reverts commit 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From a15676ddac70b5cb18d82625a9a048d59199a41b Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:48:13 +0100 Subject: [PATCH 045/122] tweaking output of test --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..23b30c0063 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -43,7 +43,7 @@ jobs: echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - - name: Test check_missing_installations.sh with missing package + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild @@ -52,12 +52,12 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then echo "captured missing package; test PASSED" exit 0 From 34b36fee213c6e207c8fac8cf8f70fc56860ca8c Mon Sep 17 00:00:00 2001 From: parosen Date: Thu, 9 Mar 2023 14:35:00 +0100 Subject: [PATCH 046/122] Trying only the QuantumEspresso dep libxc-4.3.4-GCC-9.3.0.eb --- EESSI-pilot-install-software.sh | 16 ++++++++-------- eessi-2022.11.yml | 5 +---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 6905a09ca9..0a60b3f5c0 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -315,14 +315,13 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot --fetch --force - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi +#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +# echo ">> Installing QuantumESPRESSO..." +# ok_msg="QuantumESPRESSO installed, let's go quantum!" +# fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" +# $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot +# check_exit_code $? "${ok_msg}" "${fail_msg}" +#fi #echo ">> Installing R 4.0.0 (better be patient)..." #ok_msg="R installed, wow!" @@ -455,6 +454,7 @@ $EB Python-3.9.5-GCCcore-10.3.0.eb --robot $EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot # this Package has been added to reduce the complexity of building large packages such as R $EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot +$EB libxc-4.3.4-GCC-9.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index d2348fd5ea..229cbb70c2 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -4,8 +4,5 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb -<<<<<<< HEAD - - QuantumESPRESSO-6.6-foss-2020a.eb -======= - Nextflow-22.10.1.eb ->>>>>>> 894a0a52b86fd9d31d48ca25c02d154a862855cd + - libxc-4.3.4-GCC-9.3.0.eb From a3dd3eb3ef04f6233bb0b79acf3755aea8db7b26 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 10 Mar 2023 14:36:11 +0100 Subject: [PATCH 047/122] moved Qt5 to end of list --- eessi-2022.11.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 7c3e15d18c..d70b736235 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -2,8 +2,8 @@ easyconfigs: - EasyBuild-4.7.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0.eb - - Qt5-5.14.1-GCCcore-9.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb - Nextflow-22.10.1.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb + - Qt5-5.14.1-GCCcore-9.3.0.eb From db2ab2707d5e870e1e5de53e747040fe9d773831 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 10 Mar 2023 13:57:21 +0000 Subject: [PATCH 048/122] changed the order --- eessi-2022.11.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index d70b736235..c5710dd18a 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -5,5 +5,5 @@ easyconfigs: - OpenMPI-4.1.1-GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb - Nextflow-22.10.1.eb - - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - Qt5-5.14.1-GCCcore-9.3.0.eb + - Spark-3.1.1-foss-2020a-Python-3.8.2.eb From 99231e84eade092ae45fb5fa725d752eb04edf92 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:07:09 +0100 Subject: [PATCH 049/122] if exit_code is not zero, use fatal_error (as it was) --- scripts/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/utils.sh b/scripts/utils.sh index 1f46ae69b4..d0da95e87f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - echo_red "${fail_msg}" + fatal_error "${fail_msg}" fi } From b94bd111a3dcf422e7e78fa58bdbf52a74415fb2 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:52:43 +0100 Subject: [PATCH 050/122] fix bug in test logic --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 23b30c0063..ce58fc3a97 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -59,9 +59,9 @@ jobs: echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then - echo "captured missing package; test PASSED" - exit 0 - else echo "did NOT capture missing package; test FAILED" exit 1 + else + echo "captured missing package; test PASSED" + exit 0 fi From 5e4bea9b71a37a09359def990fd12a1c01a5bf1c Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:03:20 +0100 Subject: [PATCH 051/122] added explanation for test cases (exit code vs boolean logic) --- .github/workflows/test_eessi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index ce58fc3a97..04195dd619 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -58,6 +58,11 @@ jobs: echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package if ./check_missing_installations.sh; then echo "did NOT capture missing package; test FAILED" exit 1 From b38d27ec73bdddf7f337a9db031f055f0243a503 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:33:50 +0100 Subject: [PATCH 052/122] fix bug, env var PWD is uppercase --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 004e2d4881..20334501ed 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") -export SINGULARITY_HOME="${pwd}:/eessi_bot_job" -export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir" +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty From 5128346e14b2d5dbe82963a8cadc768b2267f9a1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 15 Mar 2023 07:04:02 +0100 Subject: [PATCH 053/122] make sure that CPU arch specific directories are used as container cache dir --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 20334501ed..c8def2cdd3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -27,6 +27,7 @@ source scripts/cfg_files.sh # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then @@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} export SINGULARITY_CACHEDIR fi From 6b6867fc8abd6dd5f750a437a612496832212311 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 16 Mar 2023 01:09:55 +0100 Subject: [PATCH 054/122] adding date to when lmod cache is created --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 09c35fbf4d..6f3d5f0b7b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -477,7 +477,7 @@ exit_code=$? check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" -echo ">> Creating/updating Lmod cache..." +echo ">> Creating/updating Lmod cache on $(date) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" if [ ! -f $LMOD_RC ]; then python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} From 582f1cff7a98973103c9951e1561751f4bb7e768 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 16 Mar 2023 15:15:28 +0100 Subject: [PATCH 055/122] new PR to update lmod cache --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 6f3d5f0b7b..1083172f98 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -477,7 +477,7 @@ exit_code=$? check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" -echo ">> Creating/updating Lmod cache on $(date) ..." +echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" if [ ! -f $LMOD_RC ]; then python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} From 437ae24802c461f2b247d25ac0d28b92bec8df86 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 18:58:28 +0100 Subject: [PATCH 056/122] restore PATHs only after last run of pip installed eb --- EESSI-pilot-install-software.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 09c35fbf4d..cfaf971189 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -177,10 +177,6 @@ else eb --install-latest-eb-release &> ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" - # restore origin $PATH and $PYTHONPATH values - export PATH=${ORIG_PATH} - export PYTHONPATH=${ORIG_PYTHONPATH} - eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null if [[ $? -eq 0 ]]; then ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!" @@ -189,6 +185,10 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" fi + # restore origin $PATH and $PYTHONPATH values + export PATH=${ORIG_PATH} + export PYTHONPATH=${ORIG_PYTHONPATH} + module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> EasyBuild module installed!" From c9bf99c8e31a914fbc3f9708cfa7fa998dac4086 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 12:42:05 +0100 Subject: [PATCH 057/122] attempt to fix issue #225 --- create_tarball.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 56ac8ab7ad..08dc4f4d7e 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." files_list=${tmpdir}/files.list.txt +module_files_list=${tmpdir}/module_files.list.txt if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), @@ -49,12 +50,26 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ + >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done fi +echo "wrote file list to ${files_list}" +cat ${files_list} +echo "wrote module file list to ${module_files_list}" +cat ${module_files_list} + topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tgz} from ${topdir}..." From 3b75f790edd4770b687a75fdf6fae06c4bddb728 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 19:17:02 +0100 Subject: [PATCH 058/122] add comments, small improvements --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 08dc4f4d7e..9381407d10 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -51,13 +51,12 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \ + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then - # installation directories - # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + # installation directories but only those for which module files were created for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ @@ -65,6 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then done fi +# add a bit debug output echo "wrote file list to ${files_list}" cat ${files_list} echo "wrote module file list to ${module_files_list}" From 610879e5415e01d71e3fea94f4d47e1262a83630 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 17 Mar 2023 22:13:21 +0100 Subject: [PATCH 059/122] improve handling of potentially non existing list files --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 9381407d10..fd04a195cb 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -55,7 +55,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then +if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" @@ -66,9 +66,9 @@ fi # add a bit debug output echo "wrote file list to ${files_list}" -cat ${files_list} +[ -r ${files_list} ] && cat ${files_list} echo "wrote module file list to ${module_files_list}" -cat ${module_files_list} +[ -r ${module_files_list} ] && cat ${module_files_list} topdir=${cvmfs_repo}/versions/ From b5d8a92340e082b55dfd41eebc1060c247e8933f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Mar 2023 17:50:44 +0100 Subject: [PATCH 060/122] it seems we don't need PR 16531 with eb 4.7.0 --- check_missing_installations.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 45f276dee1..9baeaf2fb3 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -26,7 +26,8 @@ fail_msg="On no, some installations are still missing, how did that happen?!" eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +${EB:-eb} --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as From d393f159d793f4c1d8d91c4f602ca60739c43f41 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Mar 2023 21:57:11 +0100 Subject: [PATCH 061/122] replace EB v4.7.0 with EB v4.6.2 --- eessi-2022.11.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 5dac8fe4a2..cc0bd2c0ab 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,5 +1,5 @@ easyconfigs: - - EasyBuild-4.7.0.eb + - EasyBuild-4.6.2.eb - CMake-3.20.1-GCCcore-10.3.0.eb - Python-3.9.5-GCCcore-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb From 42c8da460efd47726db0579c9671c8e66cd54aa7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Mar 2023 22:29:45 +0100 Subject: [PATCH 062/122] removing additional installation of latest EB as this may change over time --- EESSI-pilot-install-software.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 32b6d4800c..cf8bd45b91 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -464,18 +464,6 @@ $EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot #cat $($EB --last-log) #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" -# add latest EasyBuild to stack -echo ">> Adding latest EasyBuild to stack..." -ok_msg="Latest EasyBuild got installed ... great!" -fail_msg="Installation of latest EasyBuild failed! Disappointed." -if [[ ${EESSI_CVMFS_REPO} == /cvmfs/pilot.eessi-hpc.org ]]; then - $EB --from-pr 14545 --include-easyblocks-from-pr 2805 --robot --install-latest-eb-release -else - $EB --robot --install-latest-eb-release -fi -exit_code=$? -check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" - echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From cdb043ec5a749ac0578d940c8a2b1b2e5d6a1383 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 21 Mar 2023 21:56:11 +0100 Subject: [PATCH 063/122] put --from-pr back or GHA will fail --- check_missing_installations.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 9baeaf2fb3..eb159d7835 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,7 +27,10 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb # ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -${EB:-eb} --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +# PR 16531 not needed since we use EB v4.7.0 +# this, however, breaks the GHA https://github.com/NorESSI/software-layer/blob/main/.github/workflows/test_eessi.yml +# because it uses the EESSI pilot which only provides EB 4.5.1, so adding it back +${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as From abc0aa2523375c8d943e305c9f9483eff01c8a46 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 26 Mar 2023 21:44:36 +0200 Subject: [PATCH 064/122] add R and bioconductor with GCC/9.3.0 --- EESSI-pilot-install-software.sh | 20 ++++++++++---------- eessi-2022.11.yml | 2 ++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 86752b3ec3..b6de0bddb2 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -323,17 +323,17 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" # check_exit_code $? "${ok_msg}" "${fail_msg}" #fi -#echo ">> Installing R 4.0.0 (better be patient)..." -#ok_msg="R installed, wow!" -#fail_msg="Installation of R failed, so sad..." -#$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -#check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing R 4.0.0 (better be patient)..." +ok_msg="R installed, wow!" +fail_msg="Installation of R failed, so sad..." +$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental +check_exit_code $? "${ok_msg}" "${fail_msg}" -#echo ">> Installing Bioconductor 3.11 bundle..." -#ok_msg="Bioconductor installed, enjoy!" -#fail_msg="Installation of Bioconductor failed, that's annoying..." -#$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing Bioconductor 3.11 bundle..." +ok_msg="Bioconductor installed, enjoy!" +fail_msg="Installation of Bioconductor failed, that's annoying..." +$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot +check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing TensorFlow 2.3.1..." #ok_msg="TensorFlow 2.3.1 installed, w00!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 1401291902..f3629ad0ae 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -9,3 +9,5 @@ easyconfigs: - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb + - R-4.0.0-foss-2020a.eb + - R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb From 12f060278cd6174ae66ee181e08742ce8efca80d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 26 Mar 2023 21:49:54 +0200 Subject: [PATCH 065/122] added easyconfigs from generic rebuild and sorted them alphabetically --- eessi-2022.11.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index f3629ad0ae..6c1726e7ff 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,13 +1,24 @@ easyconfigs: - - EasyBuild-4.6.2.eb + - CMake-3.16.4-GCCcore-9.3.0.eb - CMake-3.20.1-GCCcore-10.3.0.eb - - Python-3.9.5-GCCcore-10.3.0.eb - - OpenMPI-4.1.1-GCC-10.3.0.eb + - EasyBuild-4.6.2.eb + - GCC-9.3.0.eb + - GCC-10.3.0.eb - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb + - Java-11.eb + - Java-17.eb - Nextflow-22.10.1.eb - - Qt5-5.14.1-GCCcore-9.3.0.eb - - Spark-3.1.1-foss-2020a-Python-3.8.2.eb + - OpenBLAS-0.3.9-GCC-9.3.0.eb + - OpenMPI-4.0.3-GCC-9.3.0.eb + - OpenMPI-4.1.1-GCC-10.3.0.eb - OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb - - WRF-3.9.1.1-foss-2020a-dmpar.eb + - Perl-5.30.2-GCCcore-9.3.0.eb + - Python-2.7.18-GCCcore-9.3.0.eb + - Python-3.8.2-GCCcore-9.3.0.eb + - Python-3.9.5-GCCcore-10.3.0.eb + - Qt5-5.14.1-GCCcore-9.3.0.eb - R-4.0.0-foss-2020a.eb - R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb + - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb + - Spark-3.1.1-foss-2020a-Python-3.8.2.eb + - WRF-3.9.1.1-foss-2020a-dmpar.eb From 356760976b668fbcd74c2646eb2d6ec70818a0ca Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 27 Mar 2023 08:58:22 +0200 Subject: [PATCH 066/122] removing bioconductor --- EESSI-pilot-install-software.sh | 10 +++++----- eessi-2022.11.yml | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index b6de0bddb2..0c2ae5d16f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -329,11 +329,11 @@ fail_msg="Installation of R failed, so sad..." $EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental check_exit_code $? "${ok_msg}" "${fail_msg}" -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" +#echo ">> Installing Bioconductor 3.11 bundle..." +#ok_msg="Bioconductor installed, enjoy!" +#fail_msg="Installation of Bioconductor failed, that's annoying..." +#$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot +#check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing TensorFlow 2.3.1..." #ok_msg="TensorFlow 2.3.1 installed, w00!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 6c1726e7ff..4ac09c385e 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -18,7 +18,6 @@ easyconfigs: - Python-3.9.5-GCCcore-10.3.0.eb - Qt5-5.14.1-GCCcore-9.3.0.eb - R-4.0.0-foss-2020a.eb - - R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb From 8caa2a614cf4e9077b30b13afada56faceb34d6d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 27 Mar 2023 21:03:35 +0200 Subject: [PATCH 067/122] sort easyconfigs alphabetically, use --robot instead of -r --- EESSI-pilot-install-software.sh | 10 +++++----- eessi-2022.11.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 063d11e7c7..fbbb326788 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -365,13 +365,13 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." ok_msg="OSU-Micro-Benchmarks installed, yihaa!" fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r +$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing Spark 3.1.1..." ok_msg="Spark installed, set off the fireworks!" fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing IPython 7.15.0..." @@ -383,7 +383,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing WRF 3.9.1.1..." ok_msg="WRF installed, it's getting hot in here!" fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 +OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb --robot --include-easyblocks-from-pr 2648 check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing R 4.1.0 (better be patient)..." @@ -396,13 +396,13 @@ echo ">> Installing Nextflow 22.10.1..." ok_msg="Nextflow installed, the work must flow..." fail_msg="Installation of Nextflow failed, that's unexpected..." # Comment from Axel: PR 16531 was merged so --from-pr not needed anymore (but was used in this build) -$EB -r --from-pr 16531 Nextflow-22.10.1.eb +$EB --from-pr 16531 Nextflow-22.10.1.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." ok_msg="OSU-Micro-Benchmarks installed, yihaa!" fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing EasyBuild 4.5.1..." diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 5669be9a99..81a7f003fd 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -11,6 +11,7 @@ easyconfigs: - OpenBLAS-0.3.9-GCC-9.3.0.eb - OpenMPI-4.0.3-GCC-9.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb + - OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb - OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb - Perl-5.30.2-GCCcore-9.3.0.eb - Python-2.7.18-GCCcore-9.3.0.eb @@ -21,4 +22,3 @@ easyconfigs: - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb - - OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb From dc690f47d112d62499a3e13aa1c6e2778e116408 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 28 Mar 2023 10:50:51 +0000 Subject: [PATCH 068/122] Add FlexiBlas/3.0.4 with GCC/10.3.0 to NESSI/2022.11 --- EESSI-pilot-install-software.sh | 15 +++++++++------ eessi-2022.11.yml | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index fbbb326788..e0e734ecd7 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -437,12 +437,15 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; # using -fstack-protector-strong -fstack-clash-protection should fix that, # but it doesn't for some reason when building for ppc64le/generic... -#if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then -# $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -#else -# $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -#fi -# +echo ">> Installing FlexiBlas-3.0.4-foss-2021a..." +ok_msg="FlexiBlas. Nice!" +fail_msg="Installation of FlexiBlas failed, that's unexpected..." +if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure +else + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb + check_exit_code $? "${ok_msg}" "${fail_msg}" +fi #$EB SciPy-bundle-2021.05-foss-2021a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 81a7f003fd..df31e03405 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -22,3 +22,4 @@ easyconfigs: - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb + - FlexiBLAS-3.0.4-GCC-10.3.0.eb From 8bedb822d0bf2c3fbf1de09d9ed49adb764b033a Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 28 Mar 2023 11:05:49 +0000 Subject: [PATCH 069/122] added the --robot option --- EESSI-pilot-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index e0e734ecd7..005b35bbdc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -441,9 +441,9 @@ echo ">> Installing FlexiBlas-3.0.4-foss-2021a..." ok_msg="FlexiBlas. Nice!" fail_msg="Installation of FlexiBlas failed, that's unexpected..." if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot --ignore-test-failure else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb + $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi #$EB SciPy-bundle-2021.05-foss-2021a.eb --robot From bc37b40a33e2aa1d6b5f1c784fa35251f3f97041 Mon Sep 17 00:00:00 2001 From: parosen Date: Wed, 29 Mar 2023 13:30:35 +0200 Subject: [PATCH 070/122] Elpa failed with incorrect checksum, source url has changed, trying new one --- EESSI-pilot-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index a15dd99608..c1102a8744 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -319,6 +319,8 @@ if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then echo ">> Installing QuantumESPRESSO..." ok_msg="QuantumESPRESSO installed, let's go quantum!" fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" + # ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1 + $EB eb --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi From a4a9aec29622d219d183dffd73fe497735505d91 Mon Sep 17 00:00:00 2001 From: parosen Date: Wed, 29 Mar 2023 14:28:55 +0200 Subject: [PATCH 071/122] trying with only ELPA --from-pr (dep of QuantumEspresso) --- EESSI-pilot-install-software.sh | 23 ++++++++++++++--------- eessi-2022.11.yml | 1 - 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c1102a8744..6286739374 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -315,15 +315,20 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" #$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - # ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1 - $EB eb --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi +# ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1 +echo ">> Installing ELPA" +ok_msg="ELPA installed, wow!" +fail_msg="Installation of ELPA failed, so sad..." +$EB eb --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot +check_exit_code $? "${ok_msg}" "${fail_msg}" + +#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +# echo ">> Installing QuantumESPRESSO..." +# ok_msg="QuantumESPRESSO installed, let's go quantum!" +# fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" +# $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot +# check_exit_code $? "${ok_msg}" "${fail_msg}" +#fi echo ">> Installing R 4.0.0 (better be patient)..." ok_msg="R installed, wow!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 8e3b41454f..81a7f003fd 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -22,4 +22,3 @@ easyconfigs: - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb - - QuantumESPRESSO-6.6-foss-2020a.eb From 96ed2e65d4cec809527c2d5c6b41d690a0f33991 Mon Sep 17 00:00:00 2001 From: parosen Date: Wed, 29 Mar 2023 15:20:27 +0200 Subject: [PATCH 072/122] fixed typo of duplicate eb command --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 6286739374..ac210fd997 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -319,7 +319,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing ELPA" ok_msg="ELPA installed, wow!" fail_msg="Installation of ELPA failed, so sad..." -$EB eb --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot +$EB --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" #if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then From 6b6a18110672aa02302841dc2cbe6bd2a4c98674 Mon Sep 17 00:00:00 2001 From: parosen Date: Wed, 29 Mar 2023 16:12:33 +0200 Subject: [PATCH 073/122] install ELPA dep first, then QuantumEspresso --- EESSI-pilot-install-software.sh | 23 ++++++++++------------- eessi-2022.11.yml | 1 + 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ac210fd997..6c9fa3711d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -315,20 +315,17 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" #$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" -# ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1 -echo ">> Installing ELPA" -ok_msg="ELPA installed, wow!" -fail_msg="Installation of ELPA failed, so sad..." -$EB --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" -#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then -# echo ">> Installing QuantumESPRESSO..." -# ok_msg="QuantumESPRESSO installed, let's go quantum!" -# fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" -# $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot -# check_exit_code $? "${ok_msg}" "${fail_msg}" -#fi +if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then + echo ">> Installing QuantumESPRESSO..." + ok_msg="QuantumESPRESSO installed, let's go quantum!" + fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" + # ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1, need to use --from-pr + # until 4.7.1 is installed + $EB --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot + $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" +fi echo ">> Installing R 4.0.0 (better be patient)..." ok_msg="R installed, wow!" diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 81a7f003fd..8e3b41454f 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -22,3 +22,4 @@ easyconfigs: - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb - Spark-3.1.1-foss-2020a-Python-3.8.2.eb - WRF-3.9.1.1-foss-2020a-dmpar.eb + - QuantumESPRESSO-6.6-foss-2020a.eb From fae531666bc2a8b939caada70d772f10def6d6af Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 11 Apr 2023 12:34:39 +0200 Subject: [PATCH 074/122] update determining directories to tar + adding a note on assumptions made --- create_tarball.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index fd04a195cb..b6c72b341d 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -52,11 +52,16 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ - | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created + # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' + # using EasyBuild's standard module naming scheme) match the name of the + # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). + # If either side changes (module naming scheme or naming of software + # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ From 711cc15483a350645ea8ac7a01f75f387026f1ca Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 11 Apr 2023 09:35:44 +0200 Subject: [PATCH 075/122] removing comment to address review comment --- check_missing_installations.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index eb159d7835..5e76e23c30 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -42,9 +42,6 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim grep " required modules missing:" ${eb_missing_out} > /dev/null -# we need to process the result (from finding `No missing modules` to NOT finding -# ` required modules missing:` and no other error happened) -# # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 [[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 From 103b5cffb696a1682fa0da543676e0373e631759 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 11:44:11 +0200 Subject: [PATCH 076/122] first version of bot/check-result.sh for software-layer - follows what has been implemented in the EESSI/eessi-bot-software-layer for checking the result of a job - re-uses code from a small script that was run manually on a bot instance to check a job's result - also re-uses code from the check-result.sh script in EESSI/compatibility-layer --- bot/check-result.sh | 195 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bot/check-result.sh diff --git a/bot/check-result.sh b/bot/check-result.sh new file mode 100755 index 0000000000..38e76c3abc --- /dev/null +++ b/bot/check-result.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Script to check the result of building the EESSI software layer. +# Intended use is that it is called by a (batch) job running on a compute +# node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# result cases + +# - SUCCESS (all of) +# - working directory contains slurm-JOBID.out file +# - working directory contains eessi*tar.gz +# - no message ERROR +# - no message FAILED +# - no message ' required modules missing:' +# - one or more of 'No missing modules!' +# - message regarding created tarball +# - FAILED (one of ... implemented as NOT SUCCESS) +# - no slurm-JOBID.out file +# - no tarball +# - message with ERROR +# - message with FAILED +# - message with ' required modules missing:' +# - no message regarding created tarball + +# stop as soon as something fails +# set -e + +TOPDIR=$(dirname $(realpath $0)) + +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/scripts/cfg_files.sh + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS:" + echo " -h | --help - display this usage information [default: false]" + echo " -v | --verbose - display more information [default: false]" +} + +# set defaults for command line arguments +VERBOSE=0 + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + display_help + exit 0 + ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +job_dir=${PWD} + +[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" + +GP_slurm_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" +job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") +[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" + +GP_error='ERROR: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" +grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_failed='FAILED: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" +grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_req_missing=' required modules missing:' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" +grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_no_missing='No missing modules!' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" +grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_tgz_created="tar.gz created!" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" +grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) +TARBALL= +if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') +else + TGZ=0 +fi +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +echo "SUMMARY: ${job_dir}/${job_out}" +echo " test name : result (expected result)" +echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + +if [[ ${SLURM} -eq 1 ]] && \ + [[ ${ERROR} -eq 0 ]] && \ + [[ ${FAILED} -eq 0 ]] && \ + [[ ${MISSING} -eq 0 ]] && \ + [[ ${NO_MISSING} -eq 1 ]] && \ + [[ ${TGZ} -eq 1 ]] && \ + [[ ! -z ${TARBALL} ]]; then + # SUCCESS + echo "[RESULT]" > ${job_result_file} + echo "summary = :grin: SUCCESS" >> ${job_result_file} + echo "details =" >> ${job_result_file} +else + # FAILURE + echo "[RESULT]" > ${job_result_file} + echo "summary = :cry: FAILURE" >> ${job_result_file} + echo "details =" >> ${job_result_file} +fi + +if [[ ${SLURM} -eq 1 ]]; then + # need to indent by 4 spaces + echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} +else + echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} +fi + +if [[ ${ERROR} -eq 0 ]]; then + echo " job output lacks message matching ${GP_error}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_error}" >> ${job_result_file} +fi + +if [[ ${FAILED} -eq 0 ]]; then + echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_failed}" >> ${job_result_file} +fi + +if [[ ${MISSING} -eq 0 ]]; then + echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} +fi + +if [[ ${NO_MISSING} -eq 1 ]]; then + echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} +else + echo " found no message matching ${GP_no_missing}" >> ${job_result_file} +fi + +if [[ ${TGZ} -eq 1 ]]; then + echo " found message matching ${GP_tgz_created}" >> ${job_result_file} +else + echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} +fi + +echo "artefacts =" >> ${job_result_file} + +if [[ ! -z ${TARBALL} ]]; then + echo " ${TARBALL}" >> ${job_result_file} +fi + +exit 0 From 7403462bd1253741f1166fe0afc0a3d4fe6940d4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 14:34:27 +0200 Subject: [PATCH 077/122] fix path to helper scripts + define result file name --- bot/check-result.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 38e76c3abc..15356f10cf 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -35,8 +35,8 @@ TOPDIR=$(dirname $(realpath $0)) -source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/scripts/cfg_files.sh +source ${TOPDIR}/../scripts/utils.sh +source ${TOPDIR}/../scripts/cfg_files.sh display_help() { echo "usage: $0 [OPTIONS]" @@ -131,6 +131,8 @@ echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +job_result_file=_bot_job${SLURM_JOB_ID}.result + if [[ ${SLURM} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ From 652a6a641bfc9acac5b025cd0c60e05e798f0d7a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 15:41:45 +0200 Subject: [PATCH 078/122] define functions for standardizing result output --- bot/check-result.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 15356f10cf..dbc76e2dc9 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -151,41 +151,48 @@ else echo "details =" >> ${job_result_file} fi +function succeeded() { + echo "    :heavy_check_mark:${1}" +} + +function failed() { + echo "    :heavy_multiplication_x:${1}" +} + if [[ ${SLURM} -eq 1 ]]; then - # need to indent by 4 spaces - echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} + succeeded "job output file ${job_out}" >> ${job_result_file} else - echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} + failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} fi if [[ ${ERROR} -eq 0 ]]; then - echo " job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} else - echo " job output contains message matching ${GP_error}" >> ${job_result_file} + failed "job output contains message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} else - echo " job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "job output contains message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} else - echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then - echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} + succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - echo " found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "found no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then - echo " found message matching ${GP_tgz_created}" >> ${job_result_file} + succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From 5de041fd9e38049e177cd3113c87dab980d5eaad Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 11:29:05 +0200 Subject: [PATCH 079/122] polishing formating of status messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index dbc76e2dc9..9e34f81dd8 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo "    :heavy_check_mark:${1}" + echo " :heavy_check_mark:${1}" } function failed() { - echo "    :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x:${1}" } if [[ ${SLURM} -eq 1 ]]; then From 24af899a2f37a7f390bc20bd1af7288f24f6808e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 12:29:56 +0200 Subject: [PATCH 080/122] more polishing of comment details --- bot/check-result.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 9e34f81dd8..2d2f33be06 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -166,33 +166,33 @@ else fi if [[ ${ERROR} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "no message matching ${GP_error}" >> ${job_result_file} else - failed "job output contains message matching ${GP_error}" >> ${job_result_file} + failed "found message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "no message matching ${GP_failed}" >> ${job_result_file} else - failed "job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "found message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} else - failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "found message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - failed "found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From 060bd7df1610a113637718fcc0d057da31b38d28 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:06:15 +0200 Subject: [PATCH 081/122] minor polishing of job result messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 2d2f33be06..1598e9ca1b 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo " :heavy_check_mark:${1}" + echo " :heavy_check_mark: ${1}" } function failed() { - echo " :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x: ${1}" } if [[ ${SLURM} -eq 1 ]]; then From f5072d72200d0b85ada7434251ce860b70e9539f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:36:15 +0200 Subject: [PATCH 082/122] only log if run with --verbose --- bot/check-result.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 1598e9ca1b..55276c405a 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -123,13 +123,13 @@ else fi [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" -echo "SUMMARY: ${job_dir}/${job_out}" -echo " test name : result (expected result)" -echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" +[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" job_result_file=_bot_job${SLURM_JOB_ID}.result From e8b8ae6750fd22eca46c670494cd0c7625ba1442 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:22:40 +0200 Subject: [PATCH 083/122] fix small bug when checking for tarball msg --- bot/check-result.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 55276c405a..3f0c89ea21 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -112,9 +112,9 @@ grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" +TARBALL= [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -TARBALL= if [[ $? -eq 0 ]]; then TGZ=1 TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') From 5c6f3786159985cc4df5c225c8f5053766998343 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:31:36 +0200 Subject: [PATCH 084/122] avoid creating false positives --- bot/check-result.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 3f0c89ea21..e28ae4e38e 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -82,38 +82,42 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" GP_error='ERROR: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_failed='FAILED: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_req_missing=' required modules missing:' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_no_missing='No missing modules!' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" TARBALL= -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) if [[ $? -eq 0 ]]; then TGZ=1 @@ -121,6 +125,8 @@ if [[ $? -eq 0 ]]; then else TGZ=0 fi +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" From fcf519fb944cac59fad376b710ab305cdce5de07 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 21 Apr 2023 07:16:24 +0000 Subject: [PATCH 085/122] fixed the code for the checking process --- EESSI-pilot-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 0f5e95ba92..1c00265958 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -448,8 +448,9 @@ if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot --ignore-test-failure else $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" fi +check_exit_code $? "${ok_msg}" "${fail_msg}" + #$EB SciPy-bundle-2021.05-foss-2021a.eb --robot #check_exit_code $? "${ok_msg}" "${fail_msg}" From 1bd2166c555b2c090bc856b7cc5760045302ca7d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 10:33:55 +0200 Subject: [PATCH 086/122] prepare for next pilot version (2023.04) --- EESSI-pilot-install-software.sh | 269 +------------------------------- 1 file changed, 4 insertions(+), 265 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 1c00265958..ecbe0c500d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -138,16 +138,8 @@ else fi echo "EESSI_CVMFS_REPO=${EESSI_CVMFS_REPO}" -case ${EESSI_CVMFS_REPO} in - /cvmfs/pilot.eessi-hpc.org*) - REQ_EB_VERSION='4.5.0' - ;; - /cvmfs/pilot.nessi.no*) - REQ_EB_VERSION='4.7.0' - ;; - *) - fatal_error "unsupported CVMFS repository '${EESSI_CVMFS_REPO}'" -esac + +REQ_EB_VERSION='4.7.1' echo "REQ_EB_VERSION=${REQ_EB_VERSION}" module avail 2>&1 | grep -i easybuild @@ -217,261 +209,8 @@ else fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." - -################################################################################ -# COMMENT OUT EB install sections below to not hit GH rate limit because of -# *from-pr arguments -################################################################################ -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -#ok_msg="Java installed, off to a good (?) start!" -#fail_msg="Failed to install Java, woopsie..." -#$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -#export GCC_EC="GCC-9.3.0.eb" -#echo ">> Starting slow with ${GCC_EC}..." -#ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -#fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -#$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -#echo ">> Install CMake with fixed easyblock to take into account --sysroot" -#ok_msg="CMake installed!" -#fail_msg="Installation of CMake failed, what the ..." -#$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -#echo ">> Installing OpenBLAS..." -#ok_msg="Done with OpenBLAS!" -#fail_msg="Installation of OpenBLAS failed!" -#if [[ $GENERIC -eq 1 ]]; then -# echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." -# openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -#else -# openblas_include_easyblocks_from_pr='' -#fi -#$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing OpenMPI..." -#ok_msg="OpenMPI installed, w00!" -#fail_msg="Installation of OpenMPI failed, that's not good..." -#$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -#echo ">> Install Python 2.7.18 and Python 3.8.2..." -#ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -#fail_msg="Installation of Python failed, oh no..." -#$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing Perl..." -#ok_msg="Perl installed, making progress..." -#fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -#$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot --disable-cleanup-tmpdir -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -#echo ">> Installing SciPy-bundle" -#ok_msg="SciPy-bundle installed, yihaa!" -#fail_msg="SciPy-bundle installation failed, bummer..." -#SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -#if [[ "$(uname -m)" == "aarch64" ]]; then -# $EB $SCIPY_EC --robot --skip-test-step -#else -# $EB $SCIPY_EC --robot -#fi -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing GROMACS..." -#ok_msg="GROMACS installed, wow!" -#fail_msg="Installation of GROMACS failed, damned..." -#$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -#echo ">> Installing OpenFOAM (twice!)..." -#ok_msg="OpenFOAM installed, now we're talking!" -#fail_msg="Installation of OpenFOAM failed, we were so close..." -#$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - # ELPA source url has changed between EasyBuild 4.7.0 and 4.7.1, need to use --from-pr - # until 4.7.1 is installed - $EB --from-pr 17357 ELPA-2019.11.001-foss-2020a.eb --robot - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing Bioconductor 3.11 bundle..." -#ok_msg="Bioconductor installed, enjoy!" -#fail_msg="Installation of Bioconductor failed, that's annoying..." -#$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing TensorFlow 2.3.1..." -#ok_msg="TensorFlow 2.3.1 installed, w00!" -#fail_msg="Installation of TensorFlow failed, why am I not surprised..." -#$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing Horovod 0.21.3..." -#ok_msg="Horovod installed! Go do some parallel training!" -#fail_msg="Horovod installation failed. There comes the headache..." -#$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - -# echo ">> Installing code-server 3.7.3..." -# ok_msg="code-server 3.7.3 installed, now you can use VS Code!" -# fail_msg="Installation of code-server failed, that's going to be hard to fix..." -# $EB code-server-3.7.3.eb --robot -# check_exit_code $? "${ok_msg}" "${fail_msg}" -#fi - -#echo ">> Installing RStudio-Server 1.3.1093..." -#ok_msg="RStudio-Server installed, enjoy!" -#fail_msg="Installation of RStudio-Server failed, might be OS deps..." -#$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing IPython 7.15.0..." -#ok_msg="IPython installed, launch your Jupyter Notebooks!" -#fail_msg="Installation of IPython failed, that's unexpected..." -#$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb --robot --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing R 4.1.0 (better be patient)..." -#ok_msg="R installed, wow!" -#fail_msg="Installation of R failed, so sad..." -#$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -# Comment from Axel: PR 16531 was merged so --from-pr not needed anymore (but was used in this build) -$EB --from-pr 16531 Nextflow-22.10.1.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -#echo ">> Installing EasyBuild 4.5.1..." -#ok_msg="EasyBuild v4.5.1 installed" -#fail_msg="EasyBuild v4.5.1 failed to install" -#$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -#check_exit_code $? "${ok_msg}" "${fail_msg}" - -#LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -#check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -#echo ">> Installing SciPy-bundle with foss/2021a..." -#ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -#fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -#$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -#$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -#use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -#from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -#$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -#$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -echo ">> Installing FlexiBlas-3.0.4-foss-2021a..." -ok_msg="FlexiBlas. Nice!" -fail_msg="Installation of FlexiBlas failed, that's unexpected..." -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -#$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -#check_exit_code $? "${ok_msg}" "${fail_msg}" - - -##################### -### add packages here -##################### -$EB Python-3.9.5-GCCcore-10.3.0.eb --robot -$EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot -# this Package has been added to reduce the complexity of building large packages such as R -$EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot -# example block showing a few debugging means -#echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." -#ok_msg="CaDiCaL installed. Nice!" -#fail_msg="Installation of CaDiCaL failed, that's unexpected..." -#$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot --disable-cleanup-tmpdir -#exit_code=$? -#$EB --last-log -#cat $($EB --last-log) -#check_exit_code $exit_code "${ok_msg}" "${fail_msg}" - +eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out +${EB:-eb} --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 2c5b3d9eaacdb2fc468ef39ad97521a746679857 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 10:38:26 +0200 Subject: [PATCH 087/122] add initial easystack file for 2023.04 --- eessi-2023.04.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 eessi-2023.04.yml diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml new file mode 100644 index 0000000000..57e2a99495 --- /dev/null +++ b/eessi-2023.04.yml @@ -0,0 +1,5 @@ +easyconfigs: + - GCC-9.3.0.eb + - GCC-10.3.0.eb +# - GCC-11.3.0.eb +# - GCC-12.2.0.eb From 1f1c76b9923878e6101e20b8431ee5f353bdc31d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 12:07:38 +0200 Subject: [PATCH 088/122] add GCC 10.3.0 to NESSI/2023.04 --- eessi-2023.04.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 57e2a99495..15d853221a 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -1,5 +1,5 @@ easyconfigs: - - GCC-9.3.0.eb +# - GCC-9.3.0.eb - GCC-10.3.0.eb # - GCC-11.3.0.eb # - GCC-12.2.0.eb From 72219f2002ac75931fb8d85a86b5eab36cc94a47 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 12:35:21 +0200 Subject: [PATCH 089/122] add --robot to global EB command --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ecbe0c500d..ef6e2b805b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -210,7 +210,7 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out -${EB:-eb} --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} +${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 7c05eff3958247f07b41156d8c3bd047d8192b8c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 17:05:08 +0200 Subject: [PATCH 090/122] patch gcc easyblock to work with https://github.com/gentoo/gentoo/pull/28851 --- easyblocks/g/gcc.py | 1053 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 easyblocks/g/gcc.py diff --git a/easyblocks/g/gcc.py b/easyblocks/g/gcc.py new file mode 100644 index 0000000000..df4726e086 --- /dev/null +++ b/easyblocks/g/gcc.py @@ -0,0 +1,1053 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing GCC, implemented as an easyblock + +@author: Stijn De Weirdt (Ghent University) +@author: Dries Verdegem (Ghent University) +@author: Kenneth Hoste (Ghent University) +@author: Pieter De Baets (Ghent University) +@author: Jens Timmerman (Ghent University) +@author: Toon Willems (Ghent University) +@author: Ward Poelmans (Ghent University) +@author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +""" +import glob +import os +import re +import shutil +from copy import copy +from distutils.version import LooseVersion + +import easybuild.tools.environment as env +from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP +from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, copy_file, move_file, symlink +from easybuild.tools.filetools import which, write_file +from easybuild.tools.modules import get_software_root +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import RISCV, check_os_dependency, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_gcc_version, get_shared_lib_ext, get_os_name, get_os_type +from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC +from easybuild.tools.utilities import nub + + +# Offloading stages to build +AMD_LLVM = 'AMD_GCN_LLVM' +AMD_NEWLIB = 'AMD_GCN_NEWLIB' +HOST_COMPILER = 'HOST_COMPILER' +NVIDIA_NEWLIB = 'NVIDIA_NEWLIB' +NVPTX_TOOLS = 'NVIDIA_NVPTX_TOOLS' +# Additional symlinks to create for compiler commands +COMP_CMD_SYMLINKS = { + 'cc': 'gcc', + 'c++': 'g++', + 'f77': 'gfortran', + 'f95': 'gfortran', +} + + +class EB_GCC(ConfigureMake): + """ + Self-contained build of GCC. + Uses system compiler for initial build, then bootstraps. + """ + + @staticmethod + def extra_options(): + extra_vars = { + 'clooguseisl': [False, "Use ISL with CLooG or not", CUSTOM], + 'generic': [None, "Build GCC and support libraries such that it runs on all processors of the target " + "architecture (use False to enforce non-generic regardless of configuration)", CUSTOM], + 'languages': [[], "List of languages to build GCC for (--enable-languages)", CUSTOM], + 'multilib': [False, "Build multilib gcc (both i386 and x86_64)", CUSTOM], + 'pplwatchdog': [False, "Enable PPL watchdog", CUSTOM], + 'prefer_lib_subdir': [False, "Configure GCC to prefer 'lib' subdirs over 'lib64' when linking", CUSTOM], + 'profiled': [False, "Bootstrap GCC with profile-guided optimizations", CUSTOM], + 'use_gold_linker': [None, "Configure GCC to use GOLD as default linker " + "(default: enable automatically for GCC < 11.3.0, except on RISC-V)", CUSTOM], + 'withcloog': [False, "Build GCC with CLooG support", CUSTOM], + 'withisl': [False, "Build GCC with ISL support", CUSTOM], + 'withlibiberty': [False, "Enable installing of libiberty", CUSTOM], + 'withlto': [True, "Enable LTO support", CUSTOM], + 'withppl': [False, "Build GCC with PPL support", CUSTOM], + 'withnvptx': [False, "Build GCC with NVPTX offload support", CUSTOM], + 'withamdgcn': [False, "Build GCC with AMD GCN offload support", CUSTOM], + } + return ConfigureMake.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + super(EB_GCC, self).__init__(*args, **kwargs) + + self.stagedbuild = False + # Each build iteration is related to a specific build, first build host compiler, then potentially build NVPTX + # offloading and/or AMD GCN offloading + self.build_stages = [HOST_COMPILER] + self.current_stage = HOST_COMPILER + # Directories for additional tools needed when doing offloading to Nvidia and AMD + self.nvptx_tools_dir = None # nvptx-tools necessary for Nvidia + self.llvm_dir = None # LLVM is necessary when offloading to AMD + self.lld_dir = None # LLD is the only required component of LLVM + self.newlib_dir = None # Used by both NVPTX and AMD GCN backend + + # need to make sure version is an actual version + # required because of support in SystemCompiler generic easyblock to specify 'system' as version, + # which results in deriving the actual compiler version + # comparing a non-version like 'system' with an actual version like '2016' fails with TypeError in Python 3.x + if re.match(r'^[0-9]+\.[0-9]+.*', self.version): + version = LooseVersion(self.version) + + if version >= LooseVersion('4.8.0') and self.cfg['clooguseisl'] and not self.cfg['withisl']: + raise EasyBuildError("Using ISL bundled with CLooG is unsupported in >=GCC-4.8.0. " + "Use a seperate ISL: set withisl=True") + + # I think ISL without CLooG has no purpose in GCC < 5.0.0 ... + if version < LooseVersion('5.0.0') and self.cfg['withisl'] and not self.cfg['withcloog']: + raise EasyBuildError("Activating ISL without CLooG is pointless") + + # Disable the Gold linker by default for GCC 11.3.0 and newer, as it suffers from bitrot + if self.cfg['use_gold_linker'] is None: + self.cfg['use_gold_linker'] = version < LooseVersion('11.3.0') + + # unset some environment variables that are known to may cause nasty build errors when bootstrapping + self.cfg.update('unwanted_env_vars', ['CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH']) + # ubuntu needs the LIBRARY_PATH env var to work apparently (#363) + if get_os_name() not in ['ubuntu', 'debian']: + self.cfg.update('unwanted_env_vars', ['LIBRARY_PATH']) + + def create_dir(self, dirname): + """ + Create a dir to build in. + """ + dirpath = os.path.join(self.cfg['start_dir'], dirname) + try: + os.mkdir(dirpath) + os.chdir(dirpath) + self.log.debug("Created dir at %s" % dirpath) + return dirpath + except OSError as err: + raise EasyBuildError("Can't use dir %s to build in: %s", dirpath, err) + + def disable_lto_mpfr_old_gcc(self, objdir): + """ + # if GCC version used to build stage 1 is too old, build MPFR without LTO in stage 1 + # required for e.g. CentOS 6, cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/6374 + """ + self.log.info("Checking whether we are trying to build a recent MPFR with an old GCC...") + + # try to figure out MPFR version being built + mpfr_ver = '0.0' + mpfr_dirs = glob.glob(os.path.join(self.builddir, 'mpfr-*')) + if len(mpfr_dirs) == 1: + mpfr_dir = mpfr_dirs[0] + res = re.search('(?P[0-9.]+)$', mpfr_dir) + if res: + mpfr_ver = res.group('mpfr_ver') + self.log.info("Found MPFR version %s (based name of MPFR source dir: %s)", mpfr_ver, mpfr_dir) + else: + self.log.warning("Failed to determine MPFR version from '%s', assuming v%s", mpfr_dir, mpfr_ver) + else: + self.log.warning("Failed to isolate MPFR source dir to determine MPFR version, assuming v%s", mpfr_ver) + + # for MPFR v4.x & newer, we need a recent GCC that supports -flto + if LooseVersion(mpfr_ver) >= LooseVersion('4.0'): + + disable_mpfr_lto = False + + # check GCC version being used + # GCC 4.5 is required for -flto (cfr. https://gcc.gnu.org/gcc-4.5/changes.html) + gcc_ver = get_gcc_version() + min_gcc_ver_lto = '4.5' + if gcc_ver is None: + self.log.warning("Failed to determine GCC version, assuming it's recent enough...") + elif LooseVersion(gcc_ver) < LooseVersion(min_gcc_ver_lto): + self.log.info("Configuring MPFR to build without LTO in stage 1 (GCC %s is too old: < %s)!", + gcc_ver, min_gcc_ver_lto) + disable_mpfr_lto = True + else: + self.log.info("GCC %s (>= %s) is OK for building MPFR in stage 1 with LTO enabled", + gcc_ver, min_gcc_ver_lto) + + # check whether GCC actually supports LTO (it may be configured with --disable-lto), + # by compiling a simple C program using -flto + out, ec = run_cmd("echo 'void main() {}' | gcc -x c -flto - -o /dev/null", simple=False, log_ok=False) + gcc_path = which('gcc') + if ec: + self.log.info("GCC command %s doesn't seem to support LTO, test compile failed: %s", gcc_path, out) + disable_mpfr_lto = True + else: + self.log.info("GCC command %s provides LTO support, so using it when building MPFR", gcc_path) + + if disable_mpfr_lto: + # patch GCC's Makefile to inject --disable-lto when building MPFR + stage1_makefile = os.path.join(objdir, 'Makefile') + regex_subs = [(r'(--with-gmp-lib=\$\$r/\$\(HOST_SUBDIR\)/gmp/.libs) \\', r'\1 --disable-lto \\')] + apply_regex_substitutions(stage1_makefile, regex_subs) + + def prep_extra_src_dirs(self, stage, target_prefix=None): + """ + Prepare extra (optional) source directories, so GCC will build these as well. + """ + if LooseVersion(self.version) >= LooseVersion('4.5'): + known_stages = ["stage1", "stage2", "stage3"] + if stage not in known_stages: + raise EasyBuildError("Incorrect argument for prep_extra_src_dirs, should be one of: %s", known_stages) + + configopts = '' + if stage == "stage2": + # no MPFR/MPC needed in stage 2 + extra_src_dirs = ["gmp"] + else: + extra_src_dirs = ["gmp", "mpfr", "mpc"] + + # list of the extra dirs that are needed depending on the 'with%s' option + # the order is important: keep CLooG last! + self.with_dirs = ["isl", "ppl", "cloog"] + + # add optional ones that were selected (e.g. CLooG, PPL, ...) + for x in self.with_dirs: + if self.cfg['with%s' % x]: + extra_src_dirs.append(x) + + # see if modules are loaded + # if module is available, just use the --with-X GCC configure option + for extra in copy(extra_src_dirs): + envvar = get_software_root(extra) + if envvar: + configopts += " --with-%s=%s" % (extra, envvar) + extra_src_dirs.remove(extra) + elif extra in self.with_dirs and stage in ["stage1", "stage3"]: + # building CLooG or PPL or ISL requires a recent compiler + # our best bet is to do a 3-staged build of GCC, and + # build CLooG/PPL/ISL with the GCC we're building in stage 2 + # then (bootstrap) build GCC in stage 3 + # also, no need to stage cloog/ppl/isl in stage3 (may even cause troubles) + self.stagedbuild = True + extra_src_dirs.remove(extra) + + # try and find source directories with given prefixes + # these sources should be included in list of sources in .eb spec file, + # so EasyBuild can unpack them in the build dir + found_src_dirs = [] + versions = {} + names = {} + all_dirs = os.listdir(self.builddir) + for d in all_dirs: + for sd in extra_src_dirs: + if d.startswith(sd): + found_src_dirs.append({ + 'source_dir': d, + 'target_dir': sd + }) + # expected format: get_name[-subname]-get_version + ds = os.path.basename(d).split('-') + name = '-'.join(ds[0:-1]) + names.update({sd: name}) + ver = ds[-1] + versions.update({sd: ver}) + + # we need to find all dirs specified, or else... + if not len(found_src_dirs) == len(extra_src_dirs): + raise EasyBuildError("Couldn't find all source dirs %s: found %s from %s", + extra_src_dirs, found_src_dirs, all_dirs) + + # copy to a dir with name as expected by GCC build framework + for d in found_src_dirs: + src = os.path.join(self.builddir, d['source_dir']) + if target_prefix: + dst = os.path.join(target_prefix, d['target_dir']) + else: + dst = os.path.join(self.cfg['start_dir'], d['target_dir']) + if not os.path.exists(dst): + try: + shutil.copytree(src, dst) + except OSError as err: + raise EasyBuildError("Failed to copy src %s to dst %s: %s", src, dst, err) + self.log.debug("Copied %s to %s, so GCC can build %s" % (src, dst, d['target_dir'])) + else: + self.log.debug("No need to copy %s to %s, it's already there." % (src, dst)) + else: + # in versions prior to GCC v4.5, there's no support for extra source dirs, so return only empty info + configopts = '' + names = {} + versions = {} + + return { + 'configopts': configopts, + 'names': names, + 'versions': versions + } + + def run_configure_cmd(self, cmd): + """ + Run a configure command, with some extra checking (e.g. for unrecognized options). + """ + # note: this also triggers the use of an updated config.guess script + # (unless both the 'build_type' and 'host_type' easyconfig parameters are specified) + build_type, host_type = self.determine_build_and_host_type() + if build_type: + cmd += ' --build=' + build_type + if host_type: + cmd += ' --host=' + host_type + + (out, ec) = run_cmd("%s %s" % (self.cfg['preconfigopts'], cmd), log_all=True, simple=False) + + if ec != 0: + raise EasyBuildError("Command '%s' exited with exit code != 0 (%s)", cmd, ec) + + # configure scripts tend to simply ignore unrecognized options + # we should be more strict here, because GCC is very much a moving target + unknown_re = re.compile("WARNING: unrecognized options") + + unknown_options = unknown_re.findall(out) + if unknown_options: + raise EasyBuildError("Unrecognized options found during configure: %s", unknown_options) + + def prepare_step(self, *args, **kwargs): + """ + Prepare build environment, track currently active build stage + """ + super(EB_GCC, self).prepare_step(*args, **kwargs) + + # Set the current build stage to the specified stage based on the iteration index + self.current_stage = self.build_stages[self.iter_idx] + + def configure_step(self): + """ + Configure for GCC build: + - prepare extra source dirs (GMP, MPFR, MPC, ...) + - create obj dir to build in (GCC doesn't like to be built in source dir) + - add configure and make options, according to .eb spec file + - decide whether or not to do a staged build (which is required to enable PPL/CLooG support) + - set platform_lib based on config.guess output + """ + + sysroot = build_option('sysroot') + if sysroot: + # based on changes made to GCC in Gentoo Prefix + # https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/features/prefix/standalone/profile.bashrc + + # add --with-sysroot configure option, to instruct GCC to consider + # value set for EasyBuild's --sysroot configuration option as the root filesystem of the operating system + # (see https://gcc.gnu.org/install/configure.html) + self.cfg.update('configopts', '--with-sysroot=%s' % sysroot) + + # # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c + # apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')]) + + # # prefix dynamic linkers with sysroot + # # this patches lines like: + # # #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2" + # # for PowerPC (rs6000) we have to set DYNAMIC_LINKER_PREFIX to sysroot + # gcc_config_headers = glob.glob(os.path.join('gcc', 'config', '*', '*linux*.h')) + # regex_subs = [ + # ('(_DYNAMIC_LINKER.*[":])/lib', r'\1%s/lib' % sysroot), + # ('(DYNAMIC_LINKER_PREFIX\\s+)""', r'\1"%s"' % sysroot), + # ] + # for gcc_config_header in gcc_config_headers: + # apply_regex_substitutions(gcc_config_header, regex_subs) + + # self.configopts will be reused in a 3-staged build, + # configopts is only used in first configure + self.configopts = self.cfg['configopts'] + + # I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them + stage1_info = self.prep_extra_src_dirs("stage1") + configopts = stage1_info['configopts'] + + # II) update config options + + # enable specified language support + if self.cfg['languages']: + self.configopts += " --enable-languages=%s" % ','.join(self.cfg['languages']) + + if self.cfg['withnvptx'] or self.cfg['withamdgcn']: + + # Check that sources for optional dependencies for NVPTX and/or AMD GCN offload support are available + source_deps = [ + (['withnvptx'], 'nvptx-tools', 'nvptx_tools_dir'), + (['withamdgcn'], 'LLD', 'lld_dir'), + (['withamdgcn'], 'LLVM', 'llvm_dir'), + (['withamdgcn', 'withnvptx'], 'newlib', 'newlib_dir'), + ] + + for (with_opts, dep_name, target_var_name) in source_deps: + if any(self.cfg[x] for x in with_opts): + hits = glob.glob(os.path.join(self.builddir, dep_name.lower() + '-*')) + if len(hits) == 1: + setattr(self, target_var_name, hits[0]) + self.log.info("Found sources for %s at %s (%s)", dep_name, hits[0], target_var_name) + else: + with_opts_or = ' or '.join("'%s'" % x for x in with_opts) + error_msg = "Easyconfig enables %s, " % with_opts_or + error_msg += "but '%s' sources were not found. " % dep_name + error_msg += "Make sure that sources for %s are listed in the easyconfig file!" % dep_name + raise EasyBuildError(error_msg) + + if self.current_stage == HOST_COMPILER: + self.configopts += " --without-cuda-driver" + offload_target = [] + if self.cfg['withnvptx']: + offload_target += ['nvptx-none'] + if self.cfg['withamdgcn']: + offload_target += ['amdgcn-amdhsa'] + self.configopts += " --enable-offload-targets=%s" % ','.join(offload_target) + else: + # register installed GCC as compiler to use nvptx + path = "%s/bin:%s" % (self.installdir, os.getenv('PATH')) + env.setvar('PATH', path) + + ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % { + 'dir': self.installdir, + 'val': os.getenv('LD_LIBRARY_PATH') + } + env.setvar('LD_LIBRARY_PATH', ld_lib_path) + + if self.current_stage == NVPTX_TOOLS: + # Configure NVPTX tools and build + change_dir(self.nvptx_tools_dir) + return super(EB_GCC, self).configure_step() + + elif self.current_stage == AMD_LLVM: + # determine LLVM target to use for host CPU + cpu_arch = get_cpu_architecture() + try: + llvm_target_cpu = LLVM_ARCH_MAP[cpu_arch][0] + except KeyError: + raise EasyBuildError("Failed to determine LLVM target for host CPU (%s)" % cpu_arch) + + # Setup symlink from LLD source to 'builddir/lld' so that LLVM can find it + symlink(self.lld_dir, '%s/lld' % self.builddir) + # Build LLD tools from LLVM needed for AMD offloading + # Build LLVM in separate directory + self.create_dir('build_llvm_amdgcn') + cmd = ' '.join([ + 'cmake', + "-DLLVM_TARGETS_TO_BUILD='%s;AMDGPU'" % llvm_target_cpu, + "-DLLVM_ENABLE_PROJECTS=lld", + "-DCMAKE_BUILD_TYPE=Release", + self.llvm_dir, + ]) + run_cmd(cmd, log_all=True, simple=True) + # Need to terminate the current configuration step, but we can't run 'configure' since LLVM uses + # CMake, we therefore run 'CMake' manually and then return nothing. + # The normal make stage will build LLVM for us as expected, note that we override the install step + # further down in this file to avoid installing LLVM + return + + elif self.current_stage in (AMD_NEWLIB, NVIDIA_NEWLIB): + + symlink(os.path.join(self.newlib_dir, 'newlib'), 'newlib') + self.cfg.update('configopts', self.configopts) + + if self.current_stage == NVIDIA_NEWLIB: + # compile nvptx target compiler + self.create_dir("build-nvptx-gcc") + target = 'nvptx-none' + self.cfg.update('configopts', "--enable-newlib-io-long-long") + else: + # compile AMD GCN target compiler + self.create_dir("build-amdgcn-gcc") + target = 'amdgcn-amdhsa' + self.cfg.update('configopts', "--with-newlib") + self.cfg.update('configopts', "--disable-libquadmath") + + build_tools_dir = os.path.join(self.installdir, target, 'bin') + self.cfg.update('configopts', '--with-build-time-tools=' + build_tools_dir) + self.cfg.update('configopts', '--target=' + target) + host_type = self.determine_build_and_host_type()[1] + self.cfg.update('configopts', "--enable-as-accelerator-for=%s" % host_type) + self.cfg.update('configopts', "--disable-sjlj-exceptions") + + self.cfg['configure_cmd_prefix'] = '../' + return super(EB_GCC, self).configure_step() + + else: + raise EasyBuildError("Unknown offload configure step: %s, available: %s" + % (self.current_stage, self.build_stages)) + + # enable building of libiberty, if desired + if self.cfg['withlibiberty']: + self.configopts += " --enable-install-libiberty" + + # enable link-time-optimization (LTO) support, if desired + if self.cfg['withlto']: + self.configopts += " --enable-lto" + else: + self.configopts += " --disable-lto" + + # configure for a release build + self.configopts += " --enable-checking=release " + # enable multilib: allow both 32 and 64 bit + if self.cfg['multilib']: + glibc_32bit = [ + "glibc.i686", # Fedora, RedHat-based + "glibc.ppc", # "" on Power + "libc6-dev-i386", # Debian-based + "gcc-c++-32bit", # OpenSuSE, SLES + ] + if not any([check_os_dependency(dep) for dep in glibc_32bit]): + raise EasyBuildError("Using multilib requires 32-bit glibc (install one of %s, depending on your OS)", + ', '.join(glibc_32bit)) + self.configopts += " --enable-multilib --with-multilib-list=m32,m64" + else: + self.configopts += " --disable-multilib" + # build both static and dynamic libraries (???) + self.configopts += " --enable-shared=yes --enable-static=yes " + + # use POSIX threads + self.configopts += " --enable-threads=posix " + + # enable plugin support + self.configopts += " --enable-plugins " + + # use GOLD as default linker, except on RISC-V (since it's not supported there) + if get_cpu_family() == RISCV: + self.configopts += " --disable-gold --enable-ld=default" + elif self.cfg['use_gold_linker']: + self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold" + else: + self.configopts += " --enable-gold --enable-ld=default" + + # enable bootstrap build for self-containment (unless for staged build) + if not self.stagedbuild: + configopts += " --enable-bootstrap" + else: + configopts += " --disable-bootstrap" + + if self.stagedbuild: + # + # STAGE 1: configure GCC build that will be used to build PPL/CLooG + # + self.log.info("Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support...") + self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb') + configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.stage1installdir} + + else: + # unstaged build, so just run standard configure/make/make install + # set prefixes + self.log.info("Performing regular GCC build...") + configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir} + + # prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config + # only do this when multilib is not enabled + if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']: + cfgfile = 'gcc/config/i386/t-linux64' + multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32" + self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames) + write_file(cfgfile, multilib_osdirnames, append=True) + elif self.cfg['multilib']: + self.log.info("Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled") + + # III) create obj dir to build in, and change to it + # GCC doesn't like to be built in the source dir + if self.stagedbuild: + objdir = self.create_dir("stage1_obj") + self.stage1prefix = objdir + else: + objdir = self.create_dir("obj") + + # IV) actual configure, but not on default path + cmd = "../configure %s %s" % (self.configopts, configopts) + + self.run_configure_cmd(cmd) + + self.disable_lto_mpfr_old_gcc(objdir) + + def build_step(self): + """Custom (staged) build step for GCC.""" + + if self.iter_idx > 0: + # call standard build_step for nvptx-tools and nvptx GCC + return super(EB_GCC, self).build_step() + + if self.stagedbuild: + + # make and install stage 1 build of GCC + paracmd = '' + if self.cfg['parallel']: + paracmd = "-j %s" % self.cfg['parallel'] + + cmd = "%s make %s %s" % (self.cfg['prebuildopts'], paracmd, self.cfg['buildopts']) + run_cmd(cmd, log_all=True, simple=True) + + cmd = "make install %s" % (self.cfg['installopts']) + run_cmd(cmd, log_all=True, simple=True) + + # register built GCC as compiler to use for stage 2/3 + path = "%s/bin:%s" % (self.stage1installdir, os.getenv('PATH')) + env.setvar('PATH', path) + + ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % { + 'dir': self.stage1installdir, + 'val': os.getenv('LD_LIBRARY_PATH') + } + env.setvar('LD_LIBRARY_PATH', ld_lib_path) + + # + # STAGE 2: build GMP/PPL/CLooG for stage 3 + # + + # create dir to build GMP/PPL/CLooG in + stage2dir = "stage2_stuff" + stage2prefix = self.create_dir(stage2dir) + + # prepare directories to build GMP/PPL/CLooG + stage2_info = self.prep_extra_src_dirs("stage2", target_prefix=stage2prefix) + configopts = stage2_info['configopts'] + + # build PPL and CLooG (GMP as dependency) + + for lib in ["gmp"] + self.with_dirs: + self.log.debug("Building %s in stage 2" % lib) + if lib == "gmp" or self.cfg['with%s' % lib]: + libdir = os.path.join(stage2prefix, lib) + try: + os.chdir(libdir) + except OSError as err: + raise EasyBuildError("Failed to change to %s: %s", libdir, err) + if lib == "gmp": + cmd = "./configure --prefix=%s " % stage2prefix + cmd += "--with-pic --disable-shared --enable-cxx " + + # ensure generic build when 'generic' is set to True or when --optarch=GENERIC is used + # non-generic build can be enforced with generic=False if --optarch=GENERIC is used + optarch_generic = build_option('optarch') == OPTARCH_GENERIC + if self.cfg['generic'] or (optarch_generic and self.cfg['generic'] is not False): + cmd += "--enable-fat " + + elif lib == "ppl": + self.pplver = LooseVersion(stage2_info['versions']['ppl']) + + cmd = "./configure --prefix=%s --with-pic -disable-shared " % stage2prefix + # only enable C/C++ interfaces (Java interface is sometimes troublesome) + cmd += "--enable-interfaces='c c++' " + + # enable watchdog (or not) + if self.pplver <= LooseVersion("0.11"): + if self.cfg['pplwatchdog']: + cmd += "--enable-watchdog " + else: + cmd += "--disable-watchdog " + elif self.cfg['pplwatchdog']: + raise EasyBuildError("Enabling PPL watchdog only supported in PPL <= v0.11 .") + + # make sure GMP we just built is found + cmd += "--with-gmp=%s " % stage2prefix + elif lib == "isl": + cmd = "./configure --prefix=%s --with-pic --disable-shared " % stage2prefix + cmd += "--with-gmp=system --with-gmp-prefix=%s " % stage2prefix + + # ensure generic build when 'generic' is set to True or when --optarch=GENERIC is used + # non-generic build can be enforced with generic=False if --optarch=GENERIC is used + optarch_generic = build_option('optarch') == OPTARCH_GENERIC + if self.cfg['generic'] or (optarch_generic and self.cfg['generic'] is not False): + cmd += "--without-gcc-arch " + + elif lib == "cloog": + self.cloogname = stage2_info['names']['cloog'] + self.cloogver = LooseVersion(stage2_info['versions']['cloog']) + v0_15 = LooseVersion("0.15") + v0_16 = LooseVersion("0.16") + + cmd = "./configure --prefix=%s --with-pic --disable-shared " % stage2prefix + + # use ISL or PPL + if self.cfg['clooguseisl']: + if self.cfg['withisl']: + self.log.debug("Using external ISL for CLooG") + cmd += "--with-isl=system --with-isl-prefix=%s " % stage2prefix + elif self.cloogver >= v0_16: + self.log.debug("Using bundled ISL for CLooG") + cmd += "--with-isl=bundled " + else: + raise EasyBuildError("Using ISL is only supported in CLooG >= v0.16 (detected v%s).", + self.cloogver) + else: + if self.cloogname == "cloog-ppl" and self.cloogver >= v0_15 and self.cloogver < v0_16: + cmd += "--with-ppl=%s " % stage2prefix + else: + errormsg = "PPL only supported with CLooG-PPL v0.15.x (detected v%s)" % self.cloogver + errormsg += "\nNeither using PPL or ISL-based ClooG, I'm out of options..." + raise EasyBuildError(errormsg) + + # make sure GMP is found + if self.cloogver >= v0_15 and self.cloogver < v0_16: + cmd += "--with-gmp=%s " % stage2prefix + elif self.cloogver >= v0_16: + cmd += "--with-gmp=system --with-gmp-prefix=%s " % stage2prefix + else: + raise EasyBuildError("Don't know how to specify location of GMP to configure of CLooG v%s.", + self.cloogver) + else: + raise EasyBuildError("Don't know how to configure for %s", lib) + + # configure + self.run_configure_cmd(cmd) + + # build and 'install' + cmd = "make %s" % paracmd + run_cmd(cmd, log_all=True, simple=True) + + cmd = "make install" + run_cmd(cmd, log_all=True, simple=True) + + if lib == "gmp": + # make sure correct GMP is found + libpath = os.path.join(stage2prefix, 'lib') + + # fall back to lib64 directory if lib was not found, + # give up if neither are there + if not os.path.exists(libpath): + libpath += '64' + if not os.path.exists(libpath): + raise EasyBuildError("lib(64) subdirectory not found in %s!" % stage2prefix) + + incpath = os.path.join(stage2prefix, 'include') + + cppflags = os.getenv('CPPFLAGS', '') + env.setvar('CPPFLAGS', "%s -L%s -I%s " % (cppflags, libpath, incpath)) + + # + # STAGE 3: bootstrap build of final GCC (with PPL/CLooG support) + # + + # create new obj dir and change into it + self.create_dir("stage3_obj") + + # reconfigure for stage 3 build + self.log.info("Stage 2 of 3-staged build completed, continuing with stage 3 " + "(with CLooG and/or PPL, ISL support enabled)...") + + stage3_info = self.prep_extra_src_dirs("stage3") + configopts = stage3_info['configopts'] + configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir} + + # enable bootstrapping for self-containment + configopts += " --enable-bootstrap " + + # PPL config options + if self.cfg['withppl']: + # for PPL build and CLooG-PPL linking + for lib in ["lib64", "lib"]: + path = os.path.join(self.stage1installdir, lib, "libstdc++.a") + if os.path.exists(path): + libstdcxxpath = path + break + configopts += "--with-host-libstdcxx='-static-libgcc %s -lm' " % libstdcxxpath + + configopts += "--with-ppl=%s " % stage2prefix + + if self.pplver <= LooseVersion("0.11"): + if self.cfg['pplwatchdog']: + configopts += "--enable-watchdog " + else: + configopts += "--disable-watchdog " + + # CLooG config options + if self.cfg['withcloog']: + configopts += "--with-cloog=%s " % stage2prefix + + gccver = LooseVersion(self.version) + if self.cfg['clooguseisl'] and self.cloogver >= LooseVersion('0.16') and gccver < LooseVersion('4.8.0'): + configopts += "--enable-cloog-backend=isl " + + if self.cfg['withisl']: + configopts += "--with-isl=%s " % stage2prefix + + # configure + cmd = "../configure %s %s" % (self.configopts, configopts) + self.run_configure_cmd(cmd) + + # build with bootstrapping for self-containment + if self.cfg['profiled']: + self.cfg.update('buildopts', 'profiledbootstrap') + else: + self.cfg.update('buildopts', 'bootstrap') + + # call standard build_step + super(EB_GCC, self).build_step() + + def install_step(self, *args, **kwargs): + """Custom install step: avoid installing LLVM when building with AMD GCN offloading support""" + # When building AMD offloading do not install the LLVM source files with 'make install', instead move a few + # binaries that GCC expects when building 'newlib' so that it can reuse the LLVM GCN support + # https://gcc.gnu.org/wiki/Offloading#For_AMD_GCN: + if self.current_stage == AMD_LLVM: + llvm_binaries = [ + ('ar', 'llvm-ar'), + ('as', 'llvm-mc'), + ('ld', 'lld'), + ('nm', 'llvm-nm'), + ('ranlib', 'llvm-ar'), + ] + hits = glob.glob(os.path.join(self.builddir, 'gcc-*')) + if len(hits) == 1: + gcc_build_dir = hits[0] + for gcc_tool, llvm_tool in llvm_binaries: + from_path = os.path.join(gcc_build_dir, 'build_llvm_amdgcn', 'bin', llvm_tool) + to_path = os.path.join(self.installdir, 'amdgcn-amdhsa', 'bin', gcc_tool) + copy_file(from_path, to_path) + else: + raise EasyBuildError("Failed to isolate GCC build directory in %s", self.builddir) + + else: + super(EB_GCC, self).install_step(*args, **kwargs) + + def post_install_step(self, *args, **kwargs): + """ + Post-processing after installation: add symlinks for cc, c++, f77, f95 + """ + super(EB_GCC, self).post_install_step(*args, **kwargs) + + # Add symlinks for cc/c++/f77/f95. + bindir = os.path.join(self.installdir, 'bin') + for key in COMP_CMD_SYMLINKS: + src = COMP_CMD_SYMLINKS[key] + target = os.path.join(bindir, key) + if os.path.exists(target): + self.log.info("'%s' already exists in %s, not replacing it with symlink to '%s'", + key, bindir, src) + elif os.path.exists(os.path.join(bindir, src)): + symlink(src, target, use_abspath_source=False) + else: + raise EasyBuildError("Can't link '%s' to non-existing location %s", target, os.path.join(bindir, src)) + + # Rename include-fixed directory which includes system header files that were processed by fixincludes, + # since these may cause problems when upgrading to newer OS version. + # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/10666) + glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version, 'include-fixed') + paths = glob.glob(glob_pattern) + if paths: + # weed out paths that point to the same location, + # for example when 'lib64' is a symlink to 'lib' + include_fixed_paths = [] + for path in paths: + if not any(os.path.samefile(path, x) for x in include_fixed_paths): + include_fixed_paths.append(path) + + if len(include_fixed_paths) == 1: + include_fixed_path = include_fixed_paths[0] + + msg = "Found include-fixed subdirectory at %s, " + msg += "renaming it to avoid using system header files patched by fixincludes..." + self.log.info(msg, include_fixed_path) + + # limits.h and syslimits.h need to be copied to include/ first, + # these are strictly required (by /usr/include/limits.h for example) + include_path = os.path.join(os.path.dirname(include_fixed_path), 'include') + retained_header_files = ['limits.h', 'syslimits.h'] + for fn in retained_header_files: + from_path = os.path.join(include_fixed_path, fn) + to_path = os.path.join(include_path, fn) + if os.path.exists(from_path): + if os.path.exists(to_path): + raise EasyBuildError("%s already exists, not overwriting it with %s!", to_path, from_path) + else: + copy_file(from_path, to_path) + self.log.info("%s copied to %s before renaming %s", from_path, to_path, include_fixed_path) + else: + self.log.warning("Can't copy non-existing file %s to %s, since it doesn't exist!", + from_path, to_path) + + readme = os.path.join(include_fixed_path, 'README.easybuild') + readme_txt = '\n'.join([ + "This directory was renamed by EasyBuild to avoid that the header files in it are picked up,", + "since they may cause problems when the OS is upgraded to a new (minor) version.", + '', + "These files were copied to %s first: %s" % (include_path, ', '.join(retained_header_files)), + '', + "See https://github.com/easybuilders/easybuild-easyconfigs/issues/10666 for more information.", + '', + ]) + write_file(readme, readme_txt) + + include_fixed_renamed = include_fixed_path + '.renamed-by-easybuild' + move_file(include_fixed_path, include_fixed_renamed) + self.log.info("%s renamed to %s to avoid using the header files in it", + include_fixed_path, include_fixed_renamed) + else: + raise EasyBuildError("Exactly one 'include-fixed' directory expected, found %d: %s", + len(include_fixed_paths), include_fixed_paths) + else: + self.log.info("No include-fixed subdirectory found at %s", glob_pattern) + + def run_all_steps(self, *args, **kwargs): + """ + If 'withnvptx' or 'withamdgcn' is set, use iterated build: + iteration 0 builds the regular host compiler + iteration 1 builds nvptx-tools + iteration 2 builds the nvptx target compiler + iteration 3 builds LLD for AMD GCN offloading + iteration 4 builds the AMD GCN target compiler using LLD from above + """ + self.cfg['configopts'] = [''] + self.cfg['buildopts'] = [''] + if self.cfg['withnvptx']: + # Add two additional configure and build stages when enabling NVPTX + # this is required to first build NVPTX tools and then newlib + self.cfg['configopts'] += ['', ''] + self.cfg['buildopts'] += ['', ''] + self.build_stages.append(NVPTX_TOOLS) + self.build_stages.append(NVIDIA_NEWLIB) + if self.cfg['withamdgcn']: + # Add two additional stages when enabling AMD GCN + # this is required to first build LLVM tools and then newlib + self.cfg['configopts'] += ['', ''] + self.cfg['buildopts'] += ['', ''] + self.build_stages.append(AMD_LLVM) + self.build_stages.append(AMD_NEWLIB) + return super(EB_GCC, self).run_all_steps(*args, **kwargs) + + def sanity_check_step(self): + """ + Custom sanity check for GCC + """ + + os_type = get_os_type() + sharedlib_ext = get_shared_lib_ext() + + # determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html + # this differs across GCC versions; + # x86_64-unknown-linux-gnu was common for old GCC versions, + # x86_64-pc-linux-gnu is more likely with an updated config.guess script; + # since this is internal to GCC, we don't really care how it is named exactly, + # we only care that it's actually there + + # we may get multiple hits (libexec/, lib/), which is fine, + # but we expect the same configuration name subdirectory in each of them + glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version) + matches = glob.glob(glob_pattern) + if matches: + cands = nub([os.path.basename(os.path.dirname(x)) for x in matches]) + if len(cands) == 1: + config_name_subdir = cands[0] + else: + raise EasyBuildError("Found multiple candidates for configuration name: %s", ', '.join(cands)) + else: + raise EasyBuildError("Failed to determine configuration name: no matches for '%s'", glob_pattern) + + bin_files = ["gcov"] + lib_files = [] + if LooseVersion(self.version) >= LooseVersion('4.2'): + # libgomp was added in GCC 4.2.0 + ["libgomp.%s" % sharedlib_ext, "libgomp.a"] + if os_type == 'Linux': + lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) + # libmudflap is replaced by asan (see release notes gcc 4.9.0) + if LooseVersion(self.version) < LooseVersion("4.9.0"): + lib_files.extend(["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) + else: + lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) + libexec_files = [] + dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)] + + languages = self.cfg['languages'] or ['c', 'c++', 'fortran'] # default languages + + if 'c' in languages: + bin_files.extend(['cpp', 'gcc']) + libexec_files.extend(['cc1', 'collect2']) + + if 'c++' in languages: + bin_files.extend(['c++', 'g++']) + dirs.append('include/c++/%s' % self.version) + lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) + libexec_files.append('cc1plus') # c++ requires c, so collect2 not mentioned again + + if 'fortran' in languages: + bin_files.append('gfortran') + lib_files.extend(['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) + libexec_files.append('f951') + + if self.cfg['withlto']: + libexec_files.extend(['lto1', 'lto-wrapper']) + if os_type in ['Linux']: + libexec_files.append('liblto_plugin.%s' % sharedlib_ext) + + if self.cfg['withnvptx']: + bin_files.extend(['nvptx-none-as', 'nvptx-none-ld']) + lib_files.append('libgomp-plugin-nvptx.%s' % sharedlib_ext) + + if self.cfg['withamdgcn']: + lib_files.append('libgomp-plugin-gcn.%s' % sharedlib_ext) + + bin_files = ["bin/%s" % x for x in bin_files] + libdirs64 = ['lib64'] + libdirs32 = ['lib', 'lib32'] + libdirs = libdirs64 + libdirs32 + if self.cfg['multilib']: + # with multilib enabled, both lib and lib64 should be there + lib_files64 = [os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files] + lib_files32 = [tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files] + lib_files = lib_files64 + lib_files32 + else: + # lib64 on SuSE and Darwin, lib otherwise + lib_files = [tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files] + # lib on SuSE, libexec otherwise + libdirs = ['libexec', 'lib'] + common_infix = os.path.join('gcc', config_name_subdir, self.version) + libexec_files = [tuple([os.path.join(d, common_infix, x) for d in libdirs]) for x in libexec_files] + + old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] + + custom_paths = { + 'files': bin_files + lib_files + libexec_files + old_cmds, + 'dirs': dirs, + } + + custom_commands = [] + for lang, compiler in (('c', 'gcc'), ('c++', 'g++')): + if lang in languages: + # Simple test compile + cmd = 'echo "int main(){} " | %s -x %s -o/dev/null -' + compiler_path = os.path.join(self.installdir, 'bin', compiler) + custom_commands.append(cmd % (compiler_path, lang)) + if self.cfg['withlto']: + custom_commands.append(cmd % (compiler_path, lang + ' -flto -fuse-linker-plugin')) + if custom_commands: + # Load binutils to do the compile tests + extra_modules = [d['short_mod_name'] for d in self.cfg.dependencies() if d['name'] == 'binutils'] + else: + extra_modules = None + + super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, + extra_modules=extra_modules) + + def make_module_req_guess(self): + """ + GCC can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH + """ + guesses = super(EB_GCC, self).make_module_req_guess() + guesses.update({ + 'PATH': ['bin'], + 'CPATH': [], + 'LIBRARY_PATH': [], + 'LD_LIBRARY_PATH': ['lib', 'lib64'], + 'MANPATH': ['man', 'share/man'] + }) + return guesses From 3532ec87cbb0290873d4ebdc2cdb77c54584e302 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 17:09:06 +0200 Subject: [PATCH 091/122] use patched gcc easyblock when building gcc --- eessi-2023.04.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 15d853221a..b71ed28205 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -1,5 +1,7 @@ easyconfigs: # - GCC-9.3.0.eb - GCC-10.3.0.eb + options: + include-easyblocks: 'easyblocks/g/gcc.py' # - GCC-11.3.0.eb # - GCC-12.2.0.eb From f46e4234066b8870fd93b6b066d163164f6b5612 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 17:25:45 +0200 Subject: [PATCH 092/122] fix bug in easystack file --- eessi-2023.04.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index b71ed28205..38c3166473 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -1,6 +1,6 @@ easyconfigs: # - GCC-9.3.0.eb - - GCC-10.3.0.eb + - GCC-10.3.0.eb: options: include-easyblocks: 'easyblocks/g/gcc.py' # - GCC-11.3.0.eb From 444e8e9b9bda1bf87d411f237c956ca60633572c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 18:00:36 +0200 Subject: [PATCH 093/122] use single eb command to test special easyblock --- EESSI-pilot-install-software.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ef6e2b805b..5b9284dc03 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -208,9 +208,11 @@ else fatal_error "EasyBuild not working?!" fi -echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." -eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out -${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} +${EB:-eb} --robot GCC-10.3.0.eb --include-easyblocks=${PWD}/easyblocks/g/gcc.py + +#echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." +#eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out +#${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From d62323e6d7d43594819aee95c146acc05c8f8a83 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 18:50:00 +0200 Subject: [PATCH 094/122] keep most sysroot modifications --- easyblocks/g/gcc.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/easyblocks/g/gcc.py b/easyblocks/g/gcc.py index df4726e086..800f726696 100644 --- a/easyblocks/g/gcc.py +++ b/easyblocks/g/gcc.py @@ -360,17 +360,17 @@ def configure_step(self): # # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c # apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')]) - # # prefix dynamic linkers with sysroot - # # this patches lines like: - # # #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2" - # # for PowerPC (rs6000) we have to set DYNAMIC_LINKER_PREFIX to sysroot - # gcc_config_headers = glob.glob(os.path.join('gcc', 'config', '*', '*linux*.h')) - # regex_subs = [ - # ('(_DYNAMIC_LINKER.*[":])/lib', r'\1%s/lib' % sysroot), - # ('(DYNAMIC_LINKER_PREFIX\\s+)""', r'\1"%s"' % sysroot), - # ] - # for gcc_config_header in gcc_config_headers: - # apply_regex_substitutions(gcc_config_header, regex_subs) + # prefix dynamic linkers with sysroot + # this patches lines like: + # #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2" + # for PowerPC (rs6000) we have to set DYNAMIC_LINKER_PREFIX to sysroot + gcc_config_headers = glob.glob(os.path.join('gcc', 'config', '*', '*linux*.h')) + regex_subs = [ + ('(_DYNAMIC_LINKER.*[":])/lib', r'\1%s/lib' % sysroot), + ('(DYNAMIC_LINKER_PREFIX\\s+)""', r'\1"%s"' % sysroot), + ] + for gcc_config_header in gcc_config_headers: + apply_regex_substitutions(gcc_config_header, regex_subs) # self.configopts will be reused in a 3-staged build, # configopts is only used in first configure From 5a33e5cdeb2c44590a4a2cfac2b9674c1959ceaf Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 21 Apr 2023 22:28:41 +0200 Subject: [PATCH 095/122] use opened easyblock PR 2921 --- EESSI-pilot-install-software.sh | 8 +- easyblocks/g/gcc.py | 1053 ------------------------------- eessi-2023.04.yml | 2 +- 3 files changed, 4 insertions(+), 1059 deletions(-) delete mode 100644 easyblocks/g/gcc.py diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 5b9284dc03..ef6e2b805b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -208,11 +208,9 @@ else fatal_error "EasyBuild not working?!" fi -${EB:-eb} --robot GCC-10.3.0.eb --include-easyblocks=${PWD}/easyblocks/g/gcc.py - -#echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." -#eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out -#${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} +echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." +eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out +${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" diff --git a/easyblocks/g/gcc.py b/easyblocks/g/gcc.py deleted file mode 100644 index 800f726696..0000000000 --- a/easyblocks/g/gcc.py +++ /dev/null @@ -1,1053 +0,0 @@ -## -# Copyright 2009-2023 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing GCC, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Toon Willems (Ghent University) -@author: Ward Poelmans (Ghent University) -@author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) -""" -import glob -import os -import re -import shutil -from copy import copy -from distutils.version import LooseVersion - -import easybuild.tools.environment as env -from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_regex_substitutions, change_dir, copy_file, move_file, symlink -from easybuild.tools.filetools import which, write_file -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import RISCV, check_os_dependency, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_gcc_version, get_shared_lib_ext, get_os_name, get_os_type -from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC -from easybuild.tools.utilities import nub - - -# Offloading stages to build -AMD_LLVM = 'AMD_GCN_LLVM' -AMD_NEWLIB = 'AMD_GCN_NEWLIB' -HOST_COMPILER = 'HOST_COMPILER' -NVIDIA_NEWLIB = 'NVIDIA_NEWLIB' -NVPTX_TOOLS = 'NVIDIA_NVPTX_TOOLS' -# Additional symlinks to create for compiler commands -COMP_CMD_SYMLINKS = { - 'cc': 'gcc', - 'c++': 'g++', - 'f77': 'gfortran', - 'f95': 'gfortran', -} - - -class EB_GCC(ConfigureMake): - """ - Self-contained build of GCC. - Uses system compiler for initial build, then bootstraps. - """ - - @staticmethod - def extra_options(): - extra_vars = { - 'clooguseisl': [False, "Use ISL with CLooG or not", CUSTOM], - 'generic': [None, "Build GCC and support libraries such that it runs on all processors of the target " - "architecture (use False to enforce non-generic regardless of configuration)", CUSTOM], - 'languages': [[], "List of languages to build GCC for (--enable-languages)", CUSTOM], - 'multilib': [False, "Build multilib gcc (both i386 and x86_64)", CUSTOM], - 'pplwatchdog': [False, "Enable PPL watchdog", CUSTOM], - 'prefer_lib_subdir': [False, "Configure GCC to prefer 'lib' subdirs over 'lib64' when linking", CUSTOM], - 'profiled': [False, "Bootstrap GCC with profile-guided optimizations", CUSTOM], - 'use_gold_linker': [None, "Configure GCC to use GOLD as default linker " - "(default: enable automatically for GCC < 11.3.0, except on RISC-V)", CUSTOM], - 'withcloog': [False, "Build GCC with CLooG support", CUSTOM], - 'withisl': [False, "Build GCC with ISL support", CUSTOM], - 'withlibiberty': [False, "Enable installing of libiberty", CUSTOM], - 'withlto': [True, "Enable LTO support", CUSTOM], - 'withppl': [False, "Build GCC with PPL support", CUSTOM], - 'withnvptx': [False, "Build GCC with NVPTX offload support", CUSTOM], - 'withamdgcn': [False, "Build GCC with AMD GCN offload support", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - super(EB_GCC, self).__init__(*args, **kwargs) - - self.stagedbuild = False - # Each build iteration is related to a specific build, first build host compiler, then potentially build NVPTX - # offloading and/or AMD GCN offloading - self.build_stages = [HOST_COMPILER] - self.current_stage = HOST_COMPILER - # Directories for additional tools needed when doing offloading to Nvidia and AMD - self.nvptx_tools_dir = None # nvptx-tools necessary for Nvidia - self.llvm_dir = None # LLVM is necessary when offloading to AMD - self.lld_dir = None # LLD is the only required component of LLVM - self.newlib_dir = None # Used by both NVPTX and AMD GCN backend - - # need to make sure version is an actual version - # required because of support in SystemCompiler generic easyblock to specify 'system' as version, - # which results in deriving the actual compiler version - # comparing a non-version like 'system' with an actual version like '2016' fails with TypeError in Python 3.x - if re.match(r'^[0-9]+\.[0-9]+.*', self.version): - version = LooseVersion(self.version) - - if version >= LooseVersion('4.8.0') and self.cfg['clooguseisl'] and not self.cfg['withisl']: - raise EasyBuildError("Using ISL bundled with CLooG is unsupported in >=GCC-4.8.0. " - "Use a seperate ISL: set withisl=True") - - # I think ISL without CLooG has no purpose in GCC < 5.0.0 ... - if version < LooseVersion('5.0.0') and self.cfg['withisl'] and not self.cfg['withcloog']: - raise EasyBuildError("Activating ISL without CLooG is pointless") - - # Disable the Gold linker by default for GCC 11.3.0 and newer, as it suffers from bitrot - if self.cfg['use_gold_linker'] is None: - self.cfg['use_gold_linker'] = version < LooseVersion('11.3.0') - - # unset some environment variables that are known to may cause nasty build errors when bootstrapping - self.cfg.update('unwanted_env_vars', ['CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH']) - # ubuntu needs the LIBRARY_PATH env var to work apparently (#363) - if get_os_name() not in ['ubuntu', 'debian']: - self.cfg.update('unwanted_env_vars', ['LIBRARY_PATH']) - - def create_dir(self, dirname): - """ - Create a dir to build in. - """ - dirpath = os.path.join(self.cfg['start_dir'], dirname) - try: - os.mkdir(dirpath) - os.chdir(dirpath) - self.log.debug("Created dir at %s" % dirpath) - return dirpath - except OSError as err: - raise EasyBuildError("Can't use dir %s to build in: %s", dirpath, err) - - def disable_lto_mpfr_old_gcc(self, objdir): - """ - # if GCC version used to build stage 1 is too old, build MPFR without LTO in stage 1 - # required for e.g. CentOS 6, cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/6374 - """ - self.log.info("Checking whether we are trying to build a recent MPFR with an old GCC...") - - # try to figure out MPFR version being built - mpfr_ver = '0.0' - mpfr_dirs = glob.glob(os.path.join(self.builddir, 'mpfr-*')) - if len(mpfr_dirs) == 1: - mpfr_dir = mpfr_dirs[0] - res = re.search('(?P[0-9.]+)$', mpfr_dir) - if res: - mpfr_ver = res.group('mpfr_ver') - self.log.info("Found MPFR version %s (based name of MPFR source dir: %s)", mpfr_ver, mpfr_dir) - else: - self.log.warning("Failed to determine MPFR version from '%s', assuming v%s", mpfr_dir, mpfr_ver) - else: - self.log.warning("Failed to isolate MPFR source dir to determine MPFR version, assuming v%s", mpfr_ver) - - # for MPFR v4.x & newer, we need a recent GCC that supports -flto - if LooseVersion(mpfr_ver) >= LooseVersion('4.0'): - - disable_mpfr_lto = False - - # check GCC version being used - # GCC 4.5 is required for -flto (cfr. https://gcc.gnu.org/gcc-4.5/changes.html) - gcc_ver = get_gcc_version() - min_gcc_ver_lto = '4.5' - if gcc_ver is None: - self.log.warning("Failed to determine GCC version, assuming it's recent enough...") - elif LooseVersion(gcc_ver) < LooseVersion(min_gcc_ver_lto): - self.log.info("Configuring MPFR to build without LTO in stage 1 (GCC %s is too old: < %s)!", - gcc_ver, min_gcc_ver_lto) - disable_mpfr_lto = True - else: - self.log.info("GCC %s (>= %s) is OK for building MPFR in stage 1 with LTO enabled", - gcc_ver, min_gcc_ver_lto) - - # check whether GCC actually supports LTO (it may be configured with --disable-lto), - # by compiling a simple C program using -flto - out, ec = run_cmd("echo 'void main() {}' | gcc -x c -flto - -o /dev/null", simple=False, log_ok=False) - gcc_path = which('gcc') - if ec: - self.log.info("GCC command %s doesn't seem to support LTO, test compile failed: %s", gcc_path, out) - disable_mpfr_lto = True - else: - self.log.info("GCC command %s provides LTO support, so using it when building MPFR", gcc_path) - - if disable_mpfr_lto: - # patch GCC's Makefile to inject --disable-lto when building MPFR - stage1_makefile = os.path.join(objdir, 'Makefile') - regex_subs = [(r'(--with-gmp-lib=\$\$r/\$\(HOST_SUBDIR\)/gmp/.libs) \\', r'\1 --disable-lto \\')] - apply_regex_substitutions(stage1_makefile, regex_subs) - - def prep_extra_src_dirs(self, stage, target_prefix=None): - """ - Prepare extra (optional) source directories, so GCC will build these as well. - """ - if LooseVersion(self.version) >= LooseVersion('4.5'): - known_stages = ["stage1", "stage2", "stage3"] - if stage not in known_stages: - raise EasyBuildError("Incorrect argument for prep_extra_src_dirs, should be one of: %s", known_stages) - - configopts = '' - if stage == "stage2": - # no MPFR/MPC needed in stage 2 - extra_src_dirs = ["gmp"] - else: - extra_src_dirs = ["gmp", "mpfr", "mpc"] - - # list of the extra dirs that are needed depending on the 'with%s' option - # the order is important: keep CLooG last! - self.with_dirs = ["isl", "ppl", "cloog"] - - # add optional ones that were selected (e.g. CLooG, PPL, ...) - for x in self.with_dirs: - if self.cfg['with%s' % x]: - extra_src_dirs.append(x) - - # see if modules are loaded - # if module is available, just use the --with-X GCC configure option - for extra in copy(extra_src_dirs): - envvar = get_software_root(extra) - if envvar: - configopts += " --with-%s=%s" % (extra, envvar) - extra_src_dirs.remove(extra) - elif extra in self.with_dirs and stage in ["stage1", "stage3"]: - # building CLooG or PPL or ISL requires a recent compiler - # our best bet is to do a 3-staged build of GCC, and - # build CLooG/PPL/ISL with the GCC we're building in stage 2 - # then (bootstrap) build GCC in stage 3 - # also, no need to stage cloog/ppl/isl in stage3 (may even cause troubles) - self.stagedbuild = True - extra_src_dirs.remove(extra) - - # try and find source directories with given prefixes - # these sources should be included in list of sources in .eb spec file, - # so EasyBuild can unpack them in the build dir - found_src_dirs = [] - versions = {} - names = {} - all_dirs = os.listdir(self.builddir) - for d in all_dirs: - for sd in extra_src_dirs: - if d.startswith(sd): - found_src_dirs.append({ - 'source_dir': d, - 'target_dir': sd - }) - # expected format: get_name[-subname]-get_version - ds = os.path.basename(d).split('-') - name = '-'.join(ds[0:-1]) - names.update({sd: name}) - ver = ds[-1] - versions.update({sd: ver}) - - # we need to find all dirs specified, or else... - if not len(found_src_dirs) == len(extra_src_dirs): - raise EasyBuildError("Couldn't find all source dirs %s: found %s from %s", - extra_src_dirs, found_src_dirs, all_dirs) - - # copy to a dir with name as expected by GCC build framework - for d in found_src_dirs: - src = os.path.join(self.builddir, d['source_dir']) - if target_prefix: - dst = os.path.join(target_prefix, d['target_dir']) - else: - dst = os.path.join(self.cfg['start_dir'], d['target_dir']) - if not os.path.exists(dst): - try: - shutil.copytree(src, dst) - except OSError as err: - raise EasyBuildError("Failed to copy src %s to dst %s: %s", src, dst, err) - self.log.debug("Copied %s to %s, so GCC can build %s" % (src, dst, d['target_dir'])) - else: - self.log.debug("No need to copy %s to %s, it's already there." % (src, dst)) - else: - # in versions prior to GCC v4.5, there's no support for extra source dirs, so return only empty info - configopts = '' - names = {} - versions = {} - - return { - 'configopts': configopts, - 'names': names, - 'versions': versions - } - - def run_configure_cmd(self, cmd): - """ - Run a configure command, with some extra checking (e.g. for unrecognized options). - """ - # note: this also triggers the use of an updated config.guess script - # (unless both the 'build_type' and 'host_type' easyconfig parameters are specified) - build_type, host_type = self.determine_build_and_host_type() - if build_type: - cmd += ' --build=' + build_type - if host_type: - cmd += ' --host=' + host_type - - (out, ec) = run_cmd("%s %s" % (self.cfg['preconfigopts'], cmd), log_all=True, simple=False) - - if ec != 0: - raise EasyBuildError("Command '%s' exited with exit code != 0 (%s)", cmd, ec) - - # configure scripts tend to simply ignore unrecognized options - # we should be more strict here, because GCC is very much a moving target - unknown_re = re.compile("WARNING: unrecognized options") - - unknown_options = unknown_re.findall(out) - if unknown_options: - raise EasyBuildError("Unrecognized options found during configure: %s", unknown_options) - - def prepare_step(self, *args, **kwargs): - """ - Prepare build environment, track currently active build stage - """ - super(EB_GCC, self).prepare_step(*args, **kwargs) - - # Set the current build stage to the specified stage based on the iteration index - self.current_stage = self.build_stages[self.iter_idx] - - def configure_step(self): - """ - Configure for GCC build: - - prepare extra source dirs (GMP, MPFR, MPC, ...) - - create obj dir to build in (GCC doesn't like to be built in source dir) - - add configure and make options, according to .eb spec file - - decide whether or not to do a staged build (which is required to enable PPL/CLooG support) - - set platform_lib based on config.guess output - """ - - sysroot = build_option('sysroot') - if sysroot: - # based on changes made to GCC in Gentoo Prefix - # https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/features/prefix/standalone/profile.bashrc - - # add --with-sysroot configure option, to instruct GCC to consider - # value set for EasyBuild's --sysroot configuration option as the root filesystem of the operating system - # (see https://gcc.gnu.org/install/configure.html) - self.cfg.update('configopts', '--with-sysroot=%s' % sysroot) - - # # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c - # apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')]) - - # prefix dynamic linkers with sysroot - # this patches lines like: - # #define GLIBC_DYNAMIC_LINKER64 "/lib64/ld-linux-x86-64.so.2" - # for PowerPC (rs6000) we have to set DYNAMIC_LINKER_PREFIX to sysroot - gcc_config_headers = glob.glob(os.path.join('gcc', 'config', '*', '*linux*.h')) - regex_subs = [ - ('(_DYNAMIC_LINKER.*[":])/lib', r'\1%s/lib' % sysroot), - ('(DYNAMIC_LINKER_PREFIX\\s+)""', r'\1"%s"' % sysroot), - ] - for gcc_config_header in gcc_config_headers: - apply_regex_substitutions(gcc_config_header, regex_subs) - - # self.configopts will be reused in a 3-staged build, - # configopts is only used in first configure - self.configopts = self.cfg['configopts'] - - # I) prepare extra source dirs, e.g. for GMP, MPFR, MPC (if required), so GCC can build them - stage1_info = self.prep_extra_src_dirs("stage1") - configopts = stage1_info['configopts'] - - # II) update config options - - # enable specified language support - if self.cfg['languages']: - self.configopts += " --enable-languages=%s" % ','.join(self.cfg['languages']) - - if self.cfg['withnvptx'] or self.cfg['withamdgcn']: - - # Check that sources for optional dependencies for NVPTX and/or AMD GCN offload support are available - source_deps = [ - (['withnvptx'], 'nvptx-tools', 'nvptx_tools_dir'), - (['withamdgcn'], 'LLD', 'lld_dir'), - (['withamdgcn'], 'LLVM', 'llvm_dir'), - (['withamdgcn', 'withnvptx'], 'newlib', 'newlib_dir'), - ] - - for (with_opts, dep_name, target_var_name) in source_deps: - if any(self.cfg[x] for x in with_opts): - hits = glob.glob(os.path.join(self.builddir, dep_name.lower() + '-*')) - if len(hits) == 1: - setattr(self, target_var_name, hits[0]) - self.log.info("Found sources for %s at %s (%s)", dep_name, hits[0], target_var_name) - else: - with_opts_or = ' or '.join("'%s'" % x for x in with_opts) - error_msg = "Easyconfig enables %s, " % with_opts_or - error_msg += "but '%s' sources were not found. " % dep_name - error_msg += "Make sure that sources for %s are listed in the easyconfig file!" % dep_name - raise EasyBuildError(error_msg) - - if self.current_stage == HOST_COMPILER: - self.configopts += " --without-cuda-driver" - offload_target = [] - if self.cfg['withnvptx']: - offload_target += ['nvptx-none'] - if self.cfg['withamdgcn']: - offload_target += ['amdgcn-amdhsa'] - self.configopts += " --enable-offload-targets=%s" % ','.join(offload_target) - else: - # register installed GCC as compiler to use nvptx - path = "%s/bin:%s" % (self.installdir, os.getenv('PATH')) - env.setvar('PATH', path) - - ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % { - 'dir': self.installdir, - 'val': os.getenv('LD_LIBRARY_PATH') - } - env.setvar('LD_LIBRARY_PATH', ld_lib_path) - - if self.current_stage == NVPTX_TOOLS: - # Configure NVPTX tools and build - change_dir(self.nvptx_tools_dir) - return super(EB_GCC, self).configure_step() - - elif self.current_stage == AMD_LLVM: - # determine LLVM target to use for host CPU - cpu_arch = get_cpu_architecture() - try: - llvm_target_cpu = LLVM_ARCH_MAP[cpu_arch][0] - except KeyError: - raise EasyBuildError("Failed to determine LLVM target for host CPU (%s)" % cpu_arch) - - # Setup symlink from LLD source to 'builddir/lld' so that LLVM can find it - symlink(self.lld_dir, '%s/lld' % self.builddir) - # Build LLD tools from LLVM needed for AMD offloading - # Build LLVM in separate directory - self.create_dir('build_llvm_amdgcn') - cmd = ' '.join([ - 'cmake', - "-DLLVM_TARGETS_TO_BUILD='%s;AMDGPU'" % llvm_target_cpu, - "-DLLVM_ENABLE_PROJECTS=lld", - "-DCMAKE_BUILD_TYPE=Release", - self.llvm_dir, - ]) - run_cmd(cmd, log_all=True, simple=True) - # Need to terminate the current configuration step, but we can't run 'configure' since LLVM uses - # CMake, we therefore run 'CMake' manually and then return nothing. - # The normal make stage will build LLVM for us as expected, note that we override the install step - # further down in this file to avoid installing LLVM - return - - elif self.current_stage in (AMD_NEWLIB, NVIDIA_NEWLIB): - - symlink(os.path.join(self.newlib_dir, 'newlib'), 'newlib') - self.cfg.update('configopts', self.configopts) - - if self.current_stage == NVIDIA_NEWLIB: - # compile nvptx target compiler - self.create_dir("build-nvptx-gcc") - target = 'nvptx-none' - self.cfg.update('configopts', "--enable-newlib-io-long-long") - else: - # compile AMD GCN target compiler - self.create_dir("build-amdgcn-gcc") - target = 'amdgcn-amdhsa' - self.cfg.update('configopts', "--with-newlib") - self.cfg.update('configopts', "--disable-libquadmath") - - build_tools_dir = os.path.join(self.installdir, target, 'bin') - self.cfg.update('configopts', '--with-build-time-tools=' + build_tools_dir) - self.cfg.update('configopts', '--target=' + target) - host_type = self.determine_build_and_host_type()[1] - self.cfg.update('configopts', "--enable-as-accelerator-for=%s" % host_type) - self.cfg.update('configopts', "--disable-sjlj-exceptions") - - self.cfg['configure_cmd_prefix'] = '../' - return super(EB_GCC, self).configure_step() - - else: - raise EasyBuildError("Unknown offload configure step: %s, available: %s" - % (self.current_stage, self.build_stages)) - - # enable building of libiberty, if desired - if self.cfg['withlibiberty']: - self.configopts += " --enable-install-libiberty" - - # enable link-time-optimization (LTO) support, if desired - if self.cfg['withlto']: - self.configopts += " --enable-lto" - else: - self.configopts += " --disable-lto" - - # configure for a release build - self.configopts += " --enable-checking=release " - # enable multilib: allow both 32 and 64 bit - if self.cfg['multilib']: - glibc_32bit = [ - "glibc.i686", # Fedora, RedHat-based - "glibc.ppc", # "" on Power - "libc6-dev-i386", # Debian-based - "gcc-c++-32bit", # OpenSuSE, SLES - ] - if not any([check_os_dependency(dep) for dep in glibc_32bit]): - raise EasyBuildError("Using multilib requires 32-bit glibc (install one of %s, depending on your OS)", - ', '.join(glibc_32bit)) - self.configopts += " --enable-multilib --with-multilib-list=m32,m64" - else: - self.configopts += " --disable-multilib" - # build both static and dynamic libraries (???) - self.configopts += " --enable-shared=yes --enable-static=yes " - - # use POSIX threads - self.configopts += " --enable-threads=posix " - - # enable plugin support - self.configopts += " --enable-plugins " - - # use GOLD as default linker, except on RISC-V (since it's not supported there) - if get_cpu_family() == RISCV: - self.configopts += " --disable-gold --enable-ld=default" - elif self.cfg['use_gold_linker']: - self.configopts += " --enable-gold=default --enable-ld --with-plugin-ld=ld.gold" - else: - self.configopts += " --enable-gold --enable-ld=default" - - # enable bootstrap build for self-containment (unless for staged build) - if not self.stagedbuild: - configopts += " --enable-bootstrap" - else: - configopts += " --disable-bootstrap" - - if self.stagedbuild: - # - # STAGE 1: configure GCC build that will be used to build PPL/CLooG - # - self.log.info("Starting with stage 1 of 3-staged build to enable CLooG and/or PPL, ISL support...") - self.stage1installdir = os.path.join(self.builddir, 'GCC_stage1_eb') - configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.stage1installdir} - - else: - # unstaged build, so just run standard configure/make/make install - # set prefixes - self.log.info("Performing regular GCC build...") - configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir} - - # prioritize lib over lib{64,32,x32} for all architectures by overriding default MULTILIB_OSDIRNAMES config - # only do this when multilib is not enabled - if self.cfg['prefer_lib_subdir'] and not self.cfg['multilib']: - cfgfile = 'gcc/config/i386/t-linux64' - multilib_osdirnames = "MULTILIB_OSDIRNAMES = m64=../lib:../lib64 m32=../lib:../lib32 mx32=../lib:../libx32" - self.log.info("Patching MULTILIB_OSDIRNAMES in %s with '%s'", cfgfile, multilib_osdirnames) - write_file(cfgfile, multilib_osdirnames, append=True) - elif self.cfg['multilib']: - self.log.info("Not patching MULTILIB_OSDIRNAMES since use of --enable-multilib is enabled") - - # III) create obj dir to build in, and change to it - # GCC doesn't like to be built in the source dir - if self.stagedbuild: - objdir = self.create_dir("stage1_obj") - self.stage1prefix = objdir - else: - objdir = self.create_dir("obj") - - # IV) actual configure, but not on default path - cmd = "../configure %s %s" % (self.configopts, configopts) - - self.run_configure_cmd(cmd) - - self.disable_lto_mpfr_old_gcc(objdir) - - def build_step(self): - """Custom (staged) build step for GCC.""" - - if self.iter_idx > 0: - # call standard build_step for nvptx-tools and nvptx GCC - return super(EB_GCC, self).build_step() - - if self.stagedbuild: - - # make and install stage 1 build of GCC - paracmd = '' - if self.cfg['parallel']: - paracmd = "-j %s" % self.cfg['parallel'] - - cmd = "%s make %s %s" % (self.cfg['prebuildopts'], paracmd, self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) - - cmd = "make install %s" % (self.cfg['installopts']) - run_cmd(cmd, log_all=True, simple=True) - - # register built GCC as compiler to use for stage 2/3 - path = "%s/bin:%s" % (self.stage1installdir, os.getenv('PATH')) - env.setvar('PATH', path) - - ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % { - 'dir': self.stage1installdir, - 'val': os.getenv('LD_LIBRARY_PATH') - } - env.setvar('LD_LIBRARY_PATH', ld_lib_path) - - # - # STAGE 2: build GMP/PPL/CLooG for stage 3 - # - - # create dir to build GMP/PPL/CLooG in - stage2dir = "stage2_stuff" - stage2prefix = self.create_dir(stage2dir) - - # prepare directories to build GMP/PPL/CLooG - stage2_info = self.prep_extra_src_dirs("stage2", target_prefix=stage2prefix) - configopts = stage2_info['configopts'] - - # build PPL and CLooG (GMP as dependency) - - for lib in ["gmp"] + self.with_dirs: - self.log.debug("Building %s in stage 2" % lib) - if lib == "gmp" or self.cfg['with%s' % lib]: - libdir = os.path.join(stage2prefix, lib) - try: - os.chdir(libdir) - except OSError as err: - raise EasyBuildError("Failed to change to %s: %s", libdir, err) - if lib == "gmp": - cmd = "./configure --prefix=%s " % stage2prefix - cmd += "--with-pic --disable-shared --enable-cxx " - - # ensure generic build when 'generic' is set to True or when --optarch=GENERIC is used - # non-generic build can be enforced with generic=False if --optarch=GENERIC is used - optarch_generic = build_option('optarch') == OPTARCH_GENERIC - if self.cfg['generic'] or (optarch_generic and self.cfg['generic'] is not False): - cmd += "--enable-fat " - - elif lib == "ppl": - self.pplver = LooseVersion(stage2_info['versions']['ppl']) - - cmd = "./configure --prefix=%s --with-pic -disable-shared " % stage2prefix - # only enable C/C++ interfaces (Java interface is sometimes troublesome) - cmd += "--enable-interfaces='c c++' " - - # enable watchdog (or not) - if self.pplver <= LooseVersion("0.11"): - if self.cfg['pplwatchdog']: - cmd += "--enable-watchdog " - else: - cmd += "--disable-watchdog " - elif self.cfg['pplwatchdog']: - raise EasyBuildError("Enabling PPL watchdog only supported in PPL <= v0.11 .") - - # make sure GMP we just built is found - cmd += "--with-gmp=%s " % stage2prefix - elif lib == "isl": - cmd = "./configure --prefix=%s --with-pic --disable-shared " % stage2prefix - cmd += "--with-gmp=system --with-gmp-prefix=%s " % stage2prefix - - # ensure generic build when 'generic' is set to True or when --optarch=GENERIC is used - # non-generic build can be enforced with generic=False if --optarch=GENERIC is used - optarch_generic = build_option('optarch') == OPTARCH_GENERIC - if self.cfg['generic'] or (optarch_generic and self.cfg['generic'] is not False): - cmd += "--without-gcc-arch " - - elif lib == "cloog": - self.cloogname = stage2_info['names']['cloog'] - self.cloogver = LooseVersion(stage2_info['versions']['cloog']) - v0_15 = LooseVersion("0.15") - v0_16 = LooseVersion("0.16") - - cmd = "./configure --prefix=%s --with-pic --disable-shared " % stage2prefix - - # use ISL or PPL - if self.cfg['clooguseisl']: - if self.cfg['withisl']: - self.log.debug("Using external ISL for CLooG") - cmd += "--with-isl=system --with-isl-prefix=%s " % stage2prefix - elif self.cloogver >= v0_16: - self.log.debug("Using bundled ISL for CLooG") - cmd += "--with-isl=bundled " - else: - raise EasyBuildError("Using ISL is only supported in CLooG >= v0.16 (detected v%s).", - self.cloogver) - else: - if self.cloogname == "cloog-ppl" and self.cloogver >= v0_15 and self.cloogver < v0_16: - cmd += "--with-ppl=%s " % stage2prefix - else: - errormsg = "PPL only supported with CLooG-PPL v0.15.x (detected v%s)" % self.cloogver - errormsg += "\nNeither using PPL or ISL-based ClooG, I'm out of options..." - raise EasyBuildError(errormsg) - - # make sure GMP is found - if self.cloogver >= v0_15 and self.cloogver < v0_16: - cmd += "--with-gmp=%s " % stage2prefix - elif self.cloogver >= v0_16: - cmd += "--with-gmp=system --with-gmp-prefix=%s " % stage2prefix - else: - raise EasyBuildError("Don't know how to specify location of GMP to configure of CLooG v%s.", - self.cloogver) - else: - raise EasyBuildError("Don't know how to configure for %s", lib) - - # configure - self.run_configure_cmd(cmd) - - # build and 'install' - cmd = "make %s" % paracmd - run_cmd(cmd, log_all=True, simple=True) - - cmd = "make install" - run_cmd(cmd, log_all=True, simple=True) - - if lib == "gmp": - # make sure correct GMP is found - libpath = os.path.join(stage2prefix, 'lib') - - # fall back to lib64 directory if lib was not found, - # give up if neither are there - if not os.path.exists(libpath): - libpath += '64' - if not os.path.exists(libpath): - raise EasyBuildError("lib(64) subdirectory not found in %s!" % stage2prefix) - - incpath = os.path.join(stage2prefix, 'include') - - cppflags = os.getenv('CPPFLAGS', '') - env.setvar('CPPFLAGS', "%s -L%s -I%s " % (cppflags, libpath, incpath)) - - # - # STAGE 3: bootstrap build of final GCC (with PPL/CLooG support) - # - - # create new obj dir and change into it - self.create_dir("stage3_obj") - - # reconfigure for stage 3 build - self.log.info("Stage 2 of 3-staged build completed, continuing with stage 3 " - "(with CLooG and/or PPL, ISL support enabled)...") - - stage3_info = self.prep_extra_src_dirs("stage3") - configopts = stage3_info['configopts'] - configopts += " --prefix=%(p)s --with-local-prefix=%(p)s" % {'p': self.installdir} - - # enable bootstrapping for self-containment - configopts += " --enable-bootstrap " - - # PPL config options - if self.cfg['withppl']: - # for PPL build and CLooG-PPL linking - for lib in ["lib64", "lib"]: - path = os.path.join(self.stage1installdir, lib, "libstdc++.a") - if os.path.exists(path): - libstdcxxpath = path - break - configopts += "--with-host-libstdcxx='-static-libgcc %s -lm' " % libstdcxxpath - - configopts += "--with-ppl=%s " % stage2prefix - - if self.pplver <= LooseVersion("0.11"): - if self.cfg['pplwatchdog']: - configopts += "--enable-watchdog " - else: - configopts += "--disable-watchdog " - - # CLooG config options - if self.cfg['withcloog']: - configopts += "--with-cloog=%s " % stage2prefix - - gccver = LooseVersion(self.version) - if self.cfg['clooguseisl'] and self.cloogver >= LooseVersion('0.16') and gccver < LooseVersion('4.8.0'): - configopts += "--enable-cloog-backend=isl " - - if self.cfg['withisl']: - configopts += "--with-isl=%s " % stage2prefix - - # configure - cmd = "../configure %s %s" % (self.configopts, configopts) - self.run_configure_cmd(cmd) - - # build with bootstrapping for self-containment - if self.cfg['profiled']: - self.cfg.update('buildopts', 'profiledbootstrap') - else: - self.cfg.update('buildopts', 'bootstrap') - - # call standard build_step - super(EB_GCC, self).build_step() - - def install_step(self, *args, **kwargs): - """Custom install step: avoid installing LLVM when building with AMD GCN offloading support""" - # When building AMD offloading do not install the LLVM source files with 'make install', instead move a few - # binaries that GCC expects when building 'newlib' so that it can reuse the LLVM GCN support - # https://gcc.gnu.org/wiki/Offloading#For_AMD_GCN: - if self.current_stage == AMD_LLVM: - llvm_binaries = [ - ('ar', 'llvm-ar'), - ('as', 'llvm-mc'), - ('ld', 'lld'), - ('nm', 'llvm-nm'), - ('ranlib', 'llvm-ar'), - ] - hits = glob.glob(os.path.join(self.builddir, 'gcc-*')) - if len(hits) == 1: - gcc_build_dir = hits[0] - for gcc_tool, llvm_tool in llvm_binaries: - from_path = os.path.join(gcc_build_dir, 'build_llvm_amdgcn', 'bin', llvm_tool) - to_path = os.path.join(self.installdir, 'amdgcn-amdhsa', 'bin', gcc_tool) - copy_file(from_path, to_path) - else: - raise EasyBuildError("Failed to isolate GCC build directory in %s", self.builddir) - - else: - super(EB_GCC, self).install_step(*args, **kwargs) - - def post_install_step(self, *args, **kwargs): - """ - Post-processing after installation: add symlinks for cc, c++, f77, f95 - """ - super(EB_GCC, self).post_install_step(*args, **kwargs) - - # Add symlinks for cc/c++/f77/f95. - bindir = os.path.join(self.installdir, 'bin') - for key in COMP_CMD_SYMLINKS: - src = COMP_CMD_SYMLINKS[key] - target = os.path.join(bindir, key) - if os.path.exists(target): - self.log.info("'%s' already exists in %s, not replacing it with symlink to '%s'", - key, bindir, src) - elif os.path.exists(os.path.join(bindir, src)): - symlink(src, target, use_abspath_source=False) - else: - raise EasyBuildError("Can't link '%s' to non-existing location %s", target, os.path.join(bindir, src)) - - # Rename include-fixed directory which includes system header files that were processed by fixincludes, - # since these may cause problems when upgrading to newer OS version. - # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/10666) - glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version, 'include-fixed') - paths = glob.glob(glob_pattern) - if paths: - # weed out paths that point to the same location, - # for example when 'lib64' is a symlink to 'lib' - include_fixed_paths = [] - for path in paths: - if not any(os.path.samefile(path, x) for x in include_fixed_paths): - include_fixed_paths.append(path) - - if len(include_fixed_paths) == 1: - include_fixed_path = include_fixed_paths[0] - - msg = "Found include-fixed subdirectory at %s, " - msg += "renaming it to avoid using system header files patched by fixincludes..." - self.log.info(msg, include_fixed_path) - - # limits.h and syslimits.h need to be copied to include/ first, - # these are strictly required (by /usr/include/limits.h for example) - include_path = os.path.join(os.path.dirname(include_fixed_path), 'include') - retained_header_files = ['limits.h', 'syslimits.h'] - for fn in retained_header_files: - from_path = os.path.join(include_fixed_path, fn) - to_path = os.path.join(include_path, fn) - if os.path.exists(from_path): - if os.path.exists(to_path): - raise EasyBuildError("%s already exists, not overwriting it with %s!", to_path, from_path) - else: - copy_file(from_path, to_path) - self.log.info("%s copied to %s before renaming %s", from_path, to_path, include_fixed_path) - else: - self.log.warning("Can't copy non-existing file %s to %s, since it doesn't exist!", - from_path, to_path) - - readme = os.path.join(include_fixed_path, 'README.easybuild') - readme_txt = '\n'.join([ - "This directory was renamed by EasyBuild to avoid that the header files in it are picked up,", - "since they may cause problems when the OS is upgraded to a new (minor) version.", - '', - "These files were copied to %s first: %s" % (include_path, ', '.join(retained_header_files)), - '', - "See https://github.com/easybuilders/easybuild-easyconfigs/issues/10666 for more information.", - '', - ]) - write_file(readme, readme_txt) - - include_fixed_renamed = include_fixed_path + '.renamed-by-easybuild' - move_file(include_fixed_path, include_fixed_renamed) - self.log.info("%s renamed to %s to avoid using the header files in it", - include_fixed_path, include_fixed_renamed) - else: - raise EasyBuildError("Exactly one 'include-fixed' directory expected, found %d: %s", - len(include_fixed_paths), include_fixed_paths) - else: - self.log.info("No include-fixed subdirectory found at %s", glob_pattern) - - def run_all_steps(self, *args, **kwargs): - """ - If 'withnvptx' or 'withamdgcn' is set, use iterated build: - iteration 0 builds the regular host compiler - iteration 1 builds nvptx-tools - iteration 2 builds the nvptx target compiler - iteration 3 builds LLD for AMD GCN offloading - iteration 4 builds the AMD GCN target compiler using LLD from above - """ - self.cfg['configopts'] = [''] - self.cfg['buildopts'] = [''] - if self.cfg['withnvptx']: - # Add two additional configure and build stages when enabling NVPTX - # this is required to first build NVPTX tools and then newlib - self.cfg['configopts'] += ['', ''] - self.cfg['buildopts'] += ['', ''] - self.build_stages.append(NVPTX_TOOLS) - self.build_stages.append(NVIDIA_NEWLIB) - if self.cfg['withamdgcn']: - # Add two additional stages when enabling AMD GCN - # this is required to first build LLVM tools and then newlib - self.cfg['configopts'] += ['', ''] - self.cfg['buildopts'] += ['', ''] - self.build_stages.append(AMD_LLVM) - self.build_stages.append(AMD_NEWLIB) - return super(EB_GCC, self).run_all_steps(*args, **kwargs) - - def sanity_check_step(self): - """ - Custom sanity check for GCC - """ - - os_type = get_os_type() - sharedlib_ext = get_shared_lib_ext() - - # determine "configuration name" directory, see https://sourceware.org/autobook/autobook/autobook_17.html - # this differs across GCC versions; - # x86_64-unknown-linux-gnu was common for old GCC versions, - # x86_64-pc-linux-gnu is more likely with an updated config.guess script; - # since this is internal to GCC, we don't really care how it is named exactly, - # we only care that it's actually there - - # we may get multiple hits (libexec/, lib/), which is fine, - # but we expect the same configuration name subdirectory in each of them - glob_pattern = os.path.join(self.installdir, 'lib*', 'gcc', '*-linux-gnu', self.version) - matches = glob.glob(glob_pattern) - if matches: - cands = nub([os.path.basename(os.path.dirname(x)) for x in matches]) - if len(cands) == 1: - config_name_subdir = cands[0] - else: - raise EasyBuildError("Found multiple candidates for configuration name: %s", ', '.join(cands)) - else: - raise EasyBuildError("Failed to determine configuration name: no matches for '%s'", glob_pattern) - - bin_files = ["gcov"] - lib_files = [] - if LooseVersion(self.version) >= LooseVersion('4.2'): - # libgomp was added in GCC 4.2.0 - ["libgomp.%s" % sharedlib_ext, "libgomp.a"] - if os_type == 'Linux': - lib_files.extend(["libgcc_s.%s" % sharedlib_ext]) - # libmudflap is replaced by asan (see release notes gcc 4.9.0) - if LooseVersion(self.version) < LooseVersion("4.9.0"): - lib_files.extend(["libmudflap.%s" % sharedlib_ext, "libmudflap.a"]) - else: - lib_files.extend(["libasan.%s" % sharedlib_ext, "libasan.a"]) - libexec_files = [] - dirs = [os.path.join('lib', 'gcc', config_name_subdir, self.version)] - - languages = self.cfg['languages'] or ['c', 'c++', 'fortran'] # default languages - - if 'c' in languages: - bin_files.extend(['cpp', 'gcc']) - libexec_files.extend(['cc1', 'collect2']) - - if 'c++' in languages: - bin_files.extend(['c++', 'g++']) - dirs.append('include/c++/%s' % self.version) - lib_files.extend(["libstdc++.%s" % sharedlib_ext, "libstdc++.a"]) - libexec_files.append('cc1plus') # c++ requires c, so collect2 not mentioned again - - if 'fortran' in languages: - bin_files.append('gfortran') - lib_files.extend(['libgfortran.%s' % sharedlib_ext, 'libgfortran.a']) - libexec_files.append('f951') - - if self.cfg['withlto']: - libexec_files.extend(['lto1', 'lto-wrapper']) - if os_type in ['Linux']: - libexec_files.append('liblto_plugin.%s' % sharedlib_ext) - - if self.cfg['withnvptx']: - bin_files.extend(['nvptx-none-as', 'nvptx-none-ld']) - lib_files.append('libgomp-plugin-nvptx.%s' % sharedlib_ext) - - if self.cfg['withamdgcn']: - lib_files.append('libgomp-plugin-gcn.%s' % sharedlib_ext) - - bin_files = ["bin/%s" % x for x in bin_files] - libdirs64 = ['lib64'] - libdirs32 = ['lib', 'lib32'] - libdirs = libdirs64 + libdirs32 - if self.cfg['multilib']: - # with multilib enabled, both lib and lib64 should be there - lib_files64 = [os.path.join(libdir, x) for libdir in libdirs64 for x in lib_files] - lib_files32 = [tuple([os.path.join(libdir, x) for libdir in libdirs32]) for x in lib_files] - lib_files = lib_files64 + lib_files32 - else: - # lib64 on SuSE and Darwin, lib otherwise - lib_files = [tuple([os.path.join(libdir, x) for libdir in libdirs]) for x in lib_files] - # lib on SuSE, libexec otherwise - libdirs = ['libexec', 'lib'] - common_infix = os.path.join('gcc', config_name_subdir, self.version) - libexec_files = [tuple([os.path.join(d, common_infix, x) for d in libdirs]) for x in libexec_files] - - old_cmds = [os.path.join('bin', x) for x in COMP_CMD_SYMLINKS.keys()] - - custom_paths = { - 'files': bin_files + lib_files + libexec_files + old_cmds, - 'dirs': dirs, - } - - custom_commands = [] - for lang, compiler in (('c', 'gcc'), ('c++', 'g++')): - if lang in languages: - # Simple test compile - cmd = 'echo "int main(){} " | %s -x %s -o/dev/null -' - compiler_path = os.path.join(self.installdir, 'bin', compiler) - custom_commands.append(cmd % (compiler_path, lang)) - if self.cfg['withlto']: - custom_commands.append(cmd % (compiler_path, lang + ' -flto -fuse-linker-plugin')) - if custom_commands: - # Load binutils to do the compile tests - extra_modules = [d['short_mod_name'] for d in self.cfg.dependencies() if d['name'] == 'binutils'] - else: - extra_modules = None - - super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, - extra_modules=extra_modules) - - def make_module_req_guess(self): - """ - GCC can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH - """ - guesses = super(EB_GCC, self).make_module_req_guess() - guesses.update({ - 'PATH': ['bin'], - 'CPATH': [], - 'LIBRARY_PATH': [], - 'LD_LIBRARY_PATH': ['lib', 'lib64'], - 'MANPATH': ['man', 'share/man'] - }) - return guesses diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 38c3166473..c25f7393ce 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -2,6 +2,6 @@ easyconfigs: # - GCC-9.3.0.eb - GCC-10.3.0.eb: options: - include-easyblocks: 'easyblocks/g/gcc.py' + include-easyblocks-from-pr: 2921 # - GCC-11.3.0.eb # - GCC-12.2.0.eb From 61a417479987a7a633d9a2a08e34634bf4e5f1ad Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 22 Apr 2023 00:39:29 +0200 Subject: [PATCH 096/122] add --robot to main eb command --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ecbe0c500d..ef6e2b805b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -210,7 +210,7 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out -${EB:-eb} --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} +${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 5b57ff740624f8a1a00335d4e329029fe3087891 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 23 Apr 2023 13:59:15 +0200 Subject: [PATCH 097/122] add OpenSSL-1.1 to NESSI/2023.04 --- eessi-2023.04.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index c25f7393ce..04cbc8d77d 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -5,3 +5,4 @@ easyconfigs: include-easyblocks-from-pr: 2921 # - GCC-11.3.0.eb # - GCC-12.2.0.eb + - OpenSSL-1.1.eb From 4a49ed70657eb403587d316c61bec4f255a03237 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 23 Apr 2023 14:55:58 +0200 Subject: [PATCH 098/122] add enhanced easyblock and use it via options in easystack file --- easyblocks/o/openssl_wrapper.py | 490 ++++++++++++++++++++++++++++++++ eessi-2023.04.yml | 4 +- 2 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 easyblocks/o/openssl_wrapper.py diff --git a/easyblocks/o/openssl_wrapper.py b/easyblocks/o/openssl_wrapper.py new file mode 100644 index 0000000000..837f6c491e --- /dev/null +++ b/easyblocks/o/openssl_wrapper.py @@ -0,0 +1,490 @@ +## +# Copyright 2021-2023 Vrije Universiteit Brussel +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing a wrapper module file for OpenSSL + +@author: Alex Domingo (Vrije Universiteit Brussel) +""" +import os +import re + +from distutils.version import LooseVersion +from urllib.parse import urlparse + +from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.filetools import change_dir, expand_glob_paths, mkdir, read_file, symlink, which, write_file +from easybuild.tools.py2vs3 import string_type +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import DARWIN, LINUX, get_os_type, get_shared_lib_ext, find_library_path + + +class EB_OpenSSL_wrapper(Bundle): + """ + Find path to installation files of OpenSSL in the host system. Checks in + order: library files defined in 'openssl_libs', engines libraries, header + files and executables. Any missing component will trigger an installation + from source of the fallback component. + Libraries are located by soname using the major and minor subversions of + the wrapper version. The full version of the wrapper or the option + 'minimum_openssl_version' determine the minimum required version of OpenSSL + in the system. The wrapper checks for version strings in the library files + and the opensslv.h header. + If OpenSSL in host systems fulfills the version requirements, wrap it by + symlinking all installation files. Otherwise fall back to the bundled + component. + """ + + @staticmethod + def extra_options(extra_vars=None): + """Easyconfig parameters specific to OpenSSL wrapper""" + extra_vars = Bundle.extra_options(extra_vars=extra_vars) + extra_vars.update({ + 'wrap_system_openssl': [True, 'Detect and wrap OpenSSL installation in host system', CUSTOM], + 'minimum_openssl_version': [None, 'Minimum version of OpenSSL required in host system', CUSTOM], + }) + return extra_vars + + def __init__(self, *args, **kwargs): + """Locate the installation files of OpenSSL in the host system""" + super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) + + # Wrapper should have at least a major minor version numbers + try: + subversions = self.version.split('.') + self.majmin_version = '%s.%s' % (subversions[0], subversions[1]) + except (AttributeError, IndexError): + err_msg = "Wrapper OpenSSL version does not have any subversion: %s" + raise EasyBuildError(err_msg, self.version) + + # Set minimum OpenSSL version + min_openssl_version = self.cfg.get('minimum_openssl_version') + + if not min_openssl_version: + min_openssl_version = self.version + elif not isinstance(min_openssl_version, string_type): + min_openssl_version = str(min_openssl_version) + + # Minimum OpenSSL version can only increase depth of wrapper version + if min_openssl_version.startswith(self.version): + self.log.debug("Requiring minimum OpenSSL version: %s", min_openssl_version) + else: + err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'" + raise EasyBuildError(err_msg, min_openssl_version, self.version) + + # Regex pattern to find version strings in OpenSSL libraries and headers + full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') + openssl_version_regex = re.compile(r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) + + # Libraries packaged in OpenSSL + openssl_libs = ['libssl', 'libcrypto'] + # list of relevant library extensions per system and version of OpenSSL + # the first item should be the extension of an installation from source, + # it will be used in the sanity checks of the component + openssl_libext = { + '1.0': { + LINUX: ('so.1.0.0', 'so.10'), + DARWIN: ('1.0.dylib', ), + }, + '1.1': { + LINUX: ('so.1.1', ), + DARWIN: ('1.1.dylib', ), + }, + '3.0': { + LINUX: ('so.3', ), + DARWIN: ('3.dylib', ), + }, + } + + os_type = get_os_type() + if self.majmin_version in openssl_libext and os_type in openssl_libext[self.majmin_version]: + # generate matrix of versioned .so filenames + system_versioned_libs = [ + ['%s.%s' % (lib, ext) for lib in openssl_libs] + for ext in openssl_libext[self.majmin_version][os_type] + ] + self.log.info("Matrix of version library names: %s", system_versioned_libs) + else: + err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s" + raise EasyBuildError(err_msg, self.majmin_version, os_type) + + # by default target the first option of each OpenSSL library, + # which corresponds to installation from source + self.target_ssl_libs = system_versioned_libs[0] + self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) + + # folders containing engines libraries + openssl_engines = { + '1.0': 'engines', + '1.1': 'engines-1.1', + '3.0': 'engines-3', + } + self.target_ssl_engine = openssl_engines[self.majmin_version] + + # Paths to system libraries and headers of OpenSSL + self.system_ssl = { + 'bin': None, + 'engines': None, + 'include': None, + 'libs': [], + } + + # early return when we're not wrapping the system OpenSSL installation + if not self.cfg.get('wrap_system_openssl'): + self.log.info("Not wrapping system OpenSSL installation by user request") + return + + # Check the system libraries of OpenSSL + # Find library file and compare its version string + for idx, solibs in enumerate(system_versioned_libs): + for solib in solibs: + system_solib = find_library_path(solib) + if system_solib: + openssl_version = '0' + # get version of system library filename + try: + openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) + except AttributeError: + # filename lacks the full version, fallback to version strings within the library + solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') + try: + openssl_version = openssl_version_regex.search(solib_strings).group(1) + except AttributeError: + dbg_msg = "Could not detect the full version of system OpenSSL library: %s" + self.log.debug(dbg_msg, system_solib) + # check that system version fulfills requirements + if LooseVersion(openssl_version) >= LooseVersion(min_openssl_version): + dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" + self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) + self.system_ssl['libs'].append(system_solib) + else: + dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" + self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) + else: + # one of the OpenSSL libraries is missing, switch to next group of versioned libs + self.system_ssl['libs'] = [] + break + + if len(self.system_ssl['libs']) == len(openssl_libs): + # keep these libraries as possible targets for this installation + target_system_ssl_libs = system_versioned_libs[idx] + break + + if len(self.system_ssl['libs']) == len(openssl_libs): + self.system_ssl['version'] = openssl_version + info_msg = "Found OpenSSL library version %s in host system: %s" + self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) + else: + self.log.info("OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild") + return + + # Directory with engine libraries + lib_dir = os.path.dirname(self.system_ssl['libs'][0]) + lib_engines_dir = [ + os.path.join(lib_dir, 'openssl', self.target_ssl_engine), + os.path.join(lib_dir, self.target_ssl_engine), + ] + + for engines_path in lib_engines_dir: + if os.path.isdir(engines_path): + self.system_ssl['engines'] = engines_path + self.log.debug("Found OpenSSL engines in: %s", self.system_ssl['engines']) + break + + if not self.system_ssl['engines']: + self.log.info("OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild") + return + + # Check system include paths for OpenSSL headers + cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null" + (out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False) + + sys_include_dirs = [] + for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE): + sys_include_dirs.extend(match.groups()) + self.log.debug("Found the following include directories in host system: %s", ', '.join(sys_include_dirs)) + + # headers are located in 'include/openssl' by default + ssl_include_subdirs = ['openssl'] + if self.majmin_version == '1.1': + # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 + # prefer 'include/openssl' as long as the version of headers matches + ssl_include_subdirs.append(os.path.join('openssl11', self.name.lower())) + + ssl_include_dirs = [os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs] + ssl_include_dirs = [include for include in ssl_include_dirs if os.path.isdir(include)] + + # find location of header files for this version of the OpenSSL libraries + for include_dir in ssl_include_dirs: + opensslv_path = os.path.join(include_dir, 'opensslv.h') + self.log.debug("Checking OpenSSL version in %s...", opensslv_path) + if os.path.exists(opensslv_path): + # check version reported by opensslv.h + opensslv = read_file(opensslv_path) + try: + header_version = openssl_version_regex.search(opensslv).group(1) + except AttributeError: + err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" + raise EasyBuildError(err_msg, opensslv_path) + + if header_version == self.system_ssl['version']: + self.system_ssl['include'] = include_dir + info_msg = "Found OpenSSL headers v%s in host system: %s" + self.log.info(info_msg, header_version, self.system_ssl['include']) + break + else: + dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'" + self.log.debug(dbg_msg, header_version, self.system_ssl['version']) + else: + self.log.info("System OpenSSL header file %s not found", opensslv_path) + + if not self.system_ssl['include']: + err_msg = ("OpenSSL v%s headers not found in host system, but libraries for v%s are present. " + "Install the development package of OpenSSL for your system or force building OpenSSL from " + "source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig.") + raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) + + # Check system OpenSSL binary + if self.majmin_version == '1.1': + # prefer 'openssl11' over 'openssl' with v1.1 + self.system_ssl['bin'] = which('openssl11') + + if not self.system_ssl['bin']: + self.system_ssl['bin'] = which('openssl') + + if self.system_ssl['bin']: + self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) + else: + self.log.info("System OpenSSL binary not found!") + return + + # system OpenSSL is fine, change target libraries to the ones found in it + self.target_ssl_libs = target_system_ssl_libs + self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) + + def fetch_step(self, *args, **kwargs): + """Fetch sources if OpenSSL component is needed""" + if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): + super(EB_OpenSSL_wrapper, self).fetch_step(*args, **kwargs) + + def extract_step(self): + """Extract sources if OpenSSL component is needed""" + if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): + super(EB_OpenSSL_wrapper, self).extract_step() + + def install_step(self): + """Symlink target OpenSSL installation""" + if all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): + # note: symlink to individual files, not directories, + # since directory symlinks get resolved easily... + + # link OpenSSL libraries in system + lib64_dir = os.path.join(self.installdir, 'lib64') + lib64_engines_dir = os.path.join(lib64_dir, os.path.basename(self.system_ssl['engines'])) + mkdir(lib64_engines_dir, parents=True) + + # link existing known libraries + for libso in self.system_ssl['libs']: + symlink(libso, os.path.join(lib64_dir, os.path.basename(libso))) + + # link engines library files + engine_lib_pattern = [os.path.join(self.system_ssl['engines'], '*')] + for engine_lib in expand_glob_paths(engine_lib_pattern): + symlink(engine_lib, os.path.join(lib64_engines_dir, os.path.basename(engine_lib))) + + # relative symlink for unversioned libraries + cwd = change_dir(lib64_dir) + for libso in self.system_ssl['libs']: + libso = os.path.basename(libso) + unversioned_lib = '%s.%s' % (libso.split('.')[0], get_shared_lib_ext()) + symlink(libso, unversioned_lib, use_abspath_source=False) + change_dir(cwd) + + # link OpenSSL headers in system + include_dir = os.path.join(self.installdir, 'include', self.name.lower()) + mkdir(include_dir, parents=True) + include_pattern = [os.path.join(self.system_ssl['include'], '*')] + for header_file in expand_glob_paths(include_pattern): + symlink(header_file, os.path.join(include_dir, os.path.basename(header_file))) + + # link OpenSSL binary in system + bin_dir = os.path.join(self.installdir, 'bin') + mkdir(bin_dir) + symlink(self.system_ssl['bin'], os.path.join(bin_dir, self.name.lower())) + + # install pkg-config files + self.install_pc_files() + + elif self.cfg.get('wrap_system_openssl'): + # install OpenSSL component due to lack of OpenSSL in host system + print_warning("Not all OpenSSL components found in host system, falling back to OpenSSL in EasyBuild!") + super(EB_OpenSSL_wrapper, self).install_step() + else: + # install OpenSSL component by user request + warn_msg = "Installing OpenSSL from source in EasyBuild by user request ('wrap_system_openssl=%s')" + print_warning(warn_msg, self.cfg.get('wrap_system_openssl')) + super(EB_OpenSSL_wrapper, self).install_step() + + def sanity_check_step(self): + """Custom sanity check for OpenSSL wrapper.""" + shlib_ext = get_shared_lib_ext() + + ssl_libs = ['%s.%s' % (libso.split('.')[0], shlib_ext) for libso in self.target_ssl_libs] + ssl_libs.extend(self.target_ssl_libs) + + ssl_files = [os.path.join('bin', self.name.lower())] + ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) + + ssl_dirs = [ + os.path.join('include', self.name.lower()), + os.path.join('lib', self.target_ssl_engine), + os.path.join('lib', 'pkgconfig'), + ] + + custom_paths = { + 'files': ssl_files, + 'dirs': ssl_dirs, + } + + # use proxy to connect if https_proxy environment variable is defined + proxy_arg = '' + if os.environ.get('https_proxy'): + # only use host & port from https_proxy env var, that is, strip + # any protocol prefix and trailing slashes + proxy_parsed = urlparse(os.environ.get('https_proxy')) + if proxy_parsed.netloc: + proxy_arg = ' -proxy %(proxy_parsed.netloc)s' + + custom_commands = [ + # make sure that version mentioned in output of 'openssl version' matches version we are using + "ssl_ver=$(openssl version); [ ${ssl_ver:8:3} == '%s' ]" % self.majmin_version, + "echo | openssl s_client%(proxy_arg)s -connect github.com:443 -verify 9 | grep 'Verify return code: 0 (ok)'", + ] + + super(Bundle, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def install_pc_files(self): + """Install pkg-config files for the wrapper""" + + PC_TEMPLATE = """prefix=%(root)s +exec_prefix=${prefix} +libdir=${exec_prefix}/lib64 +includedir=${prefix}/include +%(enginesdir)s + +Name: %(name)s +Description: %(description)s +Version: %(version)s +%(requires)s +%(libs)s +%(cflags)s +""" + openssl_components = { + 'libcrypto': { + 'name': 'OpenSSL-libcrypto', + 'description': 'OpenSSL cryptography library', + 'enginesdir': self.target_ssl_engine, + }, + 'libssl': { + 'name': 'OpenSSL-libssl', + 'description': 'Secure Sockets Layer and cryptography libraries', + 'enginesdir': None, + }, + 'openssl': { + 'name': 'OpenSSL', + 'description': 'Secure Sockets Layer and cryptography libraries and tools', + 'enginesdir': None, + }, + } + + pc_install_dir = os.path.join(self.installdir, 'lib64', 'pkgconfig') + mkdir(pc_install_dir) + + for pc_comp in openssl_components: + pc_file = openssl_components[pc_comp] + + pc_file['root'] = self.installdir + pc_file['version'] = self.system_ssl['version'] + + # component name in system pkg-config + pc_name = pc_comp + if self.majmin_version == '1.1': + # check suffixed names with v1.1 + pc_name_suffix = pc_name + '11' + pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix + if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False): + self.log.info("%s exists", pc_name_suffix) + pc_name = pc_name_suffix + + # get requires from pkg-config + pc_file['requires'] = [] + for require_type in ['Requires', 'Requires.private']: + require_print = require_type.lower().replace('.', '-') + pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name) + out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_print_cmd, out) + + if out: + requires = out + # use unsuffixed names for components provided by this wrapper + for wrap_comp in openssl_components: + requires = re.sub(r'^%s[0-9]+$' % wrap_comp, wrap_comp, requires, flags=re.M) + # format requires + requires = requires.rstrip().splitlines() + pc_file['requires'].append("%s: %s" % (require_type, ' '.join(requires))) + + pc_file['requires'] = '\n'.join(pc_file['requires']) + + if pc_comp.startswith('lib'): + # add libs and cflags for library components + c_lib_name = pc_comp[3:] + pc_file['libs'] = "Libs: -L${libdir} -l%s" % c_lib_name + pc_file['cflags'] = "Cflags: -I${includedir}" + # infer private libs through pkg-config + pc_libs_cmd = "pkg-config --libs %s" % pc_name + out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_libs_cmd, out) + linker_libs = out + + pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name + out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_libs_static_cmd, out) + + libs_priv = "%s " % out.rstrip() + for flag in linker_libs.rstrip().split(' '): + libs_priv = libs_priv.replace("%s " % flag, '') + pc_file['libs'] += "\nLibs.private: %s" % libs_priv + else: + pc_file['libs'], pc_file['cflags'] = '', '' + + # format enginesdir + if pc_file['enginesdir']: + pc_file['enginesdir'] = 'enginesdir=${libdir}/%s' % pc_file['enginesdir'] + else: + pc_file['enginesdir'] = '' + + pc_path = os.path.join(pc_install_dir, '%s.pc' % pc_comp) + write_file(pc_path, PC_TEMPLATE % pc_file) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 04cbc8d77d..dcb2b63c5c 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -5,4 +5,6 @@ easyconfigs: include-easyblocks-from-pr: 2921 # - GCC-11.3.0.eb # - GCC-12.2.0.eb - - OpenSSL-1.1.eb + - OpenSSL-1.1.eb: + options: + include-easyblocks: 'easyblocks/o/openssl_wrapper.py' From fe78426c04bddf4c6b06054903df971e3ddc82d2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 23 Apr 2023 15:33:45 +0200 Subject: [PATCH 099/122] fix string formatting error --- easyblocks/o/openssl_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easyblocks/o/openssl_wrapper.py b/easyblocks/o/openssl_wrapper.py index 837f6c491e..7e61b815b6 100644 --- a/easyblocks/o/openssl_wrapper.py +++ b/easyblocks/o/openssl_wrapper.py @@ -376,12 +376,12 @@ def sanity_check_step(self): # any protocol prefix and trailing slashes proxy_parsed = urlparse(os.environ.get('https_proxy')) if proxy_parsed.netloc: - proxy_arg = ' -proxy %(proxy_parsed.netloc)s' + proxy_arg = f' -proxy {proxy_parsed.netloc}' custom_commands = [ # make sure that version mentioned in output of 'openssl version' matches version we are using "ssl_ver=$(openssl version); [ ${ssl_ver:8:3} == '%s' ]" % self.majmin_version, - "echo | openssl s_client%(proxy_arg)s -connect github.com:443 -verify 9 | grep 'Verify return code: 0 (ok)'", + "echo | openssl s_client%s -connect github.com:443 -verify 9 | grep 'Verify return code: 0 (ok)'" % proxy_arg, ] super(Bundle, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) From 9fb6a4c1f110802f8918e09fb3e013b9fff22469 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 23 Apr 2023 16:29:19 +0200 Subject: [PATCH 100/122] removed openssl_wrapper.py + using easyblock PR 2922 instead --- easyblocks/o/openssl_wrapper.py | 490 -------------------------------- eessi-2023.04.yml | 2 +- 2 files changed, 1 insertion(+), 491 deletions(-) delete mode 100644 easyblocks/o/openssl_wrapper.py diff --git a/easyblocks/o/openssl_wrapper.py b/easyblocks/o/openssl_wrapper.py deleted file mode 100644 index 7e61b815b6..0000000000 --- a/easyblocks/o/openssl_wrapper.py +++ /dev/null @@ -1,490 +0,0 @@ -## -# Copyright 2021-2023 Vrije Universiteit Brussel -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for installing a wrapper module file for OpenSSL - -@author: Alex Domingo (Vrije Universiteit Brussel) -""" -import os -import re - -from distutils.version import LooseVersion -from urllib.parse import urlparse - -from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.filetools import change_dir, expand_glob_paths, mkdir, read_file, symlink, which, write_file -from easybuild.tools.py2vs3 import string_type -from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import DARWIN, LINUX, get_os_type, get_shared_lib_ext, find_library_path - - -class EB_OpenSSL_wrapper(Bundle): - """ - Find path to installation files of OpenSSL in the host system. Checks in - order: library files defined in 'openssl_libs', engines libraries, header - files and executables. Any missing component will trigger an installation - from source of the fallback component. - Libraries are located by soname using the major and minor subversions of - the wrapper version. The full version of the wrapper or the option - 'minimum_openssl_version' determine the minimum required version of OpenSSL - in the system. The wrapper checks for version strings in the library files - and the opensslv.h header. - If OpenSSL in host systems fulfills the version requirements, wrap it by - symlinking all installation files. Otherwise fall back to the bundled - component. - """ - - @staticmethod - def extra_options(extra_vars=None): - """Easyconfig parameters specific to OpenSSL wrapper""" - extra_vars = Bundle.extra_options(extra_vars=extra_vars) - extra_vars.update({ - 'wrap_system_openssl': [True, 'Detect and wrap OpenSSL installation in host system', CUSTOM], - 'minimum_openssl_version': [None, 'Minimum version of OpenSSL required in host system', CUSTOM], - }) - return extra_vars - - def __init__(self, *args, **kwargs): - """Locate the installation files of OpenSSL in the host system""" - super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) - - # Wrapper should have at least a major minor version numbers - try: - subversions = self.version.split('.') - self.majmin_version = '%s.%s' % (subversions[0], subversions[1]) - except (AttributeError, IndexError): - err_msg = "Wrapper OpenSSL version does not have any subversion: %s" - raise EasyBuildError(err_msg, self.version) - - # Set minimum OpenSSL version - min_openssl_version = self.cfg.get('minimum_openssl_version') - - if not min_openssl_version: - min_openssl_version = self.version - elif not isinstance(min_openssl_version, string_type): - min_openssl_version = str(min_openssl_version) - - # Minimum OpenSSL version can only increase depth of wrapper version - if min_openssl_version.startswith(self.version): - self.log.debug("Requiring minimum OpenSSL version: %s", min_openssl_version) - else: - err_msg = "Requested minimum OpenSSL version '%s' does not fit in wrapper easyconfig version '%s'" - raise EasyBuildError(err_msg, min_openssl_version, self.version) - - # Regex pattern to find version strings in OpenSSL libraries and headers - full_version_regex = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+[a-z]?') - openssl_version_regex = re.compile(r'OpenSSL\s+([0-9]+\.[0-9]+(\.[0-9]+[a-z]?)*)', re.M) - - # Libraries packaged in OpenSSL - openssl_libs = ['libssl', 'libcrypto'] - # list of relevant library extensions per system and version of OpenSSL - # the first item should be the extension of an installation from source, - # it will be used in the sanity checks of the component - openssl_libext = { - '1.0': { - LINUX: ('so.1.0.0', 'so.10'), - DARWIN: ('1.0.dylib', ), - }, - '1.1': { - LINUX: ('so.1.1', ), - DARWIN: ('1.1.dylib', ), - }, - '3.0': { - LINUX: ('so.3', ), - DARWIN: ('3.dylib', ), - }, - } - - os_type = get_os_type() - if self.majmin_version in openssl_libext and os_type in openssl_libext[self.majmin_version]: - # generate matrix of versioned .so filenames - system_versioned_libs = [ - ['%s.%s' % (lib, ext) for lib in openssl_libs] - for ext in openssl_libext[self.majmin_version][os_type] - ] - self.log.info("Matrix of version library names: %s", system_versioned_libs) - else: - err_msg = "Don't know name of OpenSSL system library for version %s and OS type %s" - raise EasyBuildError(err_msg, self.majmin_version, os_type) - - # by default target the first option of each OpenSSL library, - # which corresponds to installation from source - self.target_ssl_libs = system_versioned_libs[0] - self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) - - # folders containing engines libraries - openssl_engines = { - '1.0': 'engines', - '1.1': 'engines-1.1', - '3.0': 'engines-3', - } - self.target_ssl_engine = openssl_engines[self.majmin_version] - - # Paths to system libraries and headers of OpenSSL - self.system_ssl = { - 'bin': None, - 'engines': None, - 'include': None, - 'libs': [], - } - - # early return when we're not wrapping the system OpenSSL installation - if not self.cfg.get('wrap_system_openssl'): - self.log.info("Not wrapping system OpenSSL installation by user request") - return - - # Check the system libraries of OpenSSL - # Find library file and compare its version string - for idx, solibs in enumerate(system_versioned_libs): - for solib in solibs: - system_solib = find_library_path(solib) - if system_solib: - openssl_version = '0' - # get version of system library filename - try: - openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0) - except AttributeError: - # filename lacks the full version, fallback to version strings within the library - solib_strings = read_file(system_solib, mode="rb").decode('utf-8', 'replace') - try: - openssl_version = openssl_version_regex.search(solib_strings).group(1) - except AttributeError: - dbg_msg = "Could not detect the full version of system OpenSSL library: %s" - self.log.debug(dbg_msg, system_solib) - # check that system version fulfills requirements - if LooseVersion(openssl_version) >= LooseVersion(min_openssl_version): - dbg_msg = "System OpenSSL library '%s' with version %s fulfills requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) - self.system_ssl['libs'].append(system_solib) - else: - dbg_msg = "System OpenSSL library '%s' with version %s is older than requested version %s" - self.log.debug(dbg_msg, system_solib, openssl_version, min_openssl_version) - else: - # one of the OpenSSL libraries is missing, switch to next group of versioned libs - self.system_ssl['libs'] = [] - break - - if len(self.system_ssl['libs']) == len(openssl_libs): - # keep these libraries as possible targets for this installation - target_system_ssl_libs = system_versioned_libs[idx] - break - - if len(self.system_ssl['libs']) == len(openssl_libs): - self.system_ssl['version'] = openssl_version - info_msg = "Found OpenSSL library version %s in host system: %s" - self.log.info(info_msg, self.system_ssl['version'], os.path.dirname(self.system_ssl['libs'][0])) - else: - self.log.info("OpenSSL library not found in host system, falling back to OpenSSL in EasyBuild") - return - - # Directory with engine libraries - lib_dir = os.path.dirname(self.system_ssl['libs'][0]) - lib_engines_dir = [ - os.path.join(lib_dir, 'openssl', self.target_ssl_engine), - os.path.join(lib_dir, self.target_ssl_engine), - ] - - for engines_path in lib_engines_dir: - if os.path.isdir(engines_path): - self.system_ssl['engines'] = engines_path - self.log.debug("Found OpenSSL engines in: %s", self.system_ssl['engines']) - break - - if not self.system_ssl['engines']: - self.log.info("OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild") - return - - # Check system include paths for OpenSSL headers - cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null" - (out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False) - - sys_include_dirs = [] - for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE): - sys_include_dirs.extend(match.groups()) - self.log.debug("Found the following include directories in host system: %s", ', '.join(sys_include_dirs)) - - # headers are located in 'include/openssl' by default - ssl_include_subdirs = ['openssl'] - if self.majmin_version == '1.1': - # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 - # prefer 'include/openssl' as long as the version of headers matches - ssl_include_subdirs.append(os.path.join('openssl11', self.name.lower())) - - ssl_include_dirs = [os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs] - ssl_include_dirs = [include for include in ssl_include_dirs if os.path.isdir(include)] - - # find location of header files for this version of the OpenSSL libraries - for include_dir in ssl_include_dirs: - opensslv_path = os.path.join(include_dir, 'opensslv.h') - self.log.debug("Checking OpenSSL version in %s...", opensslv_path) - if os.path.exists(opensslv_path): - # check version reported by opensslv.h - opensslv = read_file(opensslv_path) - try: - header_version = openssl_version_regex.search(opensslv).group(1) - except AttributeError: - err_msg = "System OpenSSL header '%s' does not contain any recognizable version string" - raise EasyBuildError(err_msg, opensslv_path) - - if header_version == self.system_ssl['version']: - self.system_ssl['include'] = include_dir - info_msg = "Found OpenSSL headers v%s in host system: %s" - self.log.info(info_msg, header_version, self.system_ssl['include']) - break - else: - dbg_msg = "System OpenSSL header version '%s' doesn not match library version '%s'" - self.log.debug(dbg_msg, header_version, self.system_ssl['version']) - else: - self.log.info("System OpenSSL header file %s not found", opensslv_path) - - if not self.system_ssl['include']: - err_msg = ("OpenSSL v%s headers not found in host system, but libraries for v%s are present. " - "Install the development package of OpenSSL for your system or force building OpenSSL from " - "source in EasyBuild by setting 'wrap_system_openssl = False' in the OpenSSL easyconfig.") - raise EasyBuildError(err_msg, self.version, self.system_ssl['version']) - - # Check system OpenSSL binary - if self.majmin_version == '1.1': - # prefer 'openssl11' over 'openssl' with v1.1 - self.system_ssl['bin'] = which('openssl11') - - if not self.system_ssl['bin']: - self.system_ssl['bin'] = which('openssl') - - if self.system_ssl['bin']: - self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) - else: - self.log.info("System OpenSSL binary not found!") - return - - # system OpenSSL is fine, change target libraries to the ones found in it - self.target_ssl_libs = target_system_ssl_libs - self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) - - def fetch_step(self, *args, **kwargs): - """Fetch sources if OpenSSL component is needed""" - if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): - super(EB_OpenSSL_wrapper, self).fetch_step(*args, **kwargs) - - def extract_step(self): - """Extract sources if OpenSSL component is needed""" - if not all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): - super(EB_OpenSSL_wrapper, self).extract_step() - - def install_step(self): - """Symlink target OpenSSL installation""" - if all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')): - # note: symlink to individual files, not directories, - # since directory symlinks get resolved easily... - - # link OpenSSL libraries in system - lib64_dir = os.path.join(self.installdir, 'lib64') - lib64_engines_dir = os.path.join(lib64_dir, os.path.basename(self.system_ssl['engines'])) - mkdir(lib64_engines_dir, parents=True) - - # link existing known libraries - for libso in self.system_ssl['libs']: - symlink(libso, os.path.join(lib64_dir, os.path.basename(libso))) - - # link engines library files - engine_lib_pattern = [os.path.join(self.system_ssl['engines'], '*')] - for engine_lib in expand_glob_paths(engine_lib_pattern): - symlink(engine_lib, os.path.join(lib64_engines_dir, os.path.basename(engine_lib))) - - # relative symlink for unversioned libraries - cwd = change_dir(lib64_dir) - for libso in self.system_ssl['libs']: - libso = os.path.basename(libso) - unversioned_lib = '%s.%s' % (libso.split('.')[0], get_shared_lib_ext()) - symlink(libso, unversioned_lib, use_abspath_source=False) - change_dir(cwd) - - # link OpenSSL headers in system - include_dir = os.path.join(self.installdir, 'include', self.name.lower()) - mkdir(include_dir, parents=True) - include_pattern = [os.path.join(self.system_ssl['include'], '*')] - for header_file in expand_glob_paths(include_pattern): - symlink(header_file, os.path.join(include_dir, os.path.basename(header_file))) - - # link OpenSSL binary in system - bin_dir = os.path.join(self.installdir, 'bin') - mkdir(bin_dir) - symlink(self.system_ssl['bin'], os.path.join(bin_dir, self.name.lower())) - - # install pkg-config files - self.install_pc_files() - - elif self.cfg.get('wrap_system_openssl'): - # install OpenSSL component due to lack of OpenSSL in host system - print_warning("Not all OpenSSL components found in host system, falling back to OpenSSL in EasyBuild!") - super(EB_OpenSSL_wrapper, self).install_step() - else: - # install OpenSSL component by user request - warn_msg = "Installing OpenSSL from source in EasyBuild by user request ('wrap_system_openssl=%s')" - print_warning(warn_msg, self.cfg.get('wrap_system_openssl')) - super(EB_OpenSSL_wrapper, self).install_step() - - def sanity_check_step(self): - """Custom sanity check for OpenSSL wrapper.""" - shlib_ext = get_shared_lib_ext() - - ssl_libs = ['%s.%s' % (libso.split('.')[0], shlib_ext) for libso in self.target_ssl_libs] - ssl_libs.extend(self.target_ssl_libs) - - ssl_files = [os.path.join('bin', self.name.lower())] - ssl_files.extend(os.path.join('lib', libso) for libso in ssl_libs) - - ssl_dirs = [ - os.path.join('include', self.name.lower()), - os.path.join('lib', self.target_ssl_engine), - os.path.join('lib', 'pkgconfig'), - ] - - custom_paths = { - 'files': ssl_files, - 'dirs': ssl_dirs, - } - - # use proxy to connect if https_proxy environment variable is defined - proxy_arg = '' - if os.environ.get('https_proxy'): - # only use host & port from https_proxy env var, that is, strip - # any protocol prefix and trailing slashes - proxy_parsed = urlparse(os.environ.get('https_proxy')) - if proxy_parsed.netloc: - proxy_arg = f' -proxy {proxy_parsed.netloc}' - - custom_commands = [ - # make sure that version mentioned in output of 'openssl version' matches version we are using - "ssl_ver=$(openssl version); [ ${ssl_ver:8:3} == '%s' ]" % self.majmin_version, - "echo | openssl s_client%s -connect github.com:443 -verify 9 | grep 'Verify return code: 0 (ok)'" % proxy_arg, - ] - - super(Bundle, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def install_pc_files(self): - """Install pkg-config files for the wrapper""" - - PC_TEMPLATE = """prefix=%(root)s -exec_prefix=${prefix} -libdir=${exec_prefix}/lib64 -includedir=${prefix}/include -%(enginesdir)s - -Name: %(name)s -Description: %(description)s -Version: %(version)s -%(requires)s -%(libs)s -%(cflags)s -""" - openssl_components = { - 'libcrypto': { - 'name': 'OpenSSL-libcrypto', - 'description': 'OpenSSL cryptography library', - 'enginesdir': self.target_ssl_engine, - }, - 'libssl': { - 'name': 'OpenSSL-libssl', - 'description': 'Secure Sockets Layer and cryptography libraries', - 'enginesdir': None, - }, - 'openssl': { - 'name': 'OpenSSL', - 'description': 'Secure Sockets Layer and cryptography libraries and tools', - 'enginesdir': None, - }, - } - - pc_install_dir = os.path.join(self.installdir, 'lib64', 'pkgconfig') - mkdir(pc_install_dir) - - for pc_comp in openssl_components: - pc_file = openssl_components[pc_comp] - - pc_file['root'] = self.installdir - pc_file['version'] = self.system_ssl['version'] - - # component name in system pkg-config - pc_name = pc_comp - if self.majmin_version == '1.1': - # check suffixed names with v1.1 - pc_name_suffix = pc_name + '11' - pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix - if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False): - self.log.info("%s exists", pc_name_suffix) - pc_name = pc_name_suffix - - # get requires from pkg-config - pc_file['requires'] = [] - for require_type in ['Requires', 'Requires.private']: - require_print = require_type.lower().replace('.', '-') - pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name) - out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False) - self.log.info("Output of '%s': %s", pc_print_cmd, out) - - if out: - requires = out - # use unsuffixed names for components provided by this wrapper - for wrap_comp in openssl_components: - requires = re.sub(r'^%s[0-9]+$' % wrap_comp, wrap_comp, requires, flags=re.M) - # format requires - requires = requires.rstrip().splitlines() - pc_file['requires'].append("%s: %s" % (require_type, ' '.join(requires))) - - pc_file['requires'] = '\n'.join(pc_file['requires']) - - if pc_comp.startswith('lib'): - # add libs and cflags for library components - c_lib_name = pc_comp[3:] - pc_file['libs'] = "Libs: -L${libdir} -l%s" % c_lib_name - pc_file['cflags'] = "Cflags: -I${includedir}" - # infer private libs through pkg-config - pc_libs_cmd = "pkg-config --libs %s" % pc_name - out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False) - self.log.info("Output of '%s': %s", pc_libs_cmd, out) - linker_libs = out - - pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name - out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False) - self.log.info("Output of '%s': %s", pc_libs_static_cmd, out) - - libs_priv = "%s " % out.rstrip() - for flag in linker_libs.rstrip().split(' '): - libs_priv = libs_priv.replace("%s " % flag, '') - pc_file['libs'] += "\nLibs.private: %s" % libs_priv - else: - pc_file['libs'], pc_file['cflags'] = '', '' - - # format enginesdir - if pc_file['enginesdir']: - pc_file['enginesdir'] = 'enginesdir=${libdir}/%s' % pc_file['enginesdir'] - else: - pc_file['enginesdir'] = '' - - pc_path = os.path.join(pc_install_dir, '%s.pc' % pc_comp) - write_file(pc_path, PC_TEMPLATE % pc_file) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index dcb2b63c5c..99cae7473b 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -7,4 +7,4 @@ easyconfigs: # - GCC-12.2.0.eb - OpenSSL-1.1.eb: options: - include-easyblocks: 'easyblocks/o/openssl_wrapper.py' + include-easyblocks-from-pr: 2922 From 9772fb7774edd1d3839262c090f282453651bb61 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 24 Apr 2023 12:04:46 +0200 Subject: [PATCH 101/122] check if eb ran successful and redirect stderr --- check_missing_installations.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 5e76e23c30..945bf32b19 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -3,6 +3,7 @@ # Script to check for missing installations in EESSI pilot software stack (version 2021.12) # # author: Kenneth Hoste (@boegel) +# author: Thomas Roeblitz (@trz42) # # license: GPLv2 # @@ -21,16 +22,19 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb -# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out} # PR 16531 not needed since we use EB v4.7.0 # this, however, breaks the GHA https://github.com/NorESSI/software-layer/blob/main/.github/workflows/test_eessi.yml # because it uses the EESSI pilot which only provides EB 4.5.1, so adding it back -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out} +exit_code=${PIPESTATUS[0]} + +ok_msg="Command 'eb --missing ...' succeeded, analysing output..." +fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as @@ -41,8 +45,11 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim # output does not contain any line with ` required modules missing:` grep " required modules missing:" ${eb_missing_out} > /dev/null +exit_code=$? # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 -[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" From 0ba64f1901e2ceefbcd4a4c9804e8dd88f47d066 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 25 Apr 2023 07:01:52 +0000 Subject: [PATCH 102/122] Adding CMake, Perl and OpenMPI with GCC/10.30.0 to NESSI/2023.04 --- eessi-2023.04.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 99cae7473b..04db10ba7a 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -8,3 +8,8 @@ easyconfigs: - OpenSSL-1.1.eb: options: include-easyblocks-from-pr: 2922 + - CMake-3.20.1-GCCcore-10.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - Perl-5.32.1-GCCcore-10.3.0.eb + - OpenMPI-4.1.1-GCC-10.3.0.eb From c4f8cf408e975b541e8af3c947d37262d6b5b924 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 3 May 2023 12:00:40 +0000 Subject: [PATCH 103/122] Adding parse_hook to replace Rust/1.52.1 by Rust/1.60.0 --- eb_hooks.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index df7742f999..4413f7c1ba 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -181,6 +181,29 @@ def wrf_preconfigure(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def Rustv(ec, eprefix): + """ For the new compat layer we use Rust-1.60.0 Rust-1.52.1""" + print_msg(f"#normal_deps {len(ec['dependencies'])}" ) + for dep in ec['dependencies']: + for i in range(len(ec['dependencies'])): + dep = ec['dependencies'][i] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE: Rust version has been modified to 1.60.0") + ec['dependencies'][i] = ["Rust", "1.60.0"] + for dep in ec['hiddendependencies']: + for i in range(len(ec['hiddendependencies'])): + dep = ec['hiddendependencies'][i] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE: Rust version has been modified to 1.60.0") + ec['hiddendependencies'][i] = ["Rust", "1.60.0"] + for i in range(len(ec['builddependencies'])): + for dep in ec['builddependencies']: + dep = ec['builddependencies'][i] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE: Rust version has been modified to 1.60.0") + ec['builddependencies'][i] = ["Rust", "1.60.0"] + + PARSE_HOOKS = { 'CGAL': cgal_toolchainopts_precise, 'fontconfig': fontconfig_add_fonts, From fef52e4b756e84a4eda9862f971931154e0fbdda Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 3 May 2023 12:07:13 +0000 Subject: [PATCH 104/122] Adding explanation in text mode --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4413f7c1ba..36f2b1832d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -182,7 +182,7 @@ def wrf_preconfigure(self, *args, **kwargs): def Rustv(ec, eprefix): - """ For the new compat layer we use Rust-1.60.0 Rust-1.52.1""" + """ For the new compat layer we use Rust-1.60.0 instead of Rust-1.52.1""" print_msg(f"#normal_deps {len(ec['dependencies'])}" ) for dep in ec['dependencies']: for i in range(len(ec['dependencies'])): From aefdafda77864418ffa23de468fa045642ef8fa0 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 3 May 2023 12:16:15 +0000 Subject: [PATCH 105/122] Modyfing script/removing additional comments --- eb_hooks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 36f2b1832d..77b545b78b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -183,7 +183,6 @@ def wrf_preconfigure(self, *args, **kwargs): def Rustv(ec, eprefix): """ For the new compat layer we use Rust-1.60.0 instead of Rust-1.52.1""" - print_msg(f"#normal_deps {len(ec['dependencies'])}" ) for dep in ec['dependencies']: for i in range(len(ec['dependencies'])): dep = ec['dependencies'][i] @@ -196,9 +195,9 @@ def Rustv(ec, eprefix): if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): print_msg("NOTE: Rust version has been modified to 1.60.0") ec['hiddendependencies'][i] = ["Rust", "1.60.0"] - for i in range(len(ec['builddependencies'])): - for dep in ec['builddependencies']: - dep = ec['builddependencies'][i] + for dep in ec['builddependencies']: + for i in range(len(ec['builddependencies'])): + dep = ec['builddependencies'][i] if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): print_msg("NOTE: Rust version has been modified to 1.60.0") ec['builddependencies'][i] = ["Rust", "1.60.0"] From bffbd3d2bb97b4c84e5588cdd774bb55bd7f0d9b Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 5 May 2023 09:41:40 +0000 Subject: [PATCH 106/122] modifying and enhancing the script after review by trz42 --- eb_hooks.py | 53 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 77b545b78b..7f3f53a6c0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -50,6 +50,10 @@ def parse_hook(ec, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') + + # always replace Rust/1.52.1 with Rust/1.60.0 + Rust_ver_replace(ec, eprefix) + if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) @@ -181,27 +185,34 @@ def wrf_preconfigure(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def Rustv(ec, eprefix): - """ For the new compat layer we use Rust-1.60.0 instead of Rust-1.52.1""" - for dep in ec['dependencies']: - for i in range(len(ec['dependencies'])): - dep = ec['dependencies'][i] - if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): - print_msg("NOTE: Rust version has been modified to 1.60.0") - ec['dependencies'][i] = ["Rust", "1.60.0"] - for dep in ec['hiddendependencies']: - for i in range(len(ec['hiddendependencies'])): - dep = ec['hiddendependencies'][i] - if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): - print_msg("NOTE: Rust version has been modified to 1.60.0") - ec['hiddendependencies'][i] = ["Rust", "1.60.0"] - for dep in ec['builddependencies']: - for i in range(len(ec['builddependencies'])): - dep = ec['builddependencies'][i] - if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): - print_msg("NOTE: Rust version has been modified to 1.60.0") - ec['builddependencies'][i] = ["Rust", "1.60.0"] - +def Rust_ver_replace(ec, eprefix): + """When using the new compat layer, building Rust/1.52.1 fails while Rust/1.60.0 succeeds ,the goal is to replace + Rust/1.52.1 when found as dependency/hiddendependency/buildependency by Rust/1.60.0 while building software""" + for index in range(len(ec['dependencies'])): + dep = ec['dependencies'][index] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE:Rust dependency version has been modified from Rust/1.52.1 --> Rust/1.60.0") + if isinstance(dep, list): + ec['dependencies'][index] = ["Rust", "1.60.0"] + else: + ec['dependencies'][index] = ("Rust", "1.60.0") + + for index in range(len(ec['hiddendependencies'])): + dep = ec['hiddendependencies'][index] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE:Rust hiddendependency version has been modified from Rust/1.52.1 --> Rust/1.60.0 ") + if isinstance(dep, list): + ec['hiddendependencies'][index] = ["Rust", "1.60.0"] + else: + ec['hiddendependencies'][index] = ("Rust", "1.60.0") + for index in range(len(ec['builddependencies'])): + dep = ec['builddependencies'][index] + if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'): + print_msg("NOTE:Rust builddependency version has been modified from Rust/1.52.1 --> Rust/1.60.0") + if isinstance(dep, list): + ec['builddependencies'][index] = ["Rust", "1.60.0"] + else: + ec['builddependencies'][index] = ("Rust", "1.60.0") PARSE_HOOKS = { 'CGAL': cgal_toolchainopts_precise, From b2ce883bac1db49c65ee75adcb2fdf1ec43463f9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 10 May 2023 13:16:40 +0200 Subject: [PATCH 107/122] disable test that will always fail when replace Rust/1.52.1 with Rust/1.60.0 --- .github/workflows/test_eessi.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 04195dd619..5e496a6b7d 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -31,17 +31,17 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: pilot.eessi-hpc.org - - name: Test check_missing_installations.sh script - run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash - module load EasyBuild - eb --version - export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} - export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} - env | grep ^EESSI | sort - echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh +# - name: Test check_missing_installations.sh script +# run: | +# source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash +# module load EasyBuild +# eb --version +# export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} +# export EESSI_OS_TYPE=linux +# export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} +# env | grep ^EESSI | sort +# echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" +# ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | From 867392a602a3051d3ef7fc0bfa0715ff099b0a2c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 10 May 2023 19:41:37 +0200 Subject: [PATCH 108/122] add OpenBLAS/0.3.15 with GCC/10.3.0 to NESSI/2023.04 --- EESSI-pilot-install-software.sh | 12 ++++++++++++ eessi-2023.04.yml | 1 + 2 files changed, 13 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ef6e2b805b..b0a816de6f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -209,6 +209,18 @@ else fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." + +# add OpenBLAS: requires special handling for GENERIC CPU targets +# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 +if [[ $GENERIC -eq 1 ]]; then + echo ">> Installing OpenBLAS..." + ok_msg="Done with OpenBLAS (GENERIC architecture)!" + fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" + echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." + $EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" +fi + eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out ${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out} diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 04db10ba7a..ff4904d01f 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,4 +12,5 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb + - OpenBLAS-0.3.15-GCC-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb From 657b86cbae569e7d6550098b1e6e5ad4e7de8854 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 10 May 2023 20:57:57 +0200 Subject: [PATCH 109/122] add easyblock file to repo and use it instead of *-from-pr --- EESSI-pilot-install-software.sh | 3 +- easyblocks/o/openblas-pr1946-cc74e45.py | 299 ++++++++++++++++++++++++ 2 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 easyblocks/o/openblas-pr1946-cc74e45.py diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index b0a816de6f..ea40eba544 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -217,7 +217,8 @@ if [[ $GENERIC -eq 1 ]]; then ok_msg="Done with OpenBLAS (GENERIC architecture)!" fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - $EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi diff --git a/easyblocks/o/openblas-pr1946-cc74e45.py b/easyblocks/o/openblas-pr1946-cc74e45.py new file mode 100644 index 0000000000..defd5d0691 --- /dev/null +++ b/easyblocks/o/openblas-pr1946-cc74e45.py @@ -0,0 +1,299 @@ +""" +EasyBuild support for building and installing OpenBLAS, implemented as an easyblock + +@author: Andrew Edmondson (University of Birmingham) +@author: Alex Domingo (Vrije Universiteit Brussel) +@author: Terje Kvernes (University of Oslo) +""" +import os +import re +from distutils.version import LooseVersion +from easybuild.base import fancylogger +from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import ERROR, build_option +from easybuild.tools.filetools import read_file +from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.run import run_cmd, check_log_for_errors + +_log = fancylogger.getLogger('systemtools', fname=False) + +try: + from archspec import cpu as archspec_cpu + HAVE_ARCHSPEC = True +except ImportError as err: + _log.debug("Failed to import 'archspec' Python module: %s", err) + HAVE_ARCHSPEC = False + +TARGET = 'TARGET' + + +class EB_OpenBLAS(ConfigureMake): + """Support for building/installing OpenBLAS.""" + + @staticmethod + def extra_options(): + """Custom easyconfig parameters for OpenBLAS""" + extra_vars = { + 'targetfile': ['TargetList.txt', "File containing OpenBLAS target list", CUSTOM], + } + return ConfigureMake.extra_options(extra_vars) + + def configure_step(self): + """ set up some options - but no configure command to run""" + + default_opts = { + 'BINARY': '64', + 'CC': os.getenv('CC'), + 'FC': os.getenv('FC'), + 'USE_OPENMP': '1', + 'USE_THREAD': '1', + } + + # Handle the possibility of setting the target architecture as dynamic, + # where OpenBLAS will optimize the kernel at runtime. + self._dynamic_target = False + + compiler_optarch = self._optarch_for_compiler(build_option('optarch')) + + # Retain the (m)arch part of the optarch settings across the entire object. + self._optarch_architecture = compiler_optarch + + if '%s=' % TARGET in self.cfg['buildopts']: + # Add any TARGET in buildopts to default_opts, so it is passed to testopts and installopts + for buildopt in self.cfg['buildopts'].split(): + optpair = buildopt.split('=') + if optpair[0] == TARGET: + default_opts[optpair[0]] = optpair[1] + elif LooseVersion(self.version) < LooseVersion('0.3.6') and get_cpu_architecture() == POWER: + # There doesn't seem to be a POWER9 option yet, but POWER8 should work. + print_warning("OpenBLAS 0.3.5 and lower have known issues on POWER systems") + default_opts[TARGET] = 'POWER8' + elif compiler_optarch: + compiler_family = self.toolchain.comp_family() + self.log.info("EasyBuild full optarch requested for %s: %s" % (compiler_family, compiler_optarch)) + optarch_as_target = self._parse_optarch(compiler_optarch) + mapped_target = None + + if optarch_as_target: + # Note that _parse_optarch returns lowercased results, so GENERIC has become 'generic'. + if optarch_as_target == 'generic': + self._set_dynamic_architecture(default_opts) + mapped_target = 'generic' + else: + self.log.info("EasyBuild march: %s" % optarch_as_target) + openblas_targets = self._get_openblas_targets(self.cfg['targetfile']) + mapped_target = self._get_mapped_target(optarch_as_target, openblas_targets) + else: + self.log.info("Optarch specified for %s, but no march detected", compiler_family) + + if mapped_target is None: + print_warning("optarch for %s given as '%s'\n" + "EasyBuild was unable to map this to an equivalent OpenBLAS target!\n" + "OpenBLAS will be built to optimize its kernel at runtime!\n" + % (self.toolchain.comp_family(), compiler_optarch)) + self.log.warning("Unable to map %s to an OpenBLAS target, falling back to runtime optimization." + % compiler_optarch) + self._set_dynamic_architecture(default_opts) + elif mapped_target == 'generic': + self.log.info("Optarch 'GENERIC' requested, will enable runtime optimization.") + else: + mapped_target = mapped_target.upper() + self.log.info("Optarch mapped between EasyBuild and OpenBLAS to: " + mapped_target) + default_opts[TARGET] = mapped_target + + for key in sorted(default_opts.keys()): + for opts_key in ['buildopts', 'testopts', 'installopts']: + if '%s=' % key not in self.cfg[opts_key]: + self.cfg.update(opts_key, "%s='%s'" % (key, default_opts[key])) + + self.cfg.update('installopts', 'PREFIX=%s' % self.installdir) + + def build_step(self): + """ Custom build step excluding the tests """ + + # Equivalent to `make all` without the tests + build_parts = ['libs', 'netlib'] + for buildopt in self.cfg['buildopts'].split(): + if 'BUILD_RELAPACK' in buildopt and '1' in buildopt: + build_parts += ['re_lapack'] + build_parts += ['shared'] + + # If we're doing either a dynamic build or utilizing optarch, + # strip march from all environment variables except the EBVAR-prefixed ones. + # For dynamic builds we should ignore optarch completely and for optarch-set builds + # we need to adhere to TARGET and not march. + if self._dynamic_target is True or self._optarch_architecture is True: + self.log.info('Dynamic build requested, stripping march settings from environment variables') + for k in os.environ.keys(): + optarch_to_strip = '-' + self._optarch_architecture + if 'EBVAR' not in k and self._optarch_architecture in os.environ[k]: + os.environ[k] = os.environ[k].replace(optarch_to_strip, '') + + # Pass CFLAGS through command line to avoid redefinitions (issue xianyi/OpenBLAS#818) + cflags = 'CFLAGS' + if os.environ[cflags]: + self.cfg.update('buildopts', "%s='%s'" % (cflags, os.environ[cflags])) + del os.environ[cflags] + self.log.info("Environment variable %s unset and passed through command line" % cflags) + + makecmd = 'make' + if self.cfg['parallel']: + makecmd += ' -j %s' % self.cfg['parallel'] + + cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']]) + run_cmd(cmd, log_all=True, simple=True) + + def test_step(self): + """ Mandatory test step plus optional runtest """ + + run_tests = ['tests'] + if self.cfg['runtest']: + run_tests += [self.cfg['runtest']] + + for runtest in run_tests: + cmd = "%s make %s %s" % (self.cfg['pretestopts'], runtest, self.cfg['testopts']) + (out, _) = run_cmd(cmd, log_all=True, simple=False, regexp=False) + + # Raise an error if any test failed + check_log_for_errors(out, [('FATAL ERROR', ERROR)]) + + def sanity_check_step(self): + """ Custom sanity check for OpenBLAS """ + custom_paths = { + 'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h', + 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h', + 'lib/libopenblas.a', 'lib/libopenblas.%s' % get_shared_lib_ext()], + 'dirs': [], + } + super(EB_OpenBLAS, self).sanity_check_step(custom_paths=custom_paths) + + def _optarch_for_compiler(self, optarch): + """ + Extracts the appropriate optarch for the compiler currently being used. + If it is not compiler-specific it is returned as-is. + If no optarch is found, False is returned. + :param optarch: A complete optarch statement. + https://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html + """ + if optarch is False: + return False + + compiler = self.toolchain.comp_family() + compiler_specific_optarch_string = '' + + if type(optarch) == str: + compiler_specific_optarch_string = optarch + elif type(optarch) == dict: + if compiler in optarch: + compiler_specific_optarch_string = optarch[compiler] + else: + raise EasyBuildError("optarch in an unexpected format: '%s'" % type(optarch)).__class__.__name__ + + return compiler_specific_optarch_string + + def _parse_optarch(self, compiler_optarch): + """ + Pick the march out of a given optarch. + Note that the result is lowercased. + :param compiler_optarch: An optarch for a given compiler. + https://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html + """ + + target_arch = '' + pieces = compiler_optarch.split() + + for piece in pieces: + spec = piece.split('=') + if spec[0] == 'march' or spec[0] == '-march': + target_arch = spec[1] + + return target_arch.lower() + + def _get_openblas_targets(self, targetfile): + """ + Parse the openblas target file and generate a list of targets. + :param targetfile: A file with OpenBLAS targets. + """ + targets = [] + + if os.path.isfile(targetfile): + # Assumption, the OpenBLAS TargetList.txt has one target per line and that + # single words on a line is a target if they match a simple regexp... + re_target = re.compile(r'^[A-Z0-9_]+$') + for line in read_file(targetfile).splitlines(): + match = re_target.match(line) + if match is not None: + targets.append(line.strip().lower()) + else: + print_warning("Unable to find OpenBLAS targetfile '%s'" % os.path.realpath(targetfile)) + + return targets + + def _set_dynamic_architecture(self, default_opts): + """ + Sets the DYNAMIC_ARCH option for OpenBLAS, building a library that chooses + an optimized kernel at runtime. Also removes any previous TARGET setting, if any. + """ + default_opts['DYNAMIC_ARCH'] = 1 + default_opts.pop(TARGET, None) + self._dynamic_target = True + + def _get_mapped_target(self, march, openblas_targets): + """ + Attempts to match the given march in the list of openblas targets. + If archspec is installed, will try to match directly or follow ancestors for + architectures that will work. + Returns None if no target was found. + """ + + result = None + + if HAVE_ARCHSPEC: + self.log.info("Using archspec to match optarch to openblas targets.") + + openblas_arch = set(['alpha', 'arm', 'ia64', 'mips', 'mips64', + 'power', 'sparc', 'zarch']) + openblas_arch_map = { + 'amd64': 'x86_64', + 'powerpc64': 'power', + 'i386': 'x86', + 'aarch64': 'arm64', + } + openblas_arch.update(openblas_arch_map.keys()) + openblas_arch.update(openblas_arch_map.values()) + + skylake = set(["skylake"]) + available_targets = set(openblas_targets) | skylake | openblas_arch + + try: + uarch = archspec_cpu.TARGETS[march] + except KeyError: + warning_string = "Archspec was asked to find '" + march + "' as an architecture, but failed!" + print_warning(warning_string) + self.log.warning(warning_string) + return None + + if uarch.name in available_targets: + result = uarch.name + else: + self.log.info("No direct match for '" + march + "' between archspec and OpenBLAS, traversing ancestry.") + for uarch in uarch.ancestors: + if uarch.name in available_targets: + self.log.info("Ancestral match between '" + march + "' and '" + uarch.name + "'.") + result = uarch.name + break + + # Skylake for OpenBLAS is called 'skylakex'. Handle this exception exceptionally. + if result == 'skylake': + result = 'skylakex' + + else: + self.log.info("Unable to find archspec, optarch matching will be poor.") + if march == 'skylake': + result = 'skylakex' + elif march in openblas_targets: + result = march + + return result From 4a949930365e272409870ac1e19785ee856ecc5b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 12 May 2023 14:56:28 +0200 Subject: [PATCH 110/122] fix race condition when building OpenBLAS see https://github.com/spack/spack/pull/35395 and https://github.com/xianyi/OpenBLAS/issues/3899 --- EESSI-pilot-install-software.sh | 19 ++++++++++++++ OpenBLAS-0.3.15-GCC-10.3.0.eb | 38 ++++++++++++++++++++++++++++ patches/fix-race-build-targets.patch | 14 ++++++++++ 3 files changed, 71 insertions(+) create mode 100644 OpenBLAS-0.3.15-GCC-10.3.0.eb create mode 100644 patches/fix-race-build-targets.patch diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ea40eba544..826eaf156f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -210,6 +210,25 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." +# we try to use a modified ec file for OpenBLAS shipped with this PR +if [[ $GENERIC -eq 1 ]]; then + echo ">> Installing modified OpenBLAS..." + ok_msg="Done with OpenBLAS (GENERIC architecture)!" + fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" + echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." + #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py ./OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" +else + echo ">> Installing modified OpenBLAS..." + ok_msg="Done with OpenBLAS (NON-GENERIC architecture)!" + fail_msg="Installation of OpenBLAS (NON-GENERIC architecture) failed!" + echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." + #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB ./OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" +fi + # add OpenBLAS: requires special handling for GENERIC CPU targets # If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 if [[ $GENERIC -eq 1 ]]; then diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb new file mode 100644 index 0000000000..a443833909 --- /dev/null +++ b/OpenBLAS-0.3.15-GCC-10.3.0.eb @@ -0,0 +1,38 @@ +name = 'OpenBLAS' +version = '0.3.15' + +homepage = 'https://xianyi.github.com/OpenBLAS/' +description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." + +toolchain = {'name': 'GCC', 'version': '10.3.0'} + +source_urls = [ + # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble + 'https://www.netlib.org/lapack/timing/', + 'https://github.com/xianyi/OpenBLAS/archive/', +] +sources = ['v%(version)s.tar.gz'] +patches = [ + ('large.tgz', '.'), + ('timing.tgz', '.'), + 'OpenBLAS-%(version)s_icelake-detection.patch', + 'OpenBLAS-%(version)s_fix-aarch64.patch', + 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', + 'patches/OpenBLAS-%(version)s_fix-race-build-targets.patch', + +] +checksums = [ + '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz + 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz + '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz + 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch + 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch + # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch + 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', + '7478ea187620591c0485bcc713a31d9667c19b0a90a7b2391e8cbffc5281d250', # OpenBLAS-0.3.15_fix-race-build-targets.patch +] + +# extensive testing can be enabled by uncommenting the line below +# runtest = 'PATH=.:$PATH lapack-timing' + +moduleclass = 'numlib' diff --git a/patches/fix-race-build-targets.patch b/patches/fix-race-build-targets.patch new file mode 100644 index 0000000000..d82ca10dd4 --- /dev/null +++ b/patches/fix-race-build-targets.patch @@ -0,0 +1,14 @@ +diff --git a/Makefile b/Makefile +index 555d1c46..d90c2d37 100644 +--- a/Makefile ++++ b/Makefile +@@ -39,6 +39,9 @@ SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cp + .PHONY : all libs netlib $(RELA) test ctest shared install + .NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test + ++shared: libs netlib $(RELA) ++tests: libs netlib $(RELA) shared ++ + all :: libs netlib $(RELA) tests shared + @echo + @echo " OpenBLAS build complete. ($(LIB_COMPONENTS))" From 0909aac4ded881eead6abe847d485b4ee61d6143 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 12 May 2023 19:25:53 +0200 Subject: [PATCH 111/122] ensure patch file name is same as in ec file --- ...targets.patch => OpenBLAS-0.3.15_fix-race-build-targets.patch} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename patches/{fix-race-build-targets.patch => OpenBLAS-0.3.15_fix-race-build-targets.patch} (100%) diff --git a/patches/fix-race-build-targets.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets.patch similarity index 100% rename from patches/fix-race-build-targets.patch rename to patches/OpenBLAS-0.3.15_fix-race-build-targets.patch From 397b9e928d7a9055fd01f4b244f03f2e5673cf83 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 12 May 2023 19:38:36 +0200 Subject: [PATCH 112/122] try a new patch adding targets to .NOTPARALLEL --- OpenBLAS-0.3.15-GCC-10.3.0.eb | 6 ++++-- ...penBLAS-0.3.15_fix-race-build-targets-2.patch | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb index a443833909..946c6cdd5c 100644 --- a/OpenBLAS-0.3.15-GCC-10.3.0.eb +++ b/OpenBLAS-0.3.15-GCC-10.3.0.eb @@ -18,7 +18,7 @@ patches = [ 'OpenBLAS-%(version)s_icelake-detection.patch', 'OpenBLAS-%(version)s_fix-aarch64.patch', 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', - 'patches/OpenBLAS-%(version)s_fix-race-build-targets.patch', + 'patches/OpenBLAS-%(version)s_fix-race-build-targets-2.patch', ] checksums = [ @@ -29,7 +29,9 @@ checksums = [ 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', - '7478ea187620591c0485bcc713a31d9667c19b0a90a7b2391e8cbffc5281d250', # OpenBLAS-0.3.15_fix-race-build-targets.patch + # '7478ea187620591c0485bcc713a31d9667c19b0a90a7b2391e8cbffc5281d250', # OpenBLAS-0.3.15_fix-race-build-targets.patch + 'e2d642a14c2339eb1e699e568061a4da889e590e644c48cc14e486c17d3df14b', # OpenBLAS-0.3.15_fix-race-build-targets-2.patch + ] # extensive testing can be enabled by uncommenting the line below diff --git a/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch new file mode 100644 index 0000000000..c419b310fb --- /dev/null +++ b/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch @@ -0,0 +1,16 @@ +diff --git a/Makefile b/Makefile +index 555d1c46..a725af82 100644 +--- a/Makefile ++++ b/Makefile +@@ -37,7 +37,10 @@ LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast,$(LAPACK_FFLAGS)) + SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test + + .PHONY : all libs netlib $(RELA) test ctest shared install +-.NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test ++.NOTPARALLEL : all libs netlib $(RELA) tests shared prof lapack-test install blas-test ++ ++shared: libs netlib $(RELA) ++tests: libs netlib $(RELA) shared + + all :: libs netlib $(RELA) tests shared + @echo From b07d227f4de9766273e38042eb8a4b3d9f2c4b04 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 12 May 2023 21:08:51 +0200 Subject: [PATCH 113/122] removing patch for OpenBLAS-0.3.15 + using --parallel 1 instead --- EESSI-pilot-install-software.sh | 21 +--------- OpenBLAS-0.3.15-GCC-10.3.0.eb | 40 ------------------- eessi-2023.04.yml | 4 +- ...BLAS-0.3.15_fix-race-build-targets-2.patch | 16 -------- ...enBLAS-0.3.15_fix-race-build-targets.patch | 14 ------- 5 files changed, 4 insertions(+), 91 deletions(-) delete mode 100644 OpenBLAS-0.3.15-GCC-10.3.0.eb delete mode 100644 patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch delete mode 100644 patches/OpenBLAS-0.3.15_fix-race-build-targets.patch diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 826eaf156f..021cca6259 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -210,25 +210,6 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." -# we try to use a modified ec file for OpenBLAS shipped with this PR -if [[ $GENERIC -eq 1 ]]; then - echo ">> Installing modified OpenBLAS..." - ok_msg="Done with OpenBLAS (GENERIC architecture)!" - fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py ./OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -else - echo ">> Installing modified OpenBLAS..." - ok_msg="Done with OpenBLAS (NON-GENERIC architecture)!" - fail_msg="Installation of OpenBLAS (NON-GENERIC architecture) failed!" - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - $EB ./OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - # add OpenBLAS: requires special handling for GENERIC CPU targets # If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 if [[ $GENERIC -eq 1 ]]; then @@ -237,7 +218,7 @@ if [[ $GENERIC -eq 1 ]]; then fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb deleted file mode 100644 index 946c6cdd5c..0000000000 --- a/OpenBLAS-0.3.15-GCC-10.3.0.eb +++ /dev/null @@ -1,40 +0,0 @@ -name = 'OpenBLAS' -version = '0.3.15' - -homepage = 'https://xianyi.github.com/OpenBLAS/' -description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." - -toolchain = {'name': 'GCC', 'version': '10.3.0'} - -source_urls = [ - # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble - 'https://www.netlib.org/lapack/timing/', - 'https://github.com/xianyi/OpenBLAS/archive/', -] -sources = ['v%(version)s.tar.gz'] -patches = [ - ('large.tgz', '.'), - ('timing.tgz', '.'), - 'OpenBLAS-%(version)s_icelake-detection.patch', - 'OpenBLAS-%(version)s_fix-aarch64.patch', - 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', - 'patches/OpenBLAS-%(version)s_fix-race-build-targets-2.patch', - -] -checksums = [ - '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz - 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz - '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz - 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch - 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch - # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch - 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', - # '7478ea187620591c0485bcc713a31d9667c19b0a90a7b2391e8cbffc5281d250', # OpenBLAS-0.3.15_fix-race-build-targets.patch - 'e2d642a14c2339eb1e699e568061a4da889e590e644c48cc14e486c17d3df14b', # OpenBLAS-0.3.15_fix-race-build-targets-2.patch - -] - -# extensive testing can be enabled by uncommenting the line below -# runtest = 'PATH=.:$PATH lapack-timing' - -moduleclass = 'numlib' diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index ff4904d01f..096fefb8fb 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,5 +12,7 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb + - OpenBLAS-0.3.15-GCC-10.3.0.eb: + options: + parallel: 1 - OpenMPI-4.1.1-GCC-10.3.0.eb diff --git a/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch deleted file mode 100644 index c419b310fb..0000000000 --- a/patches/OpenBLAS-0.3.15_fix-race-build-targets-2.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/Makefile b/Makefile -index 555d1c46..a725af82 100644 ---- a/Makefile -+++ b/Makefile -@@ -37,7 +37,10 @@ LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast,$(LAPACK_FFLAGS)) - SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test - - .PHONY : all libs netlib $(RELA) test ctest shared install --.NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test -+.NOTPARALLEL : all libs netlib $(RELA) tests shared prof lapack-test install blas-test -+ -+shared: libs netlib $(RELA) -+tests: libs netlib $(RELA) shared - - all :: libs netlib $(RELA) tests shared - @echo diff --git a/patches/OpenBLAS-0.3.15_fix-race-build-targets.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets.patch deleted file mode 100644 index d82ca10dd4..0000000000 --- a/patches/OpenBLAS-0.3.15_fix-race-build-targets.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/Makefile b/Makefile -index 555d1c46..d90c2d37 100644 ---- a/Makefile -+++ b/Makefile -@@ -39,6 +39,9 @@ SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cp - .PHONY : all libs netlib $(RELA) test ctest shared install - .NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test - -+shared: libs netlib $(RELA) -+tests: libs netlib $(RELA) shared -+ - all :: libs netlib $(RELA) tests shared - @echo - @echo " OpenBLAS build complete. ($(LIB_COMPONENTS))" From 94df2a21d28138c270cdaa12defe88a6ba6b5df8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 12 May 2023 22:02:15 +0200 Subject: [PATCH 114/122] 'parallel: 1' doesn't work as option in easystack file --- EESSI-pilot-install-software.sh | 13 ++++++++++++- eessi-2023.04.yml | 4 +--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 021cca6259..6a3503e434 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -212,14 +212,25 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL # add OpenBLAS: requires special handling for GENERIC CPU targets # If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 +# we need to specify --parallel 1 or run into weird linking issues supposedly +# introduced by GNU make v4.4, see for example +# https://github.com/xianyi/OpenBLAS/issues/3899 and https://github.com/xianyi/OpenBLAS/issues/3979 +# a fix may be provided with https://github.com/xianyi/OpenBLAS/pull/3983 if [[ $GENERIC -eq 1 ]]; then echo ">> Installing OpenBLAS..." ok_msg="Done with OpenBLAS (GENERIC architecture)!" fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - #$EB --include-easyblocks-from-pr 1946 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" +else + # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' + # does not work yet via the easystack file + echo ">> Installing OpenBLAS..." + ok_msg="Done with OpenBLAS (non-GENERIC architecture)!" + fail_msg="Installation of OpenBLAS (non-GENERIC architecture) failed!" + $EB --parallel 1 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + check_exit_code $? "${ok_msg}" "${fail_msg}" fi eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 096fefb8fb..ff4904d01f 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,7 +12,5 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb: - options: - parallel: 1 + - OpenBLAS-0.3.15-GCC-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb From ea134c2f7ebffdec70c5ff2afdabd03293735c22 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 13 May 2023 07:40:47 +0200 Subject: [PATCH 115/122] trying to put 1 in quotes in easystack file --- EESSI-pilot-install-software.sh | 16 ++++++++-------- eessi-2023.04.yml | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 6a3503e434..233262286b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -223,14 +223,14 @@ if [[ $GENERIC -eq 1 ]]; then echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" -else - # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' - # does not work yet via the easystack file - echo ">> Installing OpenBLAS..." - ok_msg="Done with OpenBLAS (non-GENERIC architecture)!" - fail_msg="Installation of OpenBLAS (non-GENERIC architecture) failed!" - $EB --parallel 1 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" +#else +# # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' +# # does not work yet via the easystack file +# echo ">> Installing OpenBLAS..." +# ok_msg="Done with OpenBLAS (non-GENERIC architecture)!" +# fail_msg="Installation of OpenBLAS (non-GENERIC architecture) failed!" +# $EB --parallel 1 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot +# check_exit_code $? "${ok_msg}" "${fail_msg}" fi eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index ff4904d01f..47271f0431 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,5 +12,7 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb + - OpenBLAS-0.3.15-GCC-10.3.0.eb: + options: + parallel: "1" - OpenMPI-4.1.1-GCC-10.3.0.eb From dc11c9a102987cb55decb09f4e0045007ef655da Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 13 May 2023 08:23:03 +0200 Subject: [PATCH 116/122] reintroducing new patch for OpenBLAS based on OpenBLAS/PR3983 --- EESSI-pilot-install-software.sh | 2 +- OpenBLAS-0.3.15-GCC-10.3.0.eb | 37 ++++++++++++ eessi-2023.04.yml | 4 +- ...5_fix-race-build-targets-with-pr3983.patch | 57 +++++++++++++++++++ 4 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 OpenBLAS-0.3.15-GCC-10.3.0.eb create mode 100644 patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 233262286b..a20b2db6c2 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -221,7 +221,7 @@ if [[ $GENERIC -eq 1 ]]; then ok_msg="Done with OpenBLAS (GENERIC architecture)!" fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" #else # # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb new file mode 100644 index 0000000000..3cd10280b4 --- /dev/null +++ b/OpenBLAS-0.3.15-GCC-10.3.0.eb @@ -0,0 +1,37 @@ +name = 'OpenBLAS' +version = '0.3.15' + +homepage = 'https://xianyi.github.com/OpenBLAS/' +description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." + +toolchain = {'name': 'GCC', 'version': '10.3.0'} + +source_urls = [ + # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble + 'https://www.netlib.org/lapack/timing/', + 'https://github.com/xianyi/OpenBLAS/archive/', +] +sources = ['v%(version)s.tar.gz'] +patches = [ + ('large.tgz', '.'), + ('timing.tgz', '.'), + 'OpenBLAS-%(version)s_icelake-detection.patch', + 'OpenBLAS-%(version)s_fix-aarch64.patch', + 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', + 'patches/OpenBLAS-%(version)s_fix-race-build-targets-with-pr3983.patch', +] +checksums = [ + '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz + 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz + '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz + 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch + 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch + # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch + 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', + 'a227c534fce104f2633bbc2f66a31752e9c6947143f73bf85514ad22ff85da22', # OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch +] + +# extensive testing can be enabled by uncommenting the line below +# runtest = 'PATH=.:$PATH lapack-timing' + +moduleclass = 'numlib' diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 47271f0431..ff4904d01f 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,7 +12,5 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb: - options: - parallel: "1" + - OpenBLAS-0.3.15-GCC-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb diff --git a/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch new file mode 100644 index 0000000000..1bd111e54e --- /dev/null +++ b/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch @@ -0,0 +1,57 @@ +diff --git a/Makefile b/Makefile +index 555d1c46..2c3f9660 100644 +--- a/Makefile ++++ b/Makefile +@@ -37,9 +37,9 @@ LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast,$(LAPACK_FFLAGS)) + SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test + + .PHONY : all libs netlib $(RELA) test ctest shared install +-.NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test ++.NOTPARALLEL : shared + +-all :: libs netlib $(RELA) tests shared ++all :: tests + @echo + @echo " OpenBLAS build complete. ($(LIB_COMPONENTS))" + @echo +@@ -116,7 +116,7 @@ endif + @echo "To install the library, you can run \"make PREFIX=/path/to/your/installation install\"." + @echo + +-shared : ++shared : libs netlib $(RELA) + ifneq ($(NO_SHARED), 1) + ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS Android Haiku FreeBSD DragonFly)) + @$(MAKE) -C exports so +@@ -140,7 +140,7 @@ ifeq ($(OSNAME), CYGWIN_NT) + endif + endif + +-tests : ++tests : shared + ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN))) + touch $(LIBNAME) + ifndef NO_FBLAS +diff --git a/getarch.c b/getarch.c +index f48944f3..4bd93e3a 100644 +--- a/getarch.c ++++ b/getarch.c +@@ -1547,15 +1547,15 @@ printf("ELF_VERSION=2\n"); + + #ifdef MAKE_NB_JOBS + #if MAKE_NB_JOBS > 0 +- printf("MAKE += -j %d\n", MAKE_NB_JOBS); ++ printf("MAKEFLAGS += -j %d\n", MAKE_NB_JOBS); + #else + // Let make use parent -j argument or -j1 if there + // is no make parent + #endif + #elif NO_PARALLEL_MAKE==1 +- printf("MAKE += -j 1\n"); ++ printf("MAKEFLAGS += -j 1\n"); + #else +- printf("MAKE += -j %d\n", get_num_cores()); ++ printf("MAKEFLAGS += -j %d\n", get_num_cores()); + #endif + + break; From 35a9e27015213ecc94c13e303ffce2b63757480e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 13 May 2023 08:44:54 +0200 Subject: [PATCH 117/122] Revert "reintroducing new patch for OpenBLAS based on OpenBLAS/PR3983" This reverts commit dc11c9a102987cb55decb09f4e0045007ef655da. patching OpenBLAS didn't work out. So only option to build OpenBLAS seems to be to run eb with --parallel 1 or parallel: "1" --- EESSI-pilot-install-software.sh | 2 +- OpenBLAS-0.3.15-GCC-10.3.0.eb | 37 ------------ eessi-2023.04.yml | 4 +- ...5_fix-race-build-targets-with-pr3983.patch | 57 ------------------- 4 files changed, 4 insertions(+), 96 deletions(-) delete mode 100644 OpenBLAS-0.3.15-GCC-10.3.0.eb delete mode 100644 patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index a20b2db6c2..233262286b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -221,7 +221,7 @@ if [[ $GENERIC -eq 1 ]]; then ok_msg="Done with OpenBLAS (GENERIC architecture)!" fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" #else # # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb deleted file mode 100644 index 3cd10280b4..0000000000 --- a/OpenBLAS-0.3.15-GCC-10.3.0.eb +++ /dev/null @@ -1,37 +0,0 @@ -name = 'OpenBLAS' -version = '0.3.15' - -homepage = 'https://xianyi.github.com/OpenBLAS/' -description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." - -toolchain = {'name': 'GCC', 'version': '10.3.0'} - -source_urls = [ - # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble - 'https://www.netlib.org/lapack/timing/', - 'https://github.com/xianyi/OpenBLAS/archive/', -] -sources = ['v%(version)s.tar.gz'] -patches = [ - ('large.tgz', '.'), - ('timing.tgz', '.'), - 'OpenBLAS-%(version)s_icelake-detection.patch', - 'OpenBLAS-%(version)s_fix-aarch64.patch', - 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', - 'patches/OpenBLAS-%(version)s_fix-race-build-targets-with-pr3983.patch', -] -checksums = [ - '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz - 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz - '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz - 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch - 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch - # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch - 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', - 'a227c534fce104f2633bbc2f66a31752e9c6947143f73bf85514ad22ff85da22', # OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch -] - -# extensive testing can be enabled by uncommenting the line below -# runtest = 'PATH=.:$PATH lapack-timing' - -moduleclass = 'numlib' diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index ff4904d01f..47271f0431 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,5 +12,7 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb + - OpenBLAS-0.3.15-GCC-10.3.0.eb: + options: + parallel: "1" - OpenMPI-4.1.1-GCC-10.3.0.eb diff --git a/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch b/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch deleted file mode 100644 index 1bd111e54e..0000000000 --- a/patches/OpenBLAS-0.3.15_fix-race-build-targets-with-pr3983.patch +++ /dev/null @@ -1,57 +0,0 @@ -diff --git a/Makefile b/Makefile -index 555d1c46..2c3f9660 100644 ---- a/Makefile -+++ b/Makefile -@@ -37,9 +37,9 @@ LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast,$(LAPACK_FFLAGS)) - SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test - - .PHONY : all libs netlib $(RELA) test ctest shared install --.NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test -+.NOTPARALLEL : shared - --all :: libs netlib $(RELA) tests shared -+all :: tests - @echo - @echo " OpenBLAS build complete. ($(LIB_COMPONENTS))" - @echo -@@ -116,7 +116,7 @@ endif - @echo "To install the library, you can run \"make PREFIX=/path/to/your/installation install\"." - @echo - --shared : -+shared : libs netlib $(RELA) - ifneq ($(NO_SHARED), 1) - ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS Android Haiku FreeBSD DragonFly)) - @$(MAKE) -C exports so -@@ -140,7 +140,7 @@ ifeq ($(OSNAME), CYGWIN_NT) - endif - endif - --tests : -+tests : shared - ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN))) - touch $(LIBNAME) - ifndef NO_FBLAS -diff --git a/getarch.c b/getarch.c -index f48944f3..4bd93e3a 100644 ---- a/getarch.c -+++ b/getarch.c -@@ -1547,15 +1547,15 @@ printf("ELF_VERSION=2\n"); - - #ifdef MAKE_NB_JOBS - #if MAKE_NB_JOBS > 0 -- printf("MAKE += -j %d\n", MAKE_NB_JOBS); -+ printf("MAKEFLAGS += -j %d\n", MAKE_NB_JOBS); - #else - // Let make use parent -j argument or -j1 if there - // is no make parent - #endif - #elif NO_PARALLEL_MAKE==1 -- printf("MAKE += -j 1\n"); -+ printf("MAKEFLAGS += -j 1\n"); - #else -- printf("MAKE += -j %d\n", get_num_cores()); -+ printf("MAKEFLAGS += -j %d\n", get_num_cores()); - #endif - - break; From 13a15fa4c5e3d87753379fdd50b7165e6f16505c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 13 May 2023 08:48:28 +0200 Subject: [PATCH 118/122] clarify comment + cleanup obsolete else branch --- EESSI-pilot-install-software.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 233262286b..130f56b160 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -216,6 +216,7 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL # introduced by GNU make v4.4, see for example # https://github.com/xianyi/OpenBLAS/issues/3899 and https://github.com/xianyi/OpenBLAS/issues/3979 # a fix may be provided with https://github.com/xianyi/OpenBLAS/pull/3983 +# the fix did not work, so only option seems to be --parallel 1 if [[ $GENERIC -eq 1 ]]; then echo ">> Installing OpenBLAS..." ok_msg="Done with OpenBLAS (GENERIC architecture)!" @@ -223,14 +224,6 @@ if [[ $GENERIC -eq 1 ]]; then echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" -#else -# # also install OpenBLAS for non-GENERIC targets because it seems 'parallel' -# # does not work yet via the easystack file -# echo ">> Installing OpenBLAS..." -# ok_msg="Done with OpenBLAS (non-GENERIC architecture)!" -# fail_msg="Installation of OpenBLAS (non-GENERIC architecture) failed!" -# $EB --parallel 1 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# check_exit_code $? "${ok_msg}" "${fail_msg}" fi eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out From 3bc684ee3689f04c4f1399b934f5bceb1c387232 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 16 May 2023 21:34:06 +0200 Subject: [PATCH 119/122] try to use make v4.3 as build dependency for OpenBLAS --- EESSI-pilot-install-software.sh | 3 ++- eessi-2023.04.yml | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 130f56b160..419ce586e8 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -217,12 +217,13 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL # https://github.com/xianyi/OpenBLAS/issues/3899 and https://github.com/xianyi/OpenBLAS/issues/3979 # a fix may be provided with https://github.com/xianyi/OpenBLAS/pull/3983 # the fix did not work, so only option seems to be --parallel 1 +# another option seems to be to use make v4.3 as build dependency if [[ $GENERIC -eq 1 ]]; then echo ">> Installing OpenBLAS..." ok_msg="Done with OpenBLAS (GENERIC architecture)!" fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!" echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - $EB --parallel 1 --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" fi diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml index 47271f0431..ff4904d01f 100644 --- a/eessi-2023.04.yml +++ b/eessi-2023.04.yml @@ -12,7 +12,5 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - Perl-5.32.1-GCCcore-10.3.0.eb - - OpenBLAS-0.3.15-GCC-10.3.0.eb: - options: - parallel: "1" + - OpenBLAS-0.3.15-GCC-10.3.0.eb - OpenMPI-4.1.1-GCC-10.3.0.eb From 8655929e776c77a8406bba9c43679e469d65410e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 16 May 2023 22:59:19 +0200 Subject: [PATCH 120/122] add easyconfig with additional build dependency --- OpenBLAS-0.3.15-GCC-10.3.0.eb | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 OpenBLAS-0.3.15-GCC-10.3.0.eb diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb new file mode 100644 index 0000000000..13b731bc6d --- /dev/null +++ b/OpenBLAS-0.3.15-GCC-10.3.0.eb @@ -0,0 +1,39 @@ +name = 'OpenBLAS' +version = '0.3.15' + +homepage = 'https://xianyi.github.com/OpenBLAS/' +description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version." + +toolchain = {'name': 'GCC', 'version': '10.3.0'} + +source_urls = [ + # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble + 'https://www.netlib.org/lapack/timing/', + 'https://github.com/xianyi/OpenBLAS/archive/', +] +sources = ['v%(version)s.tar.gz'] +patches = [ + ('large.tgz', '.'), + ('timing.tgz', '.'), + 'OpenBLAS-%(version)s_icelake-detection.patch', + 'OpenBLAS-%(version)s_fix-aarch64.patch', + 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch', +] +checksums = [ + '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz + 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz + '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz + 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch + 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch + # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch + 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971', +] + +builddependencies = [ + ('make', '4.3'), +] + +# extensive testing can be enabled by uncommenting the line below +# runtest = 'PATH=.:$PATH lapack-timing' + +moduleclass = 'numlib' From c9877fe89e2d7286dcd4c492fa747b2beade300a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 17 May 2023 01:14:41 +0200 Subject: [PATCH 121/122] add improved check-result.sh script --- bot/check-result.sh | 155 +++++++++++++++++++++++++++++++------------- 1 file changed, 111 insertions(+), 44 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index e28ae4e38e..24c9b55aa6 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -4,8 +4,8 @@ # Intended use is that it is called by a (batch) job running on a compute # node. # -# This script is part of the EESSI compatibility layer, see -# https://github.com/EESSI/compatibility-layer.git +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) # @@ -147,64 +147,131 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ${TGZ} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS - echo "[RESULT]" > ${job_result_file} - echo "summary = :grin: SUCCESS" >> ${job_result_file} - echo "details =" >> ${job_result_file} + summary=":grin: SUCCESS" + status="success" else # FAILURE - echo "[RESULT]" > ${job_result_file} - echo "summary = :cry: FAILURE" >> ${job_result_file} - echo "details =" >> ${job_result_file} + summary=":cry: FAILURE" + status="failure" fi -function succeeded() { - echo " :heavy_check_mark: ${1}" +### Example details/descriptions +# +#
:cry: FAILURE _(click triangle for detailed information)_Details:
    :heavy_check_mark: job output file slurm-470503.out
    :heavy_multiplication_x: found message matching ERROR:
    :heavy_multiplication_x: found message matching FAILED:
    :heavy_multiplication_x: found message matching required modules missing:
    :heavy_check_mark: found message(s) matching No missing modules!
    :heavy_check_mark: found message matching tar.gz created!
Artefacts:
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +# +#
    :grin: SUCCESS _(click triangle for detailed information)_Details:
        :heavy_check_mark: job output file slurm-470503.out
        :heavy_check_mark: found message matching ERROR:
        :heavy_check_mark: found message matching FAILED:
        :heavy_check_mark: found message matching required modules missing:
        :heavy_check_mark: found message(s) matching No missing modules!
        :heavy_check_mark: found message matching tar.gz created!
    Artefacts:
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
    • +# +### + +# construct and write complete PR comment details +comment_template="
    __SUMMARY_FMT____DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for detailed information)_" +comment_details_fmt="Details:
    __DETAILS_LIST__" +comment_artefacts_fmt="Artefacts:
    __ARTEFACTS_LIST__" +comment_success_item_fmt=":heavy_check_mark: __ITEM__" +comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" + +function print_br_item() { + format="${1}" + item="${2}" + echo -n "    ${format//__ITEM__/${item}}
    " } -function failed() { - echo " :heavy_multiplication_x: ${1}" +function print_list_item() { + format="${1}" + item="${2}" + echo -n "
  • ${format//__ITEM__/${item}}
    • " } -if [[ ${SLURM} -eq 1 ]]; then - succeeded "job output file ${job_out}" >> ${job_result_file} -else - failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} -fi +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} -if [[ ${ERROR} -eq 0 ]]; then - succeeded "no message matching ${GP_error}" >> ${job_result_file} -else - failed "found message matching ${GP_error}" >> ${job_result_file} -fi +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} -if [[ ${FAILED} -eq 0 ]]; then - succeeded "no message matching ${GP_failed}" >> ${job_result_file} -else - failed "found message matching ${GP_failed}" >> ${job_result_file} -fi +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} -if [[ ${MISSING} -eq 0 ]]; then - succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} -else - failed "found message matching ${GP_req_missing}" >> ${job_result_file} -fi +echo "[RESULT]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} -if [[ ${NO_MISSING} -eq 1 ]]; then - succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} -else - failed "no message matching ${GP_no_missing}" >> ${job_result_file} -fi +# construct values for placeholders in comment_template: +# - __SUMMARY_FMT__ -> variable $comment_summary +# - __DETAILS_FMT__ -> variable $comment_details +# - __ARTEFACTS_FMT__ -> variable $comment_artefacts -if [[ ${TGZ} -eq 1 ]]; then - succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} -else - failed "no message matching ${GP_tgz_created}" >> ${job_result_file} -fi +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" + +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file matching ${GP_slurm_out}" +CoDeList="${CoDeList}$(add_detail ${SLURM} 1 ${success_msg} ${failure_msg})" + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList="${CoDeList}$(add_detail ${ERROR} 0 ${success_msg} ${failure_msg})" + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList="${CoDeList}$(add_detail ${FAILED} 0 ${success_msg} ${failure_msg})" + +success_msg="no message matching ${GP_req_missing}" +failure_msg="found message matching ${GP_req_missing}" +CoDeList="${CoDeList}$(add_detail ${MISSING} 0 ${success_msg} ${failure_msg})" + +success_msg="found message(s) matching ${GP_no_missing}" +failure_msg="no message matching ${GP_no_missing}" +CoDeList="${CoDeList}$(add_detail ${NO_MISSING} 1 ${success_msg} ${failure_msg})" -echo "artefacts =" >> ${job_result_file} +success_msg="found message matching ${GP_tgz_created}" +failure_msg="no message matching ${GP_tgz_created}" +CoDeList="${CoDeList}$(add_detail ${TGZ} 1 ${success_msg} ${failure_msg})" +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + + +# first construct comment_artefacts_list, abbreviated CoArList +# then use it to set comment_artefacts +CoArList="" + +# TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - echo " ${TARBALL}" >> ${job_result_file} + CoArList="${CoArList}$(print_list_item '__ITEM__' ${TARBALL})" +else + CoArList="${CoArList}$(print_list_item 'No artefacts were created/found.' '')" fi +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${CoArList}}" + +# now put all pieces together creating comment_details from comment_template +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + +echo "${comment_description}" >> ${job_result_file} + +# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts +# - this should make use of subsequent steps such as deploying a tarball more +# efficient +echo "status = ${status}" >> ${job_result_file} +echo "artefacts = " >> ${job_result_file} +echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} exit 0 From 18b5175af37c6e4d62cfafae3638665798ca4589 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 17 May 2023 01:50:58 +0200 Subject: [PATCH 122/122] fix quoting for status messages --- bot/check-result.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 24c9b55aa6..8774d68f68 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -223,27 +223,27 @@ CoDeList="" success_msg="job output file ${job_out}" failure_msg="no job output file matching ${GP_slurm_out}" -CoDeList="${CoDeList}$(add_detail ${SLURM} 1 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" -CoDeList="${CoDeList}$(add_detail ${ERROR} 0 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_failed}" failure_msg="found message matching ${GP_failed}" -CoDeList="${CoDeList}$(add_detail ${FAILED} 0 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_req_missing}" failure_msg="found message matching ${GP_req_missing}" -CoDeList="${CoDeList}$(add_detail ${MISSING} 0 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" -CoDeList="${CoDeList}$(add_detail ${NO_MISSING} 1 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" -CoDeList="${CoDeList}$(add_detail ${TGZ} 1 ${success_msg} ${failure_msg})" +CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" @@ -254,11 +254,11 @@ CoArList="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - CoArList="${CoArList}$(print_list_item '__ITEM__' ${TARBALL})" + CoArList=${CoArList}$(print_list_item '__ITEM__' "${TARBALL}") else - CoArList="${CoArList}$(print_list_item 'No artefacts were created/found.' '')" + CoArList=${CoArList}$(print_list_item 'No artefacts were created/found.' '') fi -comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${CoArList}}" +comment_artefacts=${comment_artefacts_fmt/__ARTEFACTS_LIST__/${CoArList}} # now put all pieces together creating comment_details from comment_template comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}