Merge pull request #214 from EESSI/main

Branch is behind

Author: xinan1911

.github/workflows/scorecards.yml

@@ -4,16 +4,11 @@
 
 name: Scorecards supply-chain security
 on:
-  # For Branch-Protection check. Only the default branch is supported. See
-  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
-  branch_protection_rule:
   # To guarantee Maintained check is occasionally updated. See
   # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
   schedule:
     - cron: '25 15 * * 3'
   push:
-    branches: [ "main" ]
-  pull_request:
     branches:
       - main
 
@@ -36,12 +31,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -63,14 +58,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
         with:
           sarif_file: results.sarif

.github/workflows/script_module_list.yml

@@ -3,7 +3,7 @@ name: Module overview script (lint + test)
 on:
   push:
     paths:
-      - 'scripts/README.mds/**'
+      - 'scripts/**'
       - './.github/**'
   pull_request:
     paths:
@@ -25,7 +25,7 @@ jobs:
         with:
           python-version: "3.6"
       - name: Run flake8
-        uses: py-actions/flake8@v2
+        uses: py-actions/flake8@84ec6726560b6d5bd68f2a5bed83d62b52bb50ba # v2.3.0
         with:
           max-line-length: "120"
           path: "scripts/available_software"

.github/workflows/test.yml

@@ -19,7 +19,8 @@ jobs:
       uses: codespell-project/actions-codespell@22ff5a2e4b591290baf82d47c9feadac31c65441 # v1.0
       with:
         check_filenames: true
-        skip: '*.pdf,.git'
+        # MarkDown files in docs/available_software/detail are skipped because they are auto-generated
+        skip: '*.pdf,.git,*.json,./docs/available_software/detail/*.md'
 
     # - name: Markdown Linting Action
     #   uses: avto-dev/[email protected]

.github/workflows/update_available_software.yml

@@ -0,0 +1,66 @@
+name: Update overview of available software in EESSI
+on:
+  workflow_dispatch:
+  schedule:
+    # run every 4 hours
+    - cron: '0 */4 * * *'
+jobs:
+  update_available_software:
+    if: github.repository_owner == 'EESSI'   # Prevent running on forks
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+
+      - name: set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: '3.10'
+          architecture: x64
+
+      - name: Mount EESSI
+        uses: eessi/github-action-eessi@e1f8f20638ea417a18d23ab29443ee34794ff900 # v3.1.0
+        with:
+          eessi_stack_version: '2023.06'
+
+      - name: update overview of available software
+        id: update_available_software
+        run: |
+          # install required Python packages in virtual environment
+          python -m venv venv
+          . venv/bin/activate
+          pip install -r scripts/available_software/requirements.txt
+
+          # copy current JSON data, so we can compare after updating it
+          cp docs/available_software/data/json_data.json docs/available_software/data/json_data.json.orig
+          cp docs/available_software/data/json_data_detail.json docs/available_software/data/json_data_detail.json.orig
+
+          python scripts/available_software/available_software.py
+          git status
+
+          # determine whether pull request should be opened:
+          # if JSON files in docs/available_software/data have been updated, then a PR should be opened
+          json_data_changed=$(python scripts/available_software/check_json_updates.py docs/available_software/data/json_data.json.orig docs/available_software/data/json_data.json)
+          json_data_detail_changed=$(python scripts/available_software/check_json_updates.py docs/available_software/data/json_data_detail.json.orig docs/available_software/data/json_data_detail.json)
+          if [[ ${json_data_changed} == "no changes" ]] && [[ ${json_data_detail_changed} == "no changes" ]]; then
+            echo "JSON files in docs/available_software/data have not been changed, no need to open PR"
+            echo "json_data_changed=no" >> $GITHUB_OUTPUT
+          else
+            echo "JSON files in docs/available_software/data have been changed, PR should be opened"
+            echo "json_data_changed=yes" >> $GITHUB_OUTPUT
+          fi
+
+          # remove original JSON files, or they'll end up in the PR being opened
+          rm -f docs/available_software/data/json_data.json.orig  docs/available_software/data/json_data_detail.json.orig
+
+      - name: create pull request
+        uses: peter-evans/create-pull-request@6d6857d36972b65feb161a90e484f2984215f83e # v6.0.5
+        if: steps.update_available_software.outputs.json_data_changed == 'yes'
+        with:
+          add-paths: docs/available_software
+          branch: update-software-overview
+          branch-suffix: timestamp
+          commit-message: update overview of available software
+          title: update overview of available software
+          body: ''
+          token: ${{ secrets.EESSIBOT_GITHUB_TOKEN }}
+          push-to-fork: EESSIbot/docs

docs/adding_software/debugging_failed_builds.md

@@ -238,6 +238,64 @@ After some time, this build fails while trying to build `Plumed`, and we can acc
 !!! Note
     While this might be faster than the easystack-based approach, this is _not_ how the bot builds. So why it _may_ reproduce the failure the bot encounters, it may not reproduce the bug _at all_ (no failure) or run into _different_ bugs. If you want to be sure, use the easystack-based approach.
 
+## Rebuilding software
+[Rebuilding software](opening_pr.md#rebuilding_software) requires an additional step at the beginning: the software first needs to be removed. We assume you've already [checked out the feature branch](#fetching-the-feature-branch). Then, you need to start the container with the additional `--fakeroot` argument, otherwise you will not be able to remove files from the `/cvmfs` prefix. Make sure to also include the `--save` argument, as we will need the tarball later on. E.g.
+```
+SINGULARITY_CACHEDIR=${eessi_common_dir}/container_cache ./eessi_container.sh --access rw --nvidia all --host-injections ${eessi_common_dir}/host_injections --save ${eessi_pr_dir} --fakeroot
+```
+Then, initialize the EESSI environment
+```
+source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
+```
+and get the diff file for the corresponding PR, e.g. for PR 123:
+```
+wget https://github.com/EESSI/software-layer/pull/123.diff
+```
+Finally, run the `EESSI-remove-software.sh` script
+```
+./EESSI-remove-software.sh`
+```
+
+This should remove any software specified in a [rebuild easystack](opening_pr.md#rebuilding_software) that got added in your current feature branch.
+
+Now, exit the container, paying attention to the instructions that are printed to resume later, e.g.:
+
+```
+Saved contents of tmp directory '/tmp/eessi.WZxeFUemH2' to tarball '/home/myuser/pr507/EESSI-1711538681.tgz' (to resume session add '--resume /home/myuser/pr507/EESSI-1711538681.tgz')
+```
+
+Now, continue with the original instructions to start the container (i.e. either [here](#starting-a-shell-in-the-eessi-container) or [with this alternate approach](#more-efficient-approach-for-multiplecontinued-debugging-sessions)) and make sure to add the `--resume` flag. This way, you are resuming from the tarball (i.e. with the software removed that has to be rebuilt), but in a new container in which you have regular (i.e. no root) permissions.
+
+## Running the test step
+If you are still in the prefix layer (i.e. after previously building something), exit it first:
+```
+$ exit
+logout
+Leaving Gentoo Prefix with exit status 0
+```
+Then, source the EESSI init script (again):
+```
+Apptainer> source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
+Environment set up to use EESSI (2023.06), have fun!
+{EESSI 2023.06} Apptainer>
+```
+
+!!! Note
+    If you are in a SLURM environment, make sure to run `for i in $(env | grep SLURM); do unset "${i%=*}"; done` to unset any SLURM environment variables. Failing to do so will cause `mpirun` to pick up on these and e.g. infer how many slots are available. If you run into errors of the form "There are not enough slots available in the system to satisfy the X slots that were requested by the application:", you probably forgot this step.
+
+Then, execute the `run_tests.sh` script. We are assuming you are still in the root of the `software-layer` repository that you cloned earlier:
+```
+./run_tests.sh
+```
+if all goes well, you should see (part of) the EESSI test suite being run by ReFrame, finishing with something like
+
+```
+[  PASSED  ] Ran X/Y test case(s) from Z check(s) (0 failure(s), 0 skipped, 0 aborted)
+```
+
+!!! Note
+    If you are running on a system with hyperthreading enabled, you may still run into the "There are not enough slots available in the system to satisfy the X slots that were requested by the application:" error from `mpirun`, because hardware threads are not considered to be slots by default by OpenMPIs `mpirun`. In this case, run with `OMPI_MCA_hwloc_base_use_hwthreads_as_cpus=1 ./run_tests.sh` (for OpenMPI 4.X) or `PRTE_MCA_rmaps_default_mapping_policy=:hwtcpus ./run_tests.sh` (for OpenMPI 5.X).
+
 ## Known causes of issues in EESSI
 
 ### The custom system prefix of the compatibility layer

docs/adding_software/opening_pr.md

@@ -73,6 +73,7 @@ git checkout -b example_branch
 ```shell
 echo '  - example-1.2.3-GCC-12.3.0.eb' >> easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml
 ```
+Note that the naming scheme is standardized and should be `eessi-<eessi_version>-eb-<eb_version>-<toolchain_version>.yml`. See the [official EasyBuild documentation on easystack files](https://docs.easybuild.io/easystack-files/) for more information on the syntax.
 
 4) Stage and commit the changes into your your branch with a sensible message
 
@@ -95,3 +96,26 @@ git push koala example_branch
 
    If all goes well, one or more bots :robot: should almost instantly create a comment in your pull request
    with an overview of how it is configured - you will need this information when providing build instructions.
+
+### Rebuilding software
+We typically do not rebuild software, since (strictly speaking) this breaks reproducibility for anyone using the software. However, there are certain situations in which it is difficult or impossible to avoid.
+
+To do a rebuild, you add the software you want to rebuild to a dedicated easystack file in the `rebuilds` directory. Use the following naming convention: `YYYYMMDD-eb-<EB_VERSION>-<APPLICATION_NAME>-<APPLICATION_VERSION>-<SHORT_DESCRIPTION>.yml`, where `YYYYMMDD` is the opening date of your PR. E.g. `2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml` was added in a PR on the 6th of May 2024 and used to rebuild CUDA-12.1.1 using EasyBuild 4.9.1 to resolve an issue with some runtime libraries missing from the initial CUDA 12.1.1 installation.
+
+At the top of your easystack file, please use comments to include a short description, and make sure to include any relevant links to related issues (e.g. from the GitHub repositories of EESSI, EasyBuild, or the software you are rebuilding).
+
+As an example, consider the full easystack file (`2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml`) used for the aforementioned CUDA rebuild: 
+
+```yaml
+# 2024.05.06
+# Original matching of files we could ship was not done correctly. We were
+# matching the basename for files (e.g., libcudart.so from libcudart.so.12)
+# rather than the name stub (libcudart)
+# See https://github.com/EESSI/software-layer/pull/559
+easyconfigs:
+  - CUDA-12.1.1.eb:
+        options:
+                accept-eula-for: CUDA
+```
+
+By separating rebuilds in dedicated files, we still maintain a complete software bill of materials: it is transparent what got rebuilt, for which reason, and when.