From 64b5f1e5b32bf42e9aa1f14b74ef0c7955a5ecaa Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 7 Oct 2021 09:38:38 -0700 Subject: [PATCH 1/2] Prune image from GPU agent disk. It was currently only running on the CPU (ephemeral) agents. Switched to be run on the GPU agent building the image to prevent build errors caused by disk filling up. --- Jenkinsfile | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 99ad30c2..d78414e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,20 +20,6 @@ pipeline { } stages { - stage('Clean Images') { - steps { - sh '''#!/bin/bash - set -exo pipefail - # Remove images (dangling or not) created more than 120h (5 days ago) to prevent disk from filling up. - docker image prune --all --force --filter "until=120h" --filter "label=kaggle-lang=python" - # Remove any dangling images (no tags). - # All builds for the same branch uses the same tag. This means a subsequent build for the same branch - # will untag the previously built image which is safe to do. Builds for a single branch are performed - # serially. - docker image prune -f - ''' - } - } stage('Pre-build Packages from Source') { parallel { stage('torch') { @@ -123,6 +109,15 @@ pipeline { steps { sh '''#!/bin/bash set -exo pipefail + # Remove images (dangling or not) created more than 120h (5 days ago) to prevent the GPU agent disk from filling up. + # Note: CPU agents are ephemeral and do not need to have their disk cleaned up. + docker image prune --all --force --filter "until=120h" --filter "label=kaggle-lang=python" + # Remove any dangling images (no tags). + # All builds for the same branch uses the same tag. This means a subsequent build for the same branch + # will untag the previously built image which is safe to do. Builds for a single branch are performed + # serially. + docker image prune -f + ./build --gpu | ts ./push --gpu ${PRETEST_TAG} ''' From f44092447fc520b0da765d4c4d6b0b2e14135b8d Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 7 Oct 2021 09:45:21 -0700 Subject: [PATCH 2/2] Reduce prune window --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d78414e3..d884c767 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,9 +109,9 @@ pipeline { steps { sh '''#!/bin/bash set -exo pipefail - # Remove images (dangling or not) created more than 120h (5 days ago) to prevent the GPU agent disk from filling up. + # Remove images (dangling or not) created more than 72h (3 days ago) to prevent the GPU agent disk from filling up. # Note: CPU agents are ephemeral and do not need to have their disk cleaned up. - docker image prune --all --force --filter "until=120h" --filter "label=kaggle-lang=python" + docker image prune --all --force --filter "until=72h" --filter "label=kaggle-lang=python" # Remove any dangling images (no tags). # All builds for the same branch uses the same tag. This means a subsequent build for the same branch # will untag the previously built image which is safe to do. Builds for a single branch are performed