From 48150a483662ae2beb8438a8f59a8ac56c8cdd77 Mon Sep 17 00:00:00 2001 From: v-jizhang <66389669+buck-bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 09:16:02 -0700 Subject: [PATCH 1/5] Added cache for incremental build. --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 73d6353f673..16e79ca2965 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,6 +116,8 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY} export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY} eval $(aws ecr get-login --region us-east-1 --no-include-email) + - restore_cache: + key: v1-tutorial-{{ .Environment.CIRCLE_JOB }} - run: name: Build no_output_timeout: "20h" @@ -164,11 +166,23 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults fi set -x + # This also copies the cached build to docker. docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace" export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1' echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts + # Copy the last build from docker + docker cp "$id:/var/lib/jenkins/workspace/_build" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/docs" /home/circleci/project + + - save_cache: + # Save the build for incremental build. + key: v1-tutorial-{{ .Environment.CIRCLE_JOB }} + paths: + - /home/circleci/project/_build + - /home/circleci/project/docs + pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults environment: DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3" From bc77560e24a8a64b2f0f5c5c0872f29726233b94 Mon Sep 17 00:00:00 2001 From: v-jizhang <66389669+buck-bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 10:17:11 -0700 Subject: [PATCH 2/5] Test cache --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16e79ca2965..f3d7fd05d3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,7 +177,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults docker cp "$id:/var/lib/jenkins/workspace/docs" /home/circleci/project - save_cache: - # Save the build for incremental build. + # Save to cache for incremental build. key: v1-tutorial-{{ .Environment.CIRCLE_JOB }} paths: - /home/circleci/project/_build From b72ae9ca3fe1bd36f74198c0e8b9d2ff18765ed5 Mon Sep 17 00:00:00 2001 From: v-jizhang <66389669+buck-bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 11:52:31 -0700 Subject: [PATCH 3/5] Added more content to cache --- .circleci/config.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3d7fd05d3d..fc65ad39dec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -117,7 +117,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY} eval $(aws ecr get-login --region us-east-1 --no-include-email) - restore_cache: - key: v1-tutorial-{{ .Environment.CIRCLE_JOB }} + key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }} - run: name: Build no_output_timeout: "20h" @@ -175,13 +175,27 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults # Copy the last build from docker docker cp "$id:/var/lib/jenkins/workspace/_build" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/docs" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/advanced" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/beginner" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/intermediate" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/index" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/prototype" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/recipes" /home/circleci/project + docker cp "$id:/var/lib/jenkins/workspace/src" /home/circleci/project - save_cache: # Save to cache for incremental build. - key: v1-tutorial-{{ .Environment.CIRCLE_JOB }} + key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }} paths: - /home/circleci/project/_build - /home/circleci/project/docs + - /home/circleci/project/advanced + - /home/circleci/project/beginner + - /home/circleci/project/intermediate + - /home/circleci/project/index + - /home/circleci/project/prototype + - /home/circleci/project/recipes + - /home/circleci/project/src pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults environment: From e22ce3e0e59f16107eff98506529233063b59ae1 Mon Sep 17 00:00:00 2001 From: v-jizhang <66389669+buck-bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 12:42:55 -0700 Subject: [PATCH 4/5] Added more content to cache --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fc65ad39dec..ce40b097886 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -178,7 +178,6 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults docker cp "$id:/var/lib/jenkins/workspace/advanced" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/beginner" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/intermediate" /home/circleci/project - docker cp "$id:/var/lib/jenkins/workspace/index" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/prototype" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/recipes" /home/circleci/project docker cp "$id:/var/lib/jenkins/workspace/src" /home/circleci/project @@ -192,7 +191,6 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults - /home/circleci/project/advanced - /home/circleci/project/beginner - /home/circleci/project/intermediate - - /home/circleci/project/index - /home/circleci/project/prototype - /home/circleci/project/recipes - /home/circleci/project/src From 668b4577d53c0b32176a6f02c521779ab4692d7b Mon Sep 17 00:00:00 2001 From: v-jizhang <66389669+buck-bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 13:40:33 -0700 Subject: [PATCH 5/5] Test build cache --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce40b097886..90a477dd4b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,7 +183,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults docker cp "$id:/var/lib/jenkins/workspace/src" /home/circleci/project - save_cache: - # Save to cache for incremental build. + # Save to cache for incremental build key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }} paths: - /home/circleci/project/_build