From f1419f7951954b9af01c095019932193a0e06ad5 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 11:32:20 -0400 Subject: [PATCH 1/6] Set lifecycle metadata annotations during publish --- scripts/cipublish | 67 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/scripts/cipublish b/scripts/cipublish index eebc7ee0..9aa91004 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -50,21 +50,82 @@ if [[ -z ${IMAGE_TAG} ]]; then exit 1 fi +function install_oras() { + # https://oras.land/docs/installation/ + VERSION="1.1.0" + curl -LO "https://github.com/oras-project/oras/releases/download/v${VERSION}/oras_${VERSION}_linux_amd64.tar.gz" + mkdir -p oras-install/ + tar -zxf oras_${VERSION}_*.tar.gz -C oras-install/ + sudo mv oras-install/oras /usr/local/bin/ + rm -rf oras_${VERSION}_*.tar.gz oras-install/ +} + +function deprecate_image() { + local full_image_name=$1 + local previous_full_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + deprecated_since=$(date --iso-8601=seconds) + echo "Deprecating previous image $full_image_name@$previous_full_image_digest" + oras attach \ + --artifact-type "application/vnd.microsoft.artifact.lifecycle" \ + --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$deprecated_since" \ + "$full_image_name@$previous_full_image_digest" +} + +function set_lineage() { + local full_image_name=$1 + local new_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + # if the annotation already exists, do not add it again + if oras discover $full_image_name@$new_image_digest -o json | grep -q "vnd.microsoft.artifact.lineage.rolling-tag=$IMAGE_TAG"; then + echo "Lineage annotation for $IMAGE_TAG already exists, skip adding lineage annotation." + else + echo "Adding $full_image_name@$new_image_digest to lineage $IMAGE_TAG" + oras attach \ + --artifact-type "application/vnd.microsoft.artifact.lineage" \ + --annotation "vnd.microsoft.artifact.lineage.rolling-tag=$IMAGE_TAG" \ + "$full_image_name@$new_image_digest" + fi +} + function publish_image() { local local_image=$1 local published_image=$2 local full_image_name="${ACR_NAME}.azurecr.io/${published_image}:${IMAGE_TAG}" + local local_image_digest=$(docker inspect --format='{{.RepoDigests}}' "${local_image}" | cut -d'@' -f2) + local remote_image_digest=$(az acr repository show-manifests --name $ACR_NAME --repository $published_image --query "[?tags[? @ == '$IMAGE_TAG']].digest" -o tsv) + + if ["$local_image_digest" != "$remote_image_digest"]; then + # Image rolling tag exists in the registry, update the end-of-life + # annotation for the existing image. + deprecate_image $full_image_name + else + echo "No changes, licycle metadata annotation will not be attached." + fi + echo "Publishing ${local_image} to ${full_image_name}" docker tag "${local_image}" "${full_image_name}" docker push "${full_image_name}" + set_lineage $full_image_name } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - # Publish images + if ! command -v oras &> /dev/null + then + install_oras + fi - publish_image "pc-apis-stac" "public/planetary-computer-apis/stac" - publish_image "pc-apis-tiler" "public/planetary-computer-apis/tiler" + # only _tagged_ releases will be synced from pcccr to MAR + case $IMAGE_TAG in + "latest") + image_prefix="private" + ;; + *) + image_prefix="public" + ;; + esac + # Publish images + publish_image "pc-apis-stac" "$image_prefix/planetary-computer-apis/stac" + publish_image "pc-apis-tiler" "$image_prefix/planetary-computer-apis/tiler" fi From ede639cc483da8a0c7e946cdd977922bcc59dba2 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 11:50:11 -0400 Subject: [PATCH 2/6] fix oras output parse --- scripts/cipublish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cipublish b/scripts/cipublish index 9aa91004..f6cde72d 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -75,8 +75,8 @@ function set_lineage() { local full_image_name=$1 local new_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) # if the annotation already exists, do not add it again - if oras discover $full_image_name@$new_image_digest -o json | grep -q "vnd.microsoft.artifact.lineage.rolling-tag=$IMAGE_TAG"; then - echo "Lineage annotation for $IMAGE_TAG already exists, skip adding lineage annotation." + if oras discover $full_image_name@$new_image_digest -o json | jq '.manifests[].annotations."vnd.microsoft.artifact.lineage.rolling-tag"' | grep -q $IMAGE_TAG; then + echo "Lineage annotation for $IMAGE_TAG already exists, skip." else echo "Adding $full_image_name@$new_image_digest to lineage $IMAGE_TAG" oras attach \ From 37e7a7c8ffaf5b08a15f4fda723aa29c477e983e Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 11:53:19 -0400 Subject: [PATCH 3/6] Documentation update --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a030ff0..c4135db2 100644 --- a/README.md +++ b/README.md @@ -122,11 +122,13 @@ This project publishes images and helm charts, which are used in the deployment ### Images -Images following images are hosted in the [Microsoft Container Registry](https://github.com/microsoft/ContainerRegistry): +The following images are hosted in the [Microsoft Container Registry](https://github.com/microsoft/ContainerRegistry): - `mcr.microsoft.com/planetary-computer-apis/stac` - `mcr.microsoft.com/planetary-computer-apis/tiler` +Only tagged builds will be published to MCR, untagged builds will only be published to the internal ACR `pcccr`. + ### Charts See the [Helm chart repository](https://microsoft.github.io/planetary-computer-apis) published to GitHub pages for the published charts. From 019faf11b6b650ca3521896ad367d3cdb92cf2f6 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 11:57:15 -0400 Subject: [PATCH 4/6] fix wrong conditional --- scripts/cipublish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cipublish b/scripts/cipublish index f6cde72d..3c271f83 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -92,9 +92,9 @@ function publish_image() { local full_image_name="${ACR_NAME}.azurecr.io/${published_image}:${IMAGE_TAG}" local local_image_digest=$(docker inspect --format='{{.RepoDigests}}' "${local_image}" | cut -d'@' -f2) - local remote_image_digest=$(az acr repository show-manifests --name $ACR_NAME --repository $published_image --query "[?tags[? @ == '$IMAGE_TAG']].digest" -o tsv) + local remote_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) - if ["$local_image_digest" != "$remote_image_digest"]; then + if [ "$local_image_digest" != "$remote_image_digest" ]; then # Image rolling tag exists in the registry, update the end-of-life # annotation for the existing image. deprecate_image $full_image_name From 68364180b0f59b79e0acd568a199b79ede418888 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 12:23:10 -0400 Subject: [PATCH 5/6] simplify script --- scripts/cipublish | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/cipublish b/scripts/cipublish index 3c271f83..adaa8e47 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -61,28 +61,26 @@ function install_oras() { } function deprecate_image() { - local full_image_name=$1 - local previous_full_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) - deprecated_since=$(date --iso-8601=seconds) - echo "Deprecating previous image $full_image_name@$previous_full_image_digest" + local full_image_name_with_digest=$1 + deprecated_since=$(date --utc --iso-8601=seconds) + echo "Deprecating previous image $full_image_name_with_digest" oras attach \ --artifact-type "application/vnd.microsoft.artifact.lifecycle" \ --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$deprecated_since" \ - "$full_image_name@$previous_full_image_digest" + "$full_image_name_with_digest" } function set_lineage() { - local full_image_name=$1 - local new_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + local full_image_name_with_digest=$1 # if the annotation already exists, do not add it again - if oras discover $full_image_name@$new_image_digest -o json | jq '.manifests[].annotations."vnd.microsoft.artifact.lineage.rolling-tag"' | grep -q $IMAGE_TAG; then + if oras discover $full_image_name_with_digest -o json | jq '.manifests[].annotations."vnd.microsoft.artifact.lineage.rolling-tag"' | grep -q $IMAGE_TAG; then echo "Lineage annotation for $IMAGE_TAG already exists, skip." else - echo "Adding $full_image_name@$new_image_digest to lineage $IMAGE_TAG" + echo "Adding $full_image_name_with_digest to lineage $IMAGE_TAG" oras attach \ --artifact-type "application/vnd.microsoft.artifact.lineage" \ --annotation "vnd.microsoft.artifact.lineage.rolling-tag=$IMAGE_TAG" \ - "$full_image_name@$new_image_digest" + "$full_image_name_with_digest" fi } @@ -97,7 +95,7 @@ function publish_image() { if [ "$local_image_digest" != "$remote_image_digest" ]; then # Image rolling tag exists in the registry, update the end-of-life # annotation for the existing image. - deprecate_image $full_image_name + deprecate_image "$full_image_name@$remote_image_digest" else echo "No changes, licycle metadata annotation will not be attached." fi @@ -105,7 +103,8 @@ function publish_image() { echo "Publishing ${local_image} to ${full_image_name}" docker tag "${local_image}" "${full_image_name}" docker push "${full_image_name}" - set_lineage $full_image_name + local remote_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + set_lineage "$full_image_name@$remote_image_digest" } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then From db4cb333f8a10fea665106b1977b6ed5fbfd4a47 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 29 Mar 2024 12:39:30 -0400 Subject: [PATCH 6/6] better script --- scripts/cipublish | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/cipublish b/scripts/cipublish index adaa8e47..062c396f 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -63,11 +63,15 @@ function install_oras() { function deprecate_image() { local full_image_name_with_digest=$1 deprecated_since=$(date --utc --iso-8601=seconds) - echo "Deprecating previous image $full_image_name_with_digest" - oras attach \ - --artifact-type "application/vnd.microsoft.artifact.lifecycle" \ - --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$deprecated_since" \ - "$full_image_name_with_digest" + if oras discover $full_image_name_with_digest -o json | jq '.manifests[].annotations' | grep -q "vnd.microsoft.lifecycle.end-of-life.date" ; then + echo "Lifecycle metadata annotation for $full_image_name_with_digest already exists, skip." + else + echo "Deprecating previous image $full_image_name_with_digest" + oras attach \ + --artifact-type "application/vnd.microsoft.artifact.lifecycle" \ + --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$deprecated_since" \ + "$full_image_name_with_digest" + fi } function set_lineage() { @@ -92,7 +96,9 @@ function publish_image() { local local_image_digest=$(docker inspect --format='{{.RepoDigests}}' "${local_image}" | cut -d'@' -f2) local remote_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) - if [ "$local_image_digest" != "$remote_image_digest" ]; then + if [ -z "$remote_image_digest" ]; then + echo "No remote image found, will publish a new image." + elif [ "$local_image_digest" != "$remote_image_digest" ]; then # Image rolling tag exists in the registry, update the end-of-life # annotation for the existing image. deprecate_image "$full_image_name@$remote_image_digest" @@ -116,7 +122,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then # only _tagged_ releases will be synced from pcccr to MAR case $IMAGE_TAG in - "latest") + *latest*) image_prefix="private" ;; *)