chore(CI): Build test vectors from source (#724)

lucasmcdonald3 · web-flow · commit 899af22c25fd · 2024-11-21T10:54:02.000-08:00
diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml
@@ -21,6 +21,8 @@ phases:
       python: 3.11
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers
       - tox
diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml
@@ -22,6 +22,8 @@ phases:
       python: 3.11
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Fetch test vectors from Dafny ESDK's most recent run
       # (Assuming the first result is most recent; seems to be correct...)
       - |
diff --git a/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.11
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download "golden manifest"
       - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip
       - unzip python-2.3.0.zip -d python-2.3.0
diff --git a/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml
@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    TOXENV: "py311-full_decrypt-mpl"
+    TOXENV: "py311-full_decrypt"
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
       arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.11
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download previously generated vectors
       - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip
       - unzip 311_keyrings.zip
diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.11
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download previously generated vectors
       - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip
       - unzip 311_masterkey.zip
diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml
@@ -19,6 +19,8 @@ phases:
       python: 3.11
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Fetch ESDK .NET v4.0.1 Test Vectors
       - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip
       - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip
diff --git a/codebuild/py311/encrypt_keyrings.yml b/codebuild/py311/encrypt_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.11
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers
       - |
diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.11
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers
       - |
diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml
@@ -23,6 +23,8 @@ phases:
       python: 3.12
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
       - pyenv install --skip-existing 3.12.0
       - pyenv local 3.12.0
diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml
@@ -22,6 +22,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Fetch test vectors from Dafny ESDK's most recent run
       # (Assuming the first result is most recent; seems to be correct...)
       - |
diff --git a/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download "golden manifest"
       - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip
       - unzip python-2.3.0.zip -d python-2.3.0
diff --git a/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml
@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    TOXENV: "py312-full_decrypt-mpl"
+    TOXENV: "py312-full_decrypt"
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
       arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
diff --git a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download previously generated vectors
       # This manifest has coverage for both HKeyring and required encryption context CMM
       - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip
diff --git a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml
@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    TOXENV: "py312-full_decrypt-mpl"
+    TOXENV: "py312-full_decrypt"
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
       arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f
     AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
diff --git a/codebuild/py312/decrypt_keyrings_with_keyrings.yml b/codebuild/py312/decrypt_keyrings_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download previously generated vectors
       - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip
       - unzip 312_keyrings.zip
diff --git a/codebuild/py312/decrypt_masterkey_with_keyrings.yml b/codebuild/py312/decrypt_masterkey_with_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Download previously generated vectors
       - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip
       - unzip 312_masterkey.zip
diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml
@@ -19,6 +19,8 @@ phases:
       python: 3.12
   pre_build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       # Fetch ESDK .NET v4.0.1 Test Vectors
       - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip
       - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip
diff --git a/codebuild/py312/encrypt_keyrings.yml b/codebuild/py312/encrypt_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers
       - |
diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers
       - |
diff --git a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml
@@ -18,6 +18,8 @@ phases:
       python: 3.12
   build:
     commands:
+      # Build Python MPL TestVector runner from source      
+      - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
       - pip install "tox < 4.0"
       - cd test_vector_handlers/test/aws-crypto-tools-test-vector-framework
       # Checkout WIP branch with manifest containing HKeyring and required EC CMM test cases
diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt
@@ -1 +1 @@
-aws-cryptography-internal-mpl-testvectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@python-main-with-dafny-code#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python
+aws-cryptographic-material-providers==1.7.4
diff --git a/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh
@@ -0,0 +1,24 @@
+# Builds the Python MPL TestVector runner from source.
+# This package is used by the ESDK-Python test vectors for testing with the MPL.
+
+# This script is intended to be used by ESDK-Python's integration tests.
+# You may need or want to make local changes to get this work on your machine.
+
+# Change to the directory of the script
+cd "$(dirname "$0")"
+
+# Get MPL version specified in requirements.txt
+export mplVersion=$(grep 'aws-cryptographic-material-providers==' ../requirements_mpl.txt | sed -E 's/.*==(.+)/\1/')
+
+# Clone MPL repo to get test vectors runner source code and the Dafny version to use
+git clone --branch v$mplVersion --recurse-submodules https://github.com/aws/aws-cryptographic-material-providers-library.git
+
+# Download Dafny to build the test vector runner; get Dafny version from ESDK's project.properties file
+export dafnyVersion=$(grep '^dafnyVersion=' aws-cryptographic-material-providers-library/project.properties | cut -d '=' -f 2)
+curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip  -L -o dafny.zip
+unzip -qq dafny.zip && rm dafny.zip
+export PATH="$PWD/dafny:$PATH"
+
+# Build MPL test vector runner from source
+cd aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/
+make transpile_python
diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini
@@ -51,6 +51,9 @@ deps =
     -rtest/requirements.txt
     # Install the MPL requirements if the `-mpl` suffix is present
     mpl: -rrequirements_mpl.txt
+    # This is the filepath used in ESDK-Python's integration tests.
+    # You may need to change this to run this project locally.
+    mpl: scripts/aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/runtimes/python
     ..
 commands = 
     awses_local: {[testenv:base-command]commands}
