diff --git a/tests_integ/conftest.py b/tests_integ/conftest.py index f83f0e299..61c2bf9a1 100644 --- a/tests_integ/conftest.py +++ b/tests_integ/conftest.py @@ -1,5 +1,17 @@ +import json +import logging +import os + +import boto3 import pytest +logger = logging.getLogger(__name__) + + +def pytest_sessionstart(session): + _load_api_keys_from_secrets_manager() + + ## Data @@ -28,3 +40,43 @@ async def alist(items): return [item async for item in items] return alist + + +## Models + + +def _load_api_keys_from_secrets_manager(): + """Load API keys as environment variables from AWS Secrets Manager.""" + session = boto3.session.Session() + client = session.client(service_name="secretsmanager") + if "STRANDS_TEST_API_KEYS_SECRET_NAME" in os.environ: + try: + secret_name = os.getenv("STRANDS_TEST_API_KEYS_SECRET_NAME") + response = client.get_secret_value(SecretId=secret_name) + + if "SecretString" in response: + secret = json.loads(response["SecretString"]) + for key, value in secret.items(): + os.environ[f"{key.upper()}_API_KEY"] = str(value) + + except Exception as e: + logger.warning("Error retrieving secret", e) + + """ + Validate that required environment variables are set when running in GitHub Actions. + This prevents tests from being unintentionally skipped due to missing credentials. + """ + if os.environ.get("GITHUB_ACTIONS") != "true": + logger.warning("Tests running outside GitHub Actions, skipping required provider validation") + return + + required_providers = { + "ANTHROPIC_API_KEY", + "COHERE_API_KEY", + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "WRITER_API_KEY", + } + for provider in required_providers: + if provider not in os.environ or not os.environ[provider]: + raise ValueError(f"Missing required environment variables for {provider}") diff --git a/tests_integ/models/providers.py b/tests_integ/models/providers.py index 543f58480..d2ac148d3 100644 --- a/tests_integ/models/providers.py +++ b/tests_integ/models/providers.py @@ -72,11 +72,11 @@ def __init__(self): bedrock = ProviderInfo(id="bedrock", factory=lambda: BedrockModel()) cohere = ProviderInfo( id="cohere", - environment_variable="CO_API_KEY", + environment_variable="COHERE_API_KEY", factory=lambda: OpenAIModel( client_args={ "base_url": "https://api.cohere.com/compatibility/v1", - "api_key": os.getenv("CO_API_KEY"), + "api_key": os.getenv("COHERE_API_KEY"), }, model_id="command-a-03-2025", params={"stream_options": None}, diff --git a/tests_integ/models/conformance.py b/tests_integ/models/test_conformance.py similarity index 81% rename from tests_integ/models/conformance.py rename to tests_integ/models/test_conformance.py index 262e41e42..d9875bc07 100644 --- a/tests_integ/models/conformance.py +++ b/tests_integ/models/test_conformance.py @@ -1,6 +1,6 @@ import pytest -from strands.types.models import Model +from strands.models import Model from tests_integ.models.providers import ProviderInfo, all_providers @@ -9,7 +9,7 @@ def get_models(): pytest.param( provider_info, id=provider_info.id, # Adds the provider name to the test name - marks=[provider_info.mark], # ignores tests that don't have the requirements + marks=provider_info.mark, # ignores tests that don't have the requirements ) for provider_info in all_providers ] diff --git a/tests_integ/models/test_model_anthropic.py b/tests_integ/models/test_model_anthropic.py index 2ee5e7f23..62a95d06d 100644 --- a/tests_integ/models/test_model_anthropic.py +++ b/tests_integ/models/test_model_anthropic.py @@ -6,10 +6,17 @@ import strands from strands import Agent from strands.models.anthropic import AnthropicModel -from tests_integ.models import providers -# these tests only run if we have the anthropic api key -pytestmark = providers.anthropic.mark +""" +These tests only run if we have the anthropic api key + +Because of infrequent burst usage, Anthropic tests are unreliable, failing tests with 529s. +{'type': 'error', 'error': {'details': None, 'type': 'overloaded_error', 'message': 'Overloaded'}} +https://docs.anthropic.com/en/api/errors#http-errors +""" +pytestmark = pytest.skip( + "Because of infrequent burst usage, Anthropic tests are unreliable, failing with 529s", allow_module_level=True +) @pytest.fixture diff --git a/tests_integ/models/test_model_cohere.py b/tests_integ/models/test_model_cohere.py index 996b0f326..33fb1a8c6 100644 --- a/tests_integ/models/test_model_cohere.py +++ b/tests_integ/models/test_model_cohere.py @@ -16,7 +16,7 @@ def model(): return OpenAIModel( client_args={ "base_url": "https://api.cohere.com/compatibility/v1", - "api_key": os.getenv("CO_API_KEY"), + "api_key": os.getenv("COHERE_API_KEY"), }, model_id="command-a-03-2025", params={"stream_options": None},