diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7a7c6087d5..bc4a5324a1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,4 +11,12 @@ RUN mkdir -p ${HOME} && \ groupadd --gid ${GID} vscode && \ useradd --uid ${UID} --gid ${GID} --home ${HOME} vscode && \ chown -R ${UID}:${GID} /home/vscode + +# Move pyenv installation +ENV PYENV_ROOT="${HOME}/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:${PATH}" +RUN mv /root/.pyenv /home/vscode/.pyenv && \ + chown -R vscode:vscode /home/vscode/.pyenv + +# Set user USER ${UID}:${GID} diff --git a/newrelic/config.py b/newrelic/config.py index 6816c43b5b..5c4c52464f 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2448,11 +2448,6 @@ def _process_module_builtin_defaults(): "newrelic.hooks.messagebroker_kafkapython", "instrument_kafka_heartbeat", ) - _process_module_definition( - "kafka.consumer.group", - "newrelic.hooks.messagebroker_kafkapython", - "instrument_kafka_consumer_group", - ) _process_module_definition( "logging", diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index c99293a63f..618c3a32a6 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -55,6 +55,27 @@ def extractor_string(*args, **kwargs): return extractor_list +def bedrock_error_attributes(exception, request_args, client, extractor): + response = getattr(exception, "response", None) + if not response: + return {} + + request_body = request_args.get("body", "") + error_attributes = extractor(request_body)[1] + + error_attributes.update({ + "request_id": response.get("ResponseMetadata", {}).get("RequestId", ""), + "api_key_last_four_digits": client._request_signer._credentials.access_key[-4:], + "request.model": request_args.get("modelId", ""), + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": response.get("ResponseMetadata", "").get("HTTPStatusCode", ""), + "error.message": response.get("Error", "").get("Message", ""), + "error.code": response.get("Error", "").get("Code", ""), + }) + return error_attributes + + def create_chat_completion_message_event( transaction, app_name, @@ -95,99 +116,137 @@ def create_chat_completion_message_event( transaction.record_ml_event("LlmChatCompletionMessage", chat_completion_message_dict) -def extract_bedrock_titan_text_model(request_body, response_body): - response_body = json.loads(response_body) +def extract_bedrock_titan_text_model(request_body, response_body=None): request_body = json.loads(request_body) - - input_tokens = response_body["inputTextTokenCount"] - completion_tokens = sum(result["tokenCount"] for result in response_body.get("results", [])) - total_tokens = input_tokens + completion_tokens + if response_body: + response_body = json.loads(response_body) request_config = request_body.get("textGenerationConfig", {}) - message_list = [{"role": "user", "content": request_body.get("inputText", "")}] - message_list.extend( - {"role": "assistant", "content": result["outputText"]} for result in response_body.get("results", []) - ) chat_completion_summary_dict = { "request.max_tokens": request_config.get("maxTokenCount", ""), "request.temperature": request_config.get("temperature", ""), - "response.choices.finish_reason": response_body["results"][0]["completionReason"], - "response.usage.completion_tokens": completion_tokens, - "response.usage.prompt_tokens": input_tokens, - "response.usage.total_tokens": total_tokens, - "response.number_of_messages": len(message_list), } + + if response_body: + input_tokens = response_body["inputTextTokenCount"] + completion_tokens = sum(result["tokenCount"] for result in response_body.get("results", [])) + total_tokens = input_tokens + completion_tokens + + message_list = [{"role": "user", "content": request_body.get("inputText", "")}] + message_list.extend( + {"role": "assistant", "content": result["outputText"]} for result in response_body.get("results", []) + ) + + chat_completion_summary_dict.update({ + "response.choices.finish_reason": response_body["results"][0]["completionReason"], + "response.usage.completion_tokens": completion_tokens, + "response.usage.prompt_tokens": input_tokens, + "response.usage.total_tokens": total_tokens, + "response.number_of_messages": len(message_list), + }) + else: + message_list = [] + return message_list, chat_completion_summary_dict -def extract_bedrock_titan_embedding_model(request_body, response_body): - response_body = json.loads(response_body) +def extract_bedrock_titan_embedding_model(request_body, response_body=None): + if not response_body: + return [], {} # No extracted information necessary for embedding + request_body = json.loads(request_body) + response_body = json.loads(response_body) - input_tokens = response_body["inputTextTokenCount"] + input_tokens = response_body.get("inputTextTokenCount", None) embedding_dict = { "input": request_body.get("inputText", ""), "response.usage.prompt_tokens": input_tokens, "response.usage.total_tokens": input_tokens, } - return embedding_dict + return [], embedding_dict -def extract_bedrock_ai21_j2_model(request_body, response_body): - response_body = json.loads(response_body) +def extract_bedrock_ai21_j2_model(request_body, response_body=None): request_body = json.loads(request_body) - - message_list = [{"role": "user", "content": request_body.get("prompt", "")}] - message_list.extend( - {"role": "assistant", "content": result["data"]["text"]} for result in response_body.get("completions", []) - ) + if response_body: + response_body = json.loads(response_body) chat_completion_summary_dict = { "request.max_tokens": request_body.get("maxTokens", ""), "request.temperature": request_body.get("temperature", ""), - "response.choices.finish_reason": response_body["completions"][0]["finishReason"]["reason"], - "response.number_of_messages": len(message_list), - "response_id": str(response_body.get("id", "")), } + + if response_body: + message_list = [{"role": "user", "content": request_body.get("prompt", "")}] + message_list.extend( + {"role": "assistant", "content": result["data"]["text"]} for result in response_body.get("completions", []) + ) + + chat_completion_summary_dict.update({ + "response.choices.finish_reason": response_body["completions"][0]["finishReason"]["reason"], + "response.number_of_messages": len(message_list), + "response_id": str(response_body.get("id", "")), + }) + else: + message_list = [] + return message_list, chat_completion_summary_dict -def extract_bedrock_claude_model(request_body, response_body): - response_body = json.loads(response_body) +def extract_bedrock_claude_model(request_body, response_body=None): request_body = json.loads(request_body) - - message_list = [ - {"role": "user", "content": request_body.get("prompt", "")}, - {"role": "assistant", "content": response_body.get("completion", "")}, - ] + if response_body: + response_body = json.loads(response_body) chat_completion_summary_dict = { "request.max_tokens": request_body.get("max_tokens_to_sample", ""), "request.temperature": request_body.get("temperature", ""), - "response.choices.finish_reason": response_body.get("stop_reason", ""), - "response.number_of_messages": len(message_list), } + + if response_body: + message_list = [ + {"role": "user", "content": request_body.get("prompt", "")}, + {"role": "assistant", "content": response_body.get("completion", "")}, + ] + + chat_completion_summary_dict.update({ + "response.choices.finish_reason": response_body.get("stop_reason", ""), + "response.number_of_messages": len(message_list), + }) + else: + message_list = [] + return message_list, chat_completion_summary_dict -def extract_bedrock_cohere_model(request_body, response_body): - response_body = json.loads(response_body) +def extract_bedrock_cohere_model(request_body, response_body=None): request_body = json.loads(request_body) - - message_list = [{"role": "user", "content": request_body.get("prompt", "")}] - message_list.extend( - {"role": "assistant", "content": result["text"]} for result in response_body.get("generations", []) - ) + if response_body: + response_body = json.loads(response_body) chat_completion_summary_dict = { "request.max_tokens": request_body.get("max_tokens", ""), "request.temperature": request_body.get("temperature", ""), - "response.choices.finish_reason": response_body["generations"][0]["finish_reason"], - "response.number_of_messages": len(message_list), - "response_id": str(response_body.get("id", "")), } + + if response_body: + message_list = [{"role": "user", "content": request_body.get("prompt", "")}] + message_list.extend( + {"role": "assistant", "content": result["text"]} for result in response_body.get("generations", []) + ) + + chat_completion_summary_dict.update({ + "request.max_tokens": request_body.get("max_tokens", ""), + "request.temperature": request_body.get("temperature", ""), + "response.choices.finish_reason": response_body["generations"][0]["finish_reason"], + "response.number_of_messages": len(message_list), + "response_id": str(response_body.get("id", "")), + }) + else: + message_list = [] + return message_list, chat_completion_summary_dict @@ -215,17 +274,10 @@ def wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): request_body = request_body.read() kwargs["body"] = request_body - ft_name = callable_name(wrapped) - with FunctionTrace(ft_name) as ft: - response = wrapped(*args, **kwargs) - - if not response: - return response - # Determine model to be used with extractor model = kwargs.get("modelId") if not model: - return response + return wrapped(*args, **kwargs) # Determine extractor by model type for extractor_name, extractor in MODEL_EXTRACTORS: @@ -241,7 +293,24 @@ def wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): model, ) UNSUPPORTED_MODEL_WARNING_SENT = True + + extractor = lambda *args: ([], {}) # Empty extractor that returns nothing + + ft_name = callable_name(wrapped) + with FunctionTrace(ft_name) as ft: + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + try: + error_attributes = extractor(request_body) + error_attributes = bedrock_error_attributes(exc, kwargs, instance, extractor) + ft.notice_error( + attributes=error_attributes, + ) + finally: + raise + if not response: return response # Read and replace response streaming bodies @@ -265,7 +334,7 @@ def handle_embedding_event(client, transaction, extractor, model, response_body, request_id = response_headers.get("x-amzn-requestid", "") settings = transaction.settings if transaction.settings is not None else global_settings() - embedding_dict = extractor(request_body, response_body) + _, embedding_dict = extractor(request_body, response_body) embedding_dict.update({ "vendor": "bedrock", diff --git a/tests/external_botocore/_mock_external_bedrock_server.py b/tests/external_botocore/_mock_external_bedrock_server.py index c9149e3f1b..da5ff68dd9 100644 --- a/tests/external_botocore/_mock_external_bedrock_server.py +++ b/tests/external_botocore/_mock_external_bedrock_server.py @@ -29,43 +29,9 @@ # 3) This app runs on a separate thread meaning it won't block the test app. RESPONSES = { - "amazon.titan-text-express-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ - {"content-type": "application/json", "x-amzn-requestid": "660d4de9-6804-460e-8556-4ab2a019d1e3"}, - { - "inputTextTokenCount": 12, - "results": [ - { - "tokenCount": 55, - "outputText": "\nUse the formula,\n\u00b0C = (\u00b0F - 32) x 5/9\n= 212 x 5/9\n= 100 degrees Celsius\n212 degrees Fahrenheit is 100 degrees Celsius.", - "completionReason": "FINISH", - } - ], - }, - ], - "anthropic.claude-instant-v1::Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:": [ - {"content-type": "application/json", "x-amzn-requestid": "f354b9a7-9eac-4f50-a8d7-7d5d23566176"}, - { - "completion": " Here are the step-by-step workings:\n1) 212 degrees Fahrenheit \n2) To convert to Celsius, use the formula: C = (F - 32) * 5/9\n3) Plug in the values: C = (212 - 32) * 5/9 = 100 * 5/9 = 100 degrees Celsius\n\nSo, 212 degrees Fahrenheit converted to Celsius is 100 degrees Celsius.", - "stop_reason": "stop_sequence", - "stop": "\n\nHuman:", - }, - ], - "cohere.command-text-v14::What is 212 degrees Fahrenheit converted to Celsius?": [ - {"content-type": "application/json", "x-amzn-requestid": "c5188fb5-dc58-4cbe-948d-af173c69ce0d"}, - { - "generations": [ - { - "finish_reason": "MAX_TOKENS", - "id": "0730f5c0-9a49-4f35-af94-cf8f77327740", - "text": " To convert 212 degrees Fahrenheit to Celsius, we can use the conversion factor that Celsius is equal to (Fahrenheit - 32) x 5/9. \\n\\nApplying this formula, we have:\\n212°F = (212°F - 32) x 5/9\\n= (180) x 5/9\\n= 100°C.\\n\\nTherefore, 212 degrees F", - } - ], - "id": "a9cc8ce6-50b6-40b6-bf77-cf24561d8de7", - "prompt": "What is 212 degrees Fahrenheit converted to Celsius?", - }, - ], "ai21.j2-mid-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ - {"content-type": "application/json", "x-amzn-requestid": "3bf1bb6b-b6f0-4901-85a1-2fa0e814440e"}, + {"Content-Type": "application/json", "x-amzn-RequestId": "c863d9fc-888b-421c-a175-ac5256baec62"}, + 200, { "id": 1234, "prompt": { @@ -73,7 +39,7 @@ "tokens": [ { "generatedToken": { - "token": "\u2581What\u2581is", + "token": "▁What▁is", "logprob": -7.446773529052734, "raw_logprob": -7.446773529052734, }, @@ -82,7 +48,7 @@ }, { "generatedToken": { - "token": "\u2581", + "token": "▁", "logprob": -3.8046724796295166, "raw_logprob": -3.8046724796295166, }, @@ -100,7 +66,7 @@ }, { "generatedToken": { - "token": "\u2581degrees\u2581Fahrenheit", + "token": "▁degrees▁Fahrenheit", "logprob": -7.953181743621826, "raw_logprob": -7.953181743621826, }, @@ -109,7 +75,7 @@ }, { "generatedToken": { - "token": "\u2581converted\u2581to", + "token": "▁converted▁to", "logprob": -6.168096542358398, "raw_logprob": -6.168096542358398, }, @@ -118,7 +84,7 @@ }, { "generatedToken": { - "token": "\u2581Celsius", + "token": "▁Celsius", "logprob": -0.09790332615375519, "raw_logprob": -0.09790332615375519, }, @@ -152,7 +118,7 @@ }, { "generatedToken": { - "token": "\u2581", + "token": "▁", "logprob": -0.03473362699151039, "raw_logprob": -0.11261807382106781, }, @@ -170,7 +136,7 @@ }, { "generatedToken": { - "token": "\u2581degrees\u2581Fahrenheit", + "token": "▁degrees▁Fahrenheit", "logprob": -0.003579758107662201, "raw_logprob": -0.03144374489784241, }, @@ -179,7 +145,7 @@ }, { "generatedToken": { - "token": "\u2581is\u2581equal\u2581to", + "token": "▁is▁equal▁to", "logprob": -0.0027733694296330214, "raw_logprob": -0.027207009494304657, }, @@ -188,7 +154,7 @@ }, { "generatedToken": { - "token": "\u2581", + "token": "▁", "logprob": -0.0003392120997887105, "raw_logprob": -0.005458095110952854, }, @@ -206,7 +172,7 @@ }, { "generatedToken": { - "token": "\u2581degrees\u2581Celsius", + "token": "▁degrees▁Celsius", "logprob": -0.31207239627838135, "raw_logprob": -0.402545303106308, }, @@ -238,8 +204,9 @@ ], }, ], - "amazon.titan-embed-text-v1::This is an embedding test.": [ - {"content-type": "application/json", "x-amzn-requestid": "75f1d3fe-6cde-4cf5-bdaf-7101f746ccfe"}, + "amazon.titan-embed-g1-text-02::This is an embedding test.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "b10ac895-eae3-4f07-b926-10b2866c55ed"}, + 200, { "embedding": [ -0.14160156, @@ -1782,8 +1749,9 @@ "inputTextTokenCount": 6, }, ], - "amazon.titan-embed-g1-text-02::This is an embedding test.": [ - {"content-type": "application/json", "x-amzn-requestid": "f7e78265-6b7c-4b3a-b750-0c1d00347258"}, + "amazon.titan-embed-text-v1::This is an embedding test.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "11233989-07e8-4ecb-9ba6-79601ba6d8cc"}, + 200, { "embedding": [ -0.14160156, @@ -3326,6 +3294,107 @@ "inputTextTokenCount": 6, }, ], + "amazon.titan-text-express-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "03524118-8d77-430f-9e08-63b5c03a40cf"}, + 200, + { + "inputTextTokenCount": 12, + "results": [ + { + "tokenCount": 75, + "outputText": "\nUse the formula,\n°C = (°F - 32) x 5/9\n= 212 x 5/9\n= 100 degrees Celsius\n212 degrees Fahrenheit is 100 degrees Celsius.", + "completionReason": "FINISH", + } + ], + }, + ], + "anthropic.claude-instant-v1::Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "7b0b37c6-85fb-4664-8f5b-361ca7b1aa18"}, + 200, + { + "completion": " Okay, here are the conversion steps:\n212 degrees Fahrenheit\n- Subtract 32 from 212 to get 180 (to convert from Fahrenheit to Celsius scale)\n- Multiply by 5/9 (because the formula is °C = (°F - 32) × 5/9)\n- 180 × 5/9 = 100\n\nSo 212 degrees Fahrenheit converted to Celsius is 100 degrees Celsius.", + "stop_reason": "stop_sequence", + "stop": "\n\nHuman:", + }, + ], + "cohere.command-text-v14::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "e77422c8-fbbf-4e17-afeb-c758425c9f97"}, + 200, + { + "generations": [ + { + "finish_reason": "MAX_TOKENS", + "id": "d20c06b0-aafe-4230-b2c7-200f4069355e", + "text": " 212°F is equivalent to 100°C. \n\nFahrenheit and Celsius are two temperature scales commonly used in everyday life. The Fahrenheit scale is based on 32°F for the freezing point of water and 212°F for the boiling point of water. On the other hand, the Celsius scale uses 0°C and 100°C as the freezing and boiling points of water, respectively. \n\nTo convert from Fahrenheit to Celsius, we subtract 32 from the Fahrenheit temperature and multiply the result", + } + ], + "id": "e77422c8-fbbf-4e17-afeb-c758425c9f97", + "prompt": "What is 212 degrees Fahrenheit converted to Celsius?", + }, + ], + "does-not-exist::": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "f4908827-3db9-4742-9103-2bbc34578b03", + "x-amzn-ErrorType": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 400, + {"message": "The provided model identifier is invalid."}, + ], + "ai21.j2-mid-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "9021791d-3797-493d-9277-e33aa6f6d544", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-embed-g1-text-02::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "73328313-506e-4da8-af0f-51017fa6ca3f", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-embed-text-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "aece6ad7-e2ff-443b-a953-ba7d385fd0cc", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-text-express-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "15b39c8b-8e85-42c9-9623-06720301bda3", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "anthropic.claude-instant-v1::Human: Invalid Token Assistant:": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "37396f55-b721-4bae-9461-4c369f5a080d", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "cohere.command-text-v14::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], } MODEL_PATH_RE = re.compile(r"/model/([^/]+)/invoke") @@ -3346,7 +3415,7 @@ def simple_get(self): headers, response = ({}, "") for k, v in RESPONSES.items(): if prompt.startswith(k): - headers, response = v + headers, status_code, response = v break else: # If no matches found self.send_response(500) @@ -3355,7 +3424,7 @@ def simple_get(self): return # Send response code - self.send_response(200) + self.send_response(status_code) # Send headers for k, v in headers.items(): @@ -3368,7 +3437,7 @@ def simple_get(self): def extract_shortened_prompt(content, model): - prompt = content.get("inputText", None) or content.get("prompt", None) + prompt = content.get("inputText", "") or content.get("prompt", "") prompt = "::".join((model, prompt)) # Prepend model name to prompt key to keep separate copies return prompt.lstrip().split("\n")[0] @@ -3383,6 +3452,9 @@ def __init__(self, handler=simple_get, port=None, *args, **kwargs): if __name__ == "__main__": + # Use this to sort dict for easier future incremental updates + print("RESPONSES = %s" % dict(sorted(RESPONSES.items(), key=lambda i: (i[1][1], i[0])))) + with MockExternalBedrockServer() as server: print("MockExternalBedrockServer serving on port %s" % str(server.port)) while True: diff --git a/tests/external_botocore/_test_bedrock_chat_completion.py b/tests/external_botocore/_test_bedrock_chat_completion.py index fc69b1ff89..1a66d74e43 100644 --- a/tests/external_botocore/_test_bedrock_chat_completion.py +++ b/tests/external_botocore/_test_bedrock_chat_completion.py @@ -16,13 +16,13 @@ "transaction_id": None, "span_id": "span-id", "trace_id": "trace-id", - "request_id": "660d4de9-6804-460e-8556-4ab2a019d1e3", + "request_id": "03524118-8d77-430f-9e08-63b5c03a40cf", "api_key_last_four_digits": "CRET", "duration": None, # Response time varies each test run "request.model": "amazon.titan-text-express-v1", "response.model": "amazon.titan-text-express-v1", - "response.usage.completion_tokens": 55, - "response.usage.total_tokens": 67, + "response.usage.completion_tokens": 75, + "response.usage.total_tokens": 87, "response.usage.prompt_tokens": 12, "request.temperature": 0.7, "request.max_tokens": 100, @@ -38,7 +38,7 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "660d4de9-6804-460e-8556-4ab2a019d1e3", + "request_id": "03524118-8d77-430f-9e08-63b5c03a40cf", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -57,7 +57,7 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "660d4de9-6804-460e-8556-4ab2a019d1e3", + "request_id": "03524118-8d77-430f-9e08-63b5c03a40cf", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -81,7 +81,7 @@ "transaction_id": None, "span_id": "span-id", "trace_id": "trace-id", - "request_id": "3bf1bb6b-b6f0-4901-85a1-2fa0e814440e", + "request_id": "c863d9fc-888b-421c-a175-ac5256baec62", "response_id": "1234", "api_key_last_four_digits": "CRET", "duration": None, # Response time varies each test run @@ -101,7 +101,7 @@ "id": "1234-0", "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "3bf1bb6b-b6f0-4901-85a1-2fa0e814440e", + "request_id": "c863d9fc-888b-421c-a175-ac5256baec62", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -120,7 +120,7 @@ "id": "1234-1", "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "3bf1bb6b-b6f0-4901-85a1-2fa0e814440e", + "request_id": "c863d9fc-888b-421c-a175-ac5256baec62", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -144,7 +144,7 @@ "transaction_id": None, "span_id": "span-id", "trace_id": "trace-id", - "request_id": "f354b9a7-9eac-4f50-a8d7-7d5d23566176", + "request_id": "7b0b37c6-85fb-4664-8f5b-361ca7b1aa18", "api_key_last_four_digits": "CRET", "duration": None, # Response time varies each test run "request.model": "anthropic.claude-instant-v1", @@ -163,7 +163,7 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "f354b9a7-9eac-4f50-a8d7-7d5d23566176", + "request_id": "7b0b37c6-85fb-4664-8f5b-361ca7b1aa18", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -182,11 +182,11 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "f354b9a7-9eac-4f50-a8d7-7d5d23566176", + "request_id": "7b0b37c6-85fb-4664-8f5b-361ca7b1aa18", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, - "content": " Here are the step-by-step workings:\n1) 212 degrees Fahrenheit \n2) To convert to Celsius, use the formula: C = (F - 32) * 5/9\n3) Plug in the values: C = (212 - 32) * 5/9 = 100 * 5/9 = 100 degrees Celsius\n\nSo, 212 degrees Fahrenheit converted to Celsius is", + "content": " Okay, here are the conversion steps:\n212 degrees Fahrenheit\n- Subtract 32 from 212 to get 180 (to convert from Fahrenheit to Celsius scale)\n- Multiply by 5/9 (because the formula is °C = (°F - 32) × 5/9)\n- 180 × 5/9 = 100\n\nSo 212 degrees Fahrenheit c", "role": "assistant", "completion_id": None, "sequence": 1, @@ -206,7 +206,7 @@ "transaction_id": None, "span_id": "span-id", "trace_id": "trace-id", - "request_id": "c5188fb5-dc58-4cbe-948d-af173c69ce0d", + "request_id": "e77422c8-fbbf-4e17-afeb-c758425c9f97", "response_id": None, # UUID that varies with each run "api_key_last_four_digits": "CRET", "duration": None, # Response time varies each test run @@ -226,7 +226,7 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "c5188fb5-dc58-4cbe-948d-af173c69ce0d", + "request_id": "e77422c8-fbbf-4e17-afeb-c758425c9f97", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, @@ -245,11 +245,11 @@ "id": None, # UUID that varies with each run "appName": "Python Agent Test (external_botocore)", "conversation_id": "my-awesome-id", - "request_id": "c5188fb5-dc58-4cbe-948d-af173c69ce0d", + "request_id": "e77422c8-fbbf-4e17-afeb-c758425c9f97", "span_id": "span-id", "trace_id": "trace-id", "transaction_id": None, - "content": " To convert 212 degrees Fahrenheit to Celsius, we can use the conversion factor that Celsius is equal to (Fahrenheit - 32) x 5/9. \\n\\nApplying this formula, we have:\\n212°F = (212°F - 32) x 5/9\\n= (180) x 5/9\\n= 100°C.\\n\\nTherefore, 212 degrees F", + "content": " 212°F is equivalent to 100°C. \n\nFahrenheit and Celsius are two temperature scales commonly used in everyday life. The Fahrenheit scale is based on 32°F for the freezing point of water and 212°F for the boiling point of water. On the other hand, the C", "role": "assistant", "completion_id": None, "sequence": 1, @@ -260,3 +260,58 @@ ), ], } + +chat_completion_expected_client_errors = { + "amazon.titan-text-express-v1": { + "conversation_id": "my-awesome-id", + "request_id": "15b39c8b-8e85-42c9-9623-06720301bda3", + "api_key_last_four_digits": "-KEY", + "request.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "ai21.j2-mid-v1": { + "conversation_id": "my-awesome-id", + "request_id": "9021791d-3797-493d-9277-e33aa6f6d544", + "api_key_last_four_digits": "-KEY", + "request.model": "ai21.j2-mid-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "anthropic.claude-instant-v1": { + "conversation_id": "my-awesome-id", + "request_id": "37396f55-b721-4bae-9461-4c369f5a080d", + "api_key_last_four_digits": "-KEY", + "request.model": "anthropic.claude-instant-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "cohere.command-text-v14": { + "conversation_id": "my-awesome-id", + "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "api_key_last_four_digits": "-KEY", + "request.model": "cohere.command-text-v14", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, +} diff --git a/tests/external_botocore/_test_bedrock_embeddings.py b/tests/external_botocore/_test_bedrock_embeddings.py index fe4b4b839a..8fb2ceecee 100644 --- a/tests/external_botocore/_test_bedrock_embeddings.py +++ b/tests/external_botocore/_test_bedrock_embeddings.py @@ -18,7 +18,7 @@ "duration": None, # Response time varies each test run "response.model": "amazon.titan-embed-text-v1", "request.model": "amazon.titan-embed-text-v1", - "request_id": "75f1d3fe-6cde-4cf5-bdaf-7101f746ccfe", + "request_id": "11233989-07e8-4ecb-9ba6-79601ba6d8cc", "response.usage.total_tokens": 6, "response.usage.prompt_tokens": 6, "vendor": "bedrock", @@ -40,7 +40,7 @@ "duration": None, # Response time varies each test run "response.model": "amazon.titan-embed-g1-text-02", "request.model": "amazon.titan-embed-g1-text-02", - "request_id": "f7e78265-6b7c-4b3a-b750-0c1d00347258", + "request_id": "b10ac895-eae3-4f07-b926-10b2866c55ed", "response.usage.total_tokens": 6, "response.usage.prompt_tokens": 6, "vendor": "bedrock", @@ -49,3 +49,26 @@ ), ] } + +embedding_expected_client_errors = { + "amazon.titan-embed-text-v1": { + "request_id": "aece6ad7-e2ff-443b-a953-ba7d385fd0cc", + "api_key_last_four_digits": "-KEY", + "request.model": "amazon.titan-embed-text-v1", + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "amazon.titan-embed-g1-text-02": { + "request_id": "73328313-506e-4da8-af0f-51017fa6ca3f", + "api_key_last_four_digits": "-KEY", + "request.model": "amazon.titan-embed-g1-text-02", + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, +} diff --git a/tests/external_botocore/conftest.py b/tests/external_botocore/conftest.py index 67a2058239..8b19d3ce75 100644 --- a/tests/external_botocore/conftest.py +++ b/tests/external_botocore/conftest.py @@ -14,6 +14,7 @@ import json import os +import re import pytest from _mock_external_bedrock_server import ( @@ -92,56 +93,56 @@ def bedrock_server(): # Apply function wrappers to record data wrap_function_wrapper( - "botocore.client", "BaseClient._make_api_call", wrap_botocore_client_BaseClient__make_api_call + "botocore.endpoint", "Endpoint._do_get_response", wrap_botocore_endpoint_Endpoint__do_get_response ) yield client # Run tests # Write responses to audit log + bedrock_audit_log_contents = dict(sorted(BEDROCK_AUDIT_LOG_CONTENTS.items(), key=lambda i: (i[1][1], i[0]))) with open(BEDROCK_AUDIT_LOG_FILE, "w") as audit_log_fp: - json.dump(BEDROCK_AUDIT_LOG_CONTENTS, fp=audit_log_fp, indent=4) + json.dump(bedrock_audit_log_contents, fp=audit_log_fp, indent=4) # Intercept outgoing requests and log to file for mocking -RECORDED_HEADERS = set(["x-amzn-requestid", "content-type"]) +RECORDED_HEADERS = set(["x-amzn-requestid", "x-amzn-errortype", "content-type"]) -def wrap_botocore_client_BaseClient__make_api_call(wrapped, instance, args, kwargs): - from io import BytesIO - - from botocore.response import StreamingBody - - params = bind_make_api_call_params(*args, **kwargs) - if not params: +def wrap_botocore_endpoint_Endpoint__do_get_response(wrapped, instance, args, kwargs): + request = bind__do_get_response(*args, **kwargs) + if not request: return wrapped(*args, **kwargs) - body = json.loads(params["body"]) - model = params["modelId"] + body = json.loads(request.body) + + match = re.search(r"/model/([0-9a-zA-Z.-]+)/", request.url) + model = match.group(1) prompt = extract_shortened_prompt(body, model) # Send request result = wrapped(*args, **kwargs) - # Intercept body data, and replace stream - streamed_body = result["body"].read() - result["body"] = StreamingBody(BytesIO(streamed_body), len(streamed_body)) + # Unpack response + success, exception = result + response = (success or exception)[0] # Clean up data - data = json.loads(streamed_body.decode("utf-8")) - headers = dict(result["ResponseMetadata"]["HTTPHeaders"].items()) + data = json.loads(response.content.decode("utf-8")) + headers = dict(response.headers.items()) headers = dict( filter( - lambda k: k[0] in RECORDED_HEADERS or k[0].startswith("x-ratelimit"), + lambda k: k[0].lower() in RECORDED_HEADERS or k[0].startswith("x-ratelimit"), headers.items(), ) ) + status_code = response.status_code # Log response - BEDROCK_AUDIT_LOG_CONTENTS[prompt] = headers, data # Append response data to audit log + BEDROCK_AUDIT_LOG_CONTENTS[prompt] = headers, status_code, data # Append response data to audit log return result -def bind_make_api_call_params(operation_name, api_params): - return api_params +def bind__do_get_response(request, operation_model, context): + return request @pytest.fixture(scope="session") diff --git a/tests/external_botocore/test_bedrock_chat_completion.py b/tests/external_botocore/test_bedrock_chat_completion.py index 995a931633..a1eb881cc1 100644 --- a/tests/external_botocore/test_bedrock_chat_completion.py +++ b/tests/external_botocore/test_bedrock_chat_completion.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import botocore.exceptions + import copy import json from io import BytesIO @@ -20,18 +22,25 @@ from _test_bedrock_chat_completion import ( chat_completion_expected_events, chat_completion_payload_templates, + chat_completion_expected_client_errors, ) from testing_support.fixtures import ( + dt_enabled, override_application_settings, reset_core_stats_engine, ) from testing_support.validators.validate_ml_event_count import validate_ml_event_count from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) from newrelic.api.background_task import background_task from newrelic.api.time_trace import current_trace from newrelic.api.transaction import add_custom_attribute, current_transaction +from newrelic.common.object_names import callable_name @pytest.fixture(scope="session", params=[False, True], ids=["Bytes", "Stream"]) def is_file_payload(request): @@ -85,6 +94,11 @@ def expected_events_no_convo_id(model_id): return events +@pytest.fixture(scope="module") +def expected_client_error(model_id): + return chat_completion_expected_client_errors[model_id] + + _test_bedrock_chat_completion_prompt = "What is 212 degrees Fahrenheit converted to Celsius?" @@ -154,3 +168,62 @@ def test_bedrock_chat_completion_outside_txn(set_trace_info, exercise_model): def test_bedrock_chat_completion_disabled_settings(set_trace_info, exercise_model): set_trace_info() exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + + +_client_error = botocore.exceptions.ClientError +_client_error_name = callable_name(_client_error) + + +@validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "conversation_id": "my-awesome-id", + "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", + "api_key_last_four_digits": "CRET", + "request.model": "does-not-exist", + "vendor": "Bedrock", + "ingest_source": "Python", + "http.statusCode": 400, + "error.message": "The provided model identifier is invalid.", + "error.code": "ValidationException", + }, + }, +) +@background_task() +def test_bedrock_chat_completion_error_invalid_model(bedrock_server, set_trace_info): + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + with pytest.raises(_client_error): + bedrock_server.invoke_model( + body=b"{}", + modelId="does-not-exist", + accept="application/json", + contentType="application/json", + ) + + +@dt_enabled +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_incorrect_access_key(monkeypatch, bedrock_server, exercise_model, set_trace_info, expected_client_error): + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": expected_client_error, + }, + ) + @background_task() + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): # not sure where this exception actually comes from + set_trace_info() + add_custom_attribute("conversation_id", "my-awesome-id") + exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) + + _test() diff --git a/tests/external_botocore/test_bedrock_embeddings.py b/tests/external_botocore/test_bedrock_embeddings.py index 022eb07599..c374dd69c5 100644 --- a/tests/external_botocore/test_bedrock_embeddings.py +++ b/tests/external_botocore/test_bedrock_embeddings.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy +import botocore.exceptions + import json from io import BytesIO import pytest -from testing_support.fixtures import ( # override_application_settings, +from testing_support.fixtures import ( + dt_enabled, override_application_settings, reset_core_stats_engine, ) @@ -26,10 +28,17 @@ from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) from newrelic.api.background_task import background_task -from _test_bedrock_embeddings import embedding_expected_events, embedding_payload_templates +from _test_bedrock_embeddings import embedding_expected_events, embedding_payload_templates, embedding_expected_client_errors + +from newrelic.common.object_names import callable_name + disabled_ml_insights_settings = {"ml_insights_events.enabled": False} @@ -76,6 +85,11 @@ def expected_events(model_id): return embedding_expected_events[model_id] +@pytest.fixture(scope="module") +def expected_client_error(model_id): + return embedding_expected_client_errors[model_id] + + @reset_core_stats_engine() def test_bedrock_embedding(set_trace_info, exercise_model, expected_events): @validate_ml_events(expected_events) @@ -101,6 +115,10 @@ def test_bedrock_embedding_outside_txn(exercise_model): exercise_model(prompt="This is an embedding test.") +_client_error = botocore.exceptions.ClientError +_client_error_name = callable_name(_client_error) + + @override_application_settings(disabled_ml_insights_settings) @reset_core_stats_engine() @validate_ml_event_count(count=0) @@ -115,3 +133,25 @@ def test_bedrock_embedding_outside_txn(exercise_model): def test_bedrock_embedding_disabled_settings(set_trace_info, exercise_model): set_trace_info() exercise_model(prompt="This is an embedding test.") + + +@dt_enabled +@reset_core_stats_engine() +def test_bedrock_embedding_error_incorrect_access_key(monkeypatch, bedrock_server, exercise_model, set_trace_info, expected_client_error): + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": expected_client_error, + }, + ) + @background_task() + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): # not sure where this exception actually comes from + set_trace_info() + exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) + + _test() diff --git a/tests/external_botocore/test_botocore_dynamodb.py b/tests/external_botocore/test_botocore_dynamodb.py index 6ce9f12c33..932fb1743a 100644 --- a/tests/external_botocore/test_botocore_dynamodb.py +++ b/tests/external_botocore/test_botocore_dynamodb.py @@ -80,7 +80,7 @@ background_task=True, ) @background_task() -@moto.mock_dynamodb2 +@moto.mock_dynamodb def test_dynamodb(): session = botocore.session.get_session() client = session.create_client( diff --git a/tests/framework_starlette/test_application.py b/tests/framework_starlette/test_application.py index 7d36d66ccc..bd89bb9a9c 100644 --- a/tests/framework_starlette/test_application.py +++ b/tests/framework_starlette/test_application.py @@ -17,13 +17,21 @@ import pytest import starlette from testing_support.fixtures import override_ignore_status_codes +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.common.object_names import callable_name -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from newrelic.common.package_version_utils import get_package_version_tuple + +starlette_version = get_package_version_tuple("starlette")[:3] -starlette_version = tuple(int(x) for x in starlette.__version__.split(".")) @pytest.fixture(scope="session") def target_application(): @@ -78,6 +86,7 @@ def test_application_non_async(target_application, app_name): response = app.get("/non_async") assert response.status == 200 + # Starting in Starlette v0.20.1, the ExceptionMiddleware class # has been moved to the starlette.middleware.exceptions from # starlette.exceptions @@ -96,8 +105,10 @@ def test_application_non_async(target_application, app_name): ), ) + @pytest.mark.parametrize( - "app_name, transaction_name", middleware_test, + "app_name, transaction_name", + middleware_test, ) def test_application_nonexistent_route(target_application, app_name, transaction_name): @validate_transaction_metrics( @@ -117,10 +128,6 @@ def _test(): def test_exception_in_middleware(target_application, app_name): app = target_application[app_name] - from starlette import __version__ as version - - starlette_version = tuple(int(v) for v in version.split(".")) - # Starlette >=0.15 and <0.17 raises an exception group instead of reraising the ValueError # This only occurs on Python versions >=3.8 if sys.version_info[0:2] > (3, 7) and starlette_version >= (0, 15, 0) and starlette_version < (0, 17, 0): @@ -272,9 +279,8 @@ def _test(): ), ) -@pytest.mark.parametrize( - "app_name,scoped_metrics", middleware_test_exception -) + +@pytest.mark.parametrize("app_name,scoped_metrics", middleware_test_exception) def test_starlette_http_exception(target_application, app_name, scoped_metrics): @validate_transaction_errors(errors=["starlette.exceptions:HTTPException"]) @validate_transaction_metrics( diff --git a/tests/framework_starlette/test_bg_tasks.py b/tests/framework_starlette/test_bg_tasks.py index 5e30fe32e5..9ad8fe61be 100644 --- a/tests/framework_starlette/test_bg_tasks.py +++ b/tests/framework_starlette/test_bg_tasks.py @@ -15,7 +15,6 @@ import sys import pytest -from starlette import __version__ from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) @@ -23,7 +22,9 @@ validate_transaction_metrics, ) -starlette_version = tuple(int(x) for x in __version__.split(".")) +from newrelic.common.package_version_utils import get_package_version_tuple + +starlette_version = get_package_version_tuple("starlette")[:3] try: from starlette.middleware import Middleware # noqa: F401 diff --git a/tox.ini b/tox.ini index 720c301dcd..bb6102d895 100644 --- a/tox.ini +++ b/tox.ini @@ -252,12 +252,11 @@ deps = datastore_redis-redis0400: redis<4.1 external_botocore-botocorelatest: botocore external_botocore-botocorelatest: boto3 - external_botocore-botocorelatest: moto external_botocore-botocore128: botocore<1.29 external_botocore-botocore0125: botocore<1.26 - external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0 + external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam,sqs] external_botocore-py27: rsa<4.7.1 - external_botocore-py27: moto[awslambda,ec2,iam]<2.0 + external_botocore-py27: moto[awslambda,ec2,iam,sqs]<2.0 external_feedparser-feedparser05: feedparser<6 external_feedparser-feedparser06: feedparser<7 external_httplib2: httplib2<1.0